==================================================================================================== ========================================= Pod describe ========================================= ==================================================================================================== Name: lodemon-f8459c597-6tj6f Namespace: xlou Priority: 0 Service Account: k8s-svc-acct-lodemon Node: gke-xlou-cdm-default-pool-1b0769c8-7vmj/10.142.0.111 Start Time: Fri, 19 Apr 2024 18:12:27 +0000 Labels: app=lodemon app.kubernetes.io/name=lodemon pod-template-hash=f8459c597 skaffold.dev/run-id=aa6e33dd-7c59-424f-a814-29728b191884 Annotations: Status: Running IP: 10.106.42.187 IPs: IP: 10.106.42.187 Controlled By: ReplicaSet/lodemon-f8459c597 Containers: lodemon: Container ID: containerd://ecb5b54ec9783dceabfff0c60763bc5675e4e656b23c24fd5b7134dfed26afed Image: gcr.io/engineeringpit/lodestar-images/lodestarbox:master-stable Image ID: gcr.io/engineeringpit/lodestar-images/lodestarbox@sha256:03f76e6326d713f4d73dc474e0e25fbbbaecc7c1d8c0590bfd94d68ac0d98c97 Port: 8080/TCP Host Port: 0/TCP Command: python3 Args: /lodestar/scripts/lodemon_run.py -W default State: Running Started: Fri, 19 Apr 2024 18:12:27 +0000 Ready: True Restart Count: 0 Limits: cpu: 2 memory: 2Gi Requests: cpu: 1 memory: 1Gi Liveness: exec [cat /tmp/lodemon_alive] delay=20s timeout=1s period=10s #success=1 #failure=3 Readiness: exec [cat /tmp/lodemon_alive] delay=20s timeout=1s period=10s #success=1 #failure=3 Environment: SKAFFOLD_PROFILE: medium Mounts: /lodestar/config/config.yaml from config (rw,path="config.yaml") /lodestar/config/deployments_to_monitor.yaml from deployments-config (rw,path="deployments_to_monitor.yaml") /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-5wkz6 (ro) Conditions: Type Status Initialized True Ready True ContainersReady True PodScheduled True Volumes: config: Type: ConfigMap (a volume populated by a ConfigMap) Name: lodemon-config Optional: false deployments-config: Type: ConfigMap (a volume populated by a ConfigMap) Name: lodemon-deployments Optional: false kube-api-access-5wkz6: Type: Projected (a volume that contains injected data from multiple sources) TokenExpirationSeconds: 3607 ConfigMapName: kube-root-ca.crt ConfigMapOptional: DownwardAPI: true QoS Class: Burstable Node-Selectors: Tolerations: node.kubernetes.io/not-ready:NoExecute op=Exists for 300s node.kubernetes.io/unreachable:NoExecute op=Exists for 300s Events: ==================================================================================================== =========================================== Pod logs =========================================== ==================================================================================================== * Serving Flask app 'lodemon_run' * Debug mode: off WARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead. * Running on all addresses (0.0.0.0) * Running on http://127.0.0.1:8080 * Running on http://10.106.42.187:8080 Press CTRL+C to quit 19:13:24 INFO 19:13:24 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:13:24 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:13:24 INFO [loop_until]: OK (rc = 0) 19:13:24 DEBUG --- stdout --- 19:13:24 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:13:24 DEBUG --- stderr --- 19:13:24 DEBUG 19:13:24 INFO 19:13:24 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:13:24 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:13:24 INFO [loop_until]: OK (rc = 0) 19:13:24 DEBUG --- stdout --- 19:13:24 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:13:24 DEBUG --- stderr --- 19:13:24 DEBUG 19:13:24 INFO 19:13:24 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:13:24 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:13:24 INFO [loop_until]: OK (rc = 0) 19:13:24 DEBUG --- stdout --- 19:13:24 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:13:24 DEBUG --- stderr --- 19:13:24 DEBUG 19:13:24 INFO 19:13:24 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:13:24 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:13:24 INFO [loop_until]: OK (rc = 0) 19:13:24 DEBUG --- stdout --- 19:13:24 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:13:24 DEBUG --- stderr --- 19:13:24 DEBUG 19:13:24 INFO 19:13:24 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:13:24 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:13:24 INFO [loop_until]: OK (rc = 0) 19:13:24 DEBUG --- stdout --- 19:13:24 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:13:24 DEBUG --- stderr --- 19:13:24 DEBUG 19:13:25 INFO 19:13:25 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:13:25 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:13:25 INFO [loop_until]: OK (rc = 0) 19:13:25 DEBUG --- stdout --- 19:13:25 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:13:25 DEBUG --- stderr --- 19:13:25 DEBUG 19:13:25 INFO 19:13:25 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:13:25 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:13:25 INFO [loop_until]: OK (rc = 0) 19:13:25 DEBUG --- stdout --- 19:13:25 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:13:25 DEBUG --- stderr --- 19:13:25 DEBUG 19:13:25 INFO 19:13:25 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:13:25 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:13:25 INFO [loop_until]: OK (rc = 0) 19:13:25 DEBUG --- stdout --- 19:13:25 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:13:25 DEBUG --- stderr --- 19:13:25 DEBUG 19:13:25 INFO 19:13:25 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:13:25 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:13:25 INFO [loop_until]: OK (rc = 0) 19:13:25 DEBUG --- stdout --- 19:13:25 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:13:25 DEBUG --- stderr --- 19:13:25 DEBUG 19:13:25 INFO 19:13:25 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:13:25 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:13:25 INFO [loop_until]: OK (rc = 0) 19:13:25 DEBUG --- stdout --- 19:13:25 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:13:25 DEBUG --- stderr --- 19:13:25 DEBUG 19:13:25 INFO 19:13:25 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:13:25 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:13:25 INFO [loop_until]: OK (rc = 0) 19:13:25 DEBUG --- stdout --- 19:13:25 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:13:25 DEBUG --- stderr --- 19:13:25 DEBUG 19:13:25 INFO 19:13:25 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:13:25 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:13:25 INFO [loop_until]: OK (rc = 0) 19:13:25 DEBUG --- stdout --- 19:13:25 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:13:25 DEBUG --- stderr --- 19:13:25 DEBUG 19:13:25 INFO 19:13:25 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:13:25 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:13:26 INFO [loop_until]: OK (rc = 0) 19:13:26 DEBUG --- stdout --- 19:13:26 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:13:26 DEBUG --- stderr --- 19:13:26 DEBUG 19:13:26 INFO 19:13:26 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:13:26 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:13:26 INFO [loop_until]: OK (rc = 0) 19:13:26 DEBUG --- stdout --- 19:13:26 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:13:26 DEBUG --- stderr --- 19:13:26 DEBUG 19:13:26 INFO 19:13:26 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:13:26 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:13:26 INFO [loop_until]: OK (rc = 0) 19:13:26 DEBUG --- stdout --- 19:13:26 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:13:26 DEBUG --- stderr --- 19:13:26 DEBUG 19:13:26 INFO 19:13:26 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:13:26 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:13:26 INFO [loop_until]: OK (rc = 0) 19:13:26 DEBUG --- stdout --- 19:13:26 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:13:26 DEBUG --- stderr --- 19:13:26 DEBUG 19:13:26 INFO 19:13:26 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:13:26 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:13:26 INFO [loop_until]: OK (rc = 0) 19:13:26 DEBUG --- stdout --- 19:13:26 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:13:26 DEBUG --- stderr --- 19:13:26 DEBUG 19:13:26 INFO 19:13:26 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:13:26 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:13:26 INFO [loop_until]: OK (rc = 0) 19:13:26 DEBUG --- stdout --- 19:13:26 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:13:26 DEBUG --- stderr --- 19:13:26 DEBUG 19:13:26 INFO 19:13:26 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:13:26 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:13:26 INFO [loop_until]: OK (rc = 0) 19:13:26 DEBUG --- stdout --- 19:13:26 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:13:26 DEBUG --- stderr --- 19:13:26 DEBUG 19:13:26 INFO 19:13:26 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:13:26 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:13:27 INFO [loop_until]: OK (rc = 0) 19:13:27 DEBUG --- stdout --- 19:13:27 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:13:27 DEBUG --- stderr --- 19:13:27 DEBUG 19:13:27 INFO 19:13:27 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:13:27 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:13:27 INFO [loop_until]: OK (rc = 0) 19:13:27 DEBUG --- stdout --- 19:13:27 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:13:27 DEBUG --- stderr --- 19:13:27 DEBUG 19:13:27 INFO 19:13:27 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:13:27 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:13:27 INFO [loop_until]: OK (rc = 0) 19:13:27 DEBUG --- stdout --- 19:13:27 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:13:27 DEBUG --- stderr --- 19:13:27 DEBUG 19:13:27 INFO 19:13:27 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:13:27 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:13:27 INFO [loop_until]: OK (rc = 0) 19:13:27 DEBUG --- stdout --- 19:13:27 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:13:27 DEBUG --- stderr --- 19:13:27 DEBUG 19:13:27 INFO 19:13:27 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:13:27 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:13:27 INFO [loop_until]: OK (rc = 0) 19:13:27 DEBUG --- stdout --- 19:13:27 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:13:27 DEBUG --- stderr --- 19:13:27 DEBUG 19:13:27 INFO 19:13:27 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:13:27 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:13:27 INFO [loop_until]: OK (rc = 0) 19:13:27 DEBUG --- stdout --- 19:13:27 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:13:27 DEBUG --- stderr --- 19:13:27 DEBUG 19:13:27 INFO 19:13:27 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:13:27 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:13:27 INFO [loop_until]: OK (rc = 0) 19:13:27 DEBUG --- stdout --- 19:13:27 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:13:27 DEBUG --- stderr --- 19:13:27 DEBUG 19:13:27 INFO 19:13:27 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:13:27 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:13:27 INFO [loop_until]: OK (rc = 0) 19:13:27 DEBUG --- stdout --- 19:13:27 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:13:27 DEBUG --- stderr --- 19:13:27 DEBUG 19:13:27 INFO Initializing monitoring instance threads 19:13:27 DEBUG Monitoring instance thread list: [, , , , , , , , , , , , , , , , , , , , , , , , , , , , ] 19:13:27 INFO Starting instance threads 19:13:27 INFO 19:13:27 INFO Thread started 19:13:27 INFO [loop_until]: kubectl --namespace=xlou top node 19:13:27 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:13:27 INFO 19:13:27 INFO Thread started 19:13:27 INFO [loop_until]: kubectl --namespace=xlou top pods 19:13:27 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:13:27 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550407" 19:13:27 INFO Thread started 19:13:27 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550407" 19:13:27 INFO Thread started 19:13:27 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550407" 19:13:27 INFO Thread started 19:13:27 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550407" 19:13:27 INFO Thread started 19:13:27 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713550407" 19:13:27 INFO Thread started 19:13:27 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713550407" 19:13:27 INFO Thread started 19:13:27 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550407" 19:13:27 INFO Thread started 19:13:27 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550407" 19:13:27 INFO Thread started 19:13:27 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550407" 19:13:27 INFO Thread started 19:13:27 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550407" 19:13:27 INFO Thread started 19:13:27 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550407" 19:13:27 INFO Thread started 19:13:27 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550407" 19:13:27 INFO Thread started 19:13:27 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550407" 19:13:27 INFO Thread started 19:13:27 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550407" 19:13:27 INFO Thread started 19:13:27 INFO [http_cmd]: http status code OK 19:13:27 INFO [http_cmd]: http status code OK 19:13:27 INFO [http_cmd]: http status code OK 19:13:27 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550407" 19:13:27 INFO [http_cmd]: http status code OK 19:13:27 INFO [http_cmd]: http status code OK 19:13:27 INFO Thread started 19:13:27 INFO [http_cmd]: http status code OK 19:13:27 DEBUG --- status code --- 19:13:27 DEBUG --- status code --- 19:13:27 DEBUG --- status code --- 19:13:28 DEBUG --- status code --- 19:13:28 INFO [http_cmd]: http status code OK 19:13:28 INFO [http_cmd]: http status code OK 19:13:28 INFO [http_cmd]: http status code OK 19:13:28 INFO [http_cmd]: http status code OK 19:13:28 INFO [http_cmd]: http status code OK 19:13:28 DEBUG --- status code --- 19:13:28 INFO [http_cmd]: http status code OK 19:13:28 INFO [http_cmd]: http status code OK 19:13:28 DEBUG --- status code --- 19:13:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550408" 19:13:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:28 INFO Thread started 19:13:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:28 DEBUG --- status code --- 19:13:28 DEBUG --- status code --- 19:13:28 DEBUG --- status code --- 19:13:28 DEBUG --- status code --- 19:13:28 DEBUG --- status code --- 19:13:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:28 DEBUG --- http response --- 19:13:28 DEBUG --- status code --- 19:13:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:28 DEBUG --- http response --- 19:13:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550408" 19:13:28 DEBUG --- http response --- 19:13:28 INFO Thread started 19:13:28 DEBUG --- http response --- 19:13:28 DEBUG --- http response --- 19:13:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:28 DEBUG --- status code --- 19:13:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:13:28 INFO [http_cmd]: http status code OK 19:13:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:28 INFO [http_cmd]: http status code OK 19:13:28 DEBUG --- http response --- 19:13:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (161 lines): 19:13:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (161 lines): 19:13:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550408" 19:13:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (161 lines): 19:13:28 INFO Thread started 19:13:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:13:28 DEBUG --- http response --- 19:13:28 DEBUG --- http response --- 19:13:28 DEBUG --- http response --- 19:13:28 DEBUG --- http response --- 19:13:28 DEBUG --- http response --- 19:13:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:28 DEBUG ----- output ----- 19:13:28 DEBUG --- status code --- 19:13:28 DEBUG --- http response --- 19:13:28 DEBUG --- status code --- 19:13:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (161 lines): 19:13:28 DEBUG ----- output ----- 19:13:28 DEBUG ----- output ----- 19:13:28 DEBUG ----- output ----- 19:13:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550408" 19:13:28 DEBUG ----- output ----- 19:13:28 INFO Thread started 19:13:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:13:28 INFO [loop_until]: OK (rc = 0) 19:13:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:13:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:13:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:13:28 DEBUG --- http response --- 19:13:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:13:28 DEBUG { 19:13:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:28 INFO [http_cmd]: http status code OK 19:13:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:13:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:28 DEBUG ----- output ----- 19:13:28 DEBUG { 19:13:28 DEBUG { 19:13:28 DEBUG { 19:13:28 DEBUG { 19:13:28 INFO [http_cmd]: http status code OK 19:13:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550408" 19:13:28 DEBUG ----- output ----- 19:13:28 INFO Thread started 19:13:28 DEBUG --- stdout --- 19:13:28 DEBUG ----- output ----- 19:13:28 DEBUG ----- output ----- 19:13:28 DEBUG ----- output ----- 19:13:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:13:28 DEBUG ----- output ----- 19:13:28 DEBUG "status": "success", 19:13:28 DEBUG --- http response --- 19:13:28 DEBUG --- status code --- 19:13:28 DEBUG ----- output ----- 19:13:28 DEBUG --- http response --- 19:13:28 DEBUG { 19:13:28 DEBUG "status": "success", 19:13:28 DEBUG "status": "success", 19:13:28 DEBUG "status": "success", 19:13:28 DEBUG "status": "success", 19:13:28 DEBUG --- status code --- 19:13:28 DEBUG { 19:13:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550408" 19:13:28 INFO Thread started 19:13:28 DEBUG { 19:13:28 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 7m 1086Mi am-767f6b67b5-qcqz2 10m 2613Mi am-767f6b67b5-vbltz 14m 1073Mi ds-cts-0 5m 10752Mi ds-cts-1 5m 11018Mi ds-cts-2 5m 10762Mi ds-idrepo-0 9m 3440Mi ds-idrepo-1 9m 3412Mi ds-idrepo-2 8m 3486Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1174Mi idm-b85f9d978-wnhxb 4m 1217Mi lodemon-f8459c597-6tj6f 2m 54Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 1m 2619Mi 19:13:28 DEBUG { 19:13:28 DEBUG { 19:13:28 DEBUG ----- output ----- 19:13:28 DEBUG { 19:13:28 DEBUG "data": { 19:13:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:13:28 DEBUG { 19:13:28 DEBUG "status": "success", 19:13:28 DEBUG "data": { 19:13:28 DEBUG "data": { 19:13:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:13:28 DEBUG "data": { 19:13:28 INFO [http_cmd]: http status code OK 19:13:28 DEBUG "data": { 19:13:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:28 DEBUG "status": "success", 19:13:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713550408" 19:13:28 DEBUG "status": "success", 19:13:28 INFO Thread started 19:13:28 INFO [http_cmd]: http status code OK 19:13:28 DEBUG --- stderr --- 19:13:28 INFO [loop_until]: OK (rc = 0) 19:13:28 DEBUG "status": "success", 19:13:28 DEBUG "status": "success", 19:13:28 DEBUG { 19:13:28 DEBUG "status": "success", 19:13:28 DEBUG "resultType": "vector", 19:13:28 DEBUG --- http response --- 19:13:28 DEBUG ----- output ----- 19:13:28 DEBUG "status": "success", 19:13:28 DEBUG "data": { 19:13:28 DEBUG "resultType": "vector", 19:13:28 DEBUG "result": [ 19:13:28 DEBUG ----- output ----- 19:13:28 DEBUG "resultType": "vector", 19:13:28 DEBUG --- status code --- 19:13:28 DEBUG "resultType": "vector", 19:13:28 DEBUG --- http response --- 19:13:28 DEBUG "data": { 19:13:28 DEBUG "resultType": "vector", 19:13:28 DEBUG --- status code --- 19:13:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713550408" 19:13:28 DEBUG 19:13:28 INFO Thread started 19:13:28 DEBUG --- stdout --- 19:13:28 DEBUG "data": { 19:13:28 DEBUG "data": { 19:13:28 DEBUG "status": "success", 19:13:28 DEBUG "data": { 19:13:28 DEBUG "result": [ 19:13:28 DEBUG { 19:13:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:13:28 DEBUG "data": { 19:13:28 DEBUG "resultType": "vector", 19:13:28 INFO [http_cmd]: http status code OK 19:13:28 DEBUG { 19:13:28 DEBUG "metric": { 19:13:28 DEBUG { 19:13:28 DEBUG "result": [ 19:13:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:28 DEBUG "result": [ 19:13:28 DEBUG { 19:13:28 DEBUG "result": [ 19:13:28 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:13:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713550408" 19:13:28 INFO Thread started 19:13:28 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 140m 0% 6203Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 67m 0% 7006Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 61m 0% 3771Mi 3% gke-xlou-cdm-default-pool-1b0769c8-qkns 81m 0% 3357Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 58m 0% 13823Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 133m 0% 5205Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 73m 0% 5466Mi 4% gke-xlou-cdm-ds-0f601751-cf7b 82m 0% 3129Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 73m 0% 3241Mi 2% gke-xlou-cdm-ds-0f601751-j488 62m 0% 6704Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 69m 0% 15830Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 94m 0% 7293Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 62m 0% 13277Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 65m 0% 6452Mi 11% 19:13:28 DEBUG "resultType": "vector", 19:13:28 DEBUG "resultType": "vector", 19:13:28 DEBUG "data": { 19:13:28 DEBUG "resultType": "vector", 19:13:28 DEBUG { 19:13:28 DEBUG "status": "success", 19:13:28 DEBUG ----- output ----- 19:13:28 DEBUG "resultType": "vector", 19:13:28 DEBUG "result": [ 19:13:28 DEBUG --- status code --- 19:13:28 DEBUG "resultType": "vector", 19:13:28 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:13:28 DEBUG "status": "success", 19:13:28 DEBUG { 19:13:28 DEBUG --- http response --- 19:13:28 INFO [http_cmd]: http status code OK 19:13:28 DEBUG "data": { 19:13:28 DEBUG "metric": { 19:13:28 DEBUG { 19:13:28 DEBUG --- http response --- 19:13:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550408" 19:13:28 DEBUG --- stderr --- 19:13:28 INFO Thread started 19:13:28 DEBUG "result": [ 19:13:28 INFO [http_cmd]: http status code OK 19:13:28 DEBUG "result": [ 19:13:28 DEBUG "resultType": "vector", 19:13:28 DEBUG "result": [ 19:13:28 DEBUG "metric": { 19:13:28 DEBUG "data": { 19:13:28 DEBUG { 19:13:28 DEBUG "result": [ 19:13:28 DEBUG { 19:13:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:28 DEBUG --- http response --- 19:13:28 DEBUG }, 19:13:28 DEBUG "data": { 19:13:28 DEBUG "metric": { 19:13:28 DEBUG --- status code --- 19:13:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:13:28 DEBUG "resultType": "vector", 19:13:28 INFO [http_cmd]: http status code OK 19:13:28 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:13:28 DEBUG "metric": { 19:13:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:13:28 DEBUG 19:13:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550408" 19:13:28 DEBUG { 19:13:28 INFO Thread started 19:13:28 DEBUG --- status code --- 19:13:28 DEBUG { 19:13:28 DEBUG "result": [ 19:13:28 DEBUG { 19:13:28 DEBUG "metric": { 19:13:28 DEBUG "resultType": "vector", 19:13:28 DEBUG "status": "success", 19:13:28 DEBUG { 19:13:28 DEBUG "metric": { 19:13:28 DEBUG "result": [ 19:13:28 DEBUG "value": [ 19:13:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:13:28 DEBUG "resultType": "vector", 19:13:28 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:13:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:28 DEBUG ----- output ----- 19:13:28 DEBUG "result": [ 19:13:28 DEBUG --- status code --- 19:13:28 DEBUG }, 19:13:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:13:28 DEBUG ----- output ----- 19:13:28 DEBUG "metric": { 19:13:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550408" 19:13:28 INFO Thread started 19:13:28 DEBUG "metric": { 19:13:28 DEBUG { 19:13:28 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:13:28 DEBUG "pod": "ds-cts-0" 19:13:28 DEBUG "result": [ 19:13:28 DEBUG "data": { 19:13:28 DEBUG "metric": { 19:13:28 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:13:28 DEBUG { 19:13:28 DEBUG 19:13:28 DEBUG ----- output ----- 19:13:28 DEBUG "result": [ 19:13:28 DEBUG }, 19:13:28 DEBUG --- http response --- 19:13:28 DEBUG { 19:13:28 DEBUG { 19:13:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:28 DEBUG "value": [ 19:13:28 ERROR [http_cmd]: ERROR 19:13:28 DEBUG }, 19:13:28 DEBUG { 19:13:28 DEBUG "status": "success", 19:13:28 DEBUG --- http response --- 19:13:28 INFO All threads has been started 127.0.0.1 - - [19/Apr/2024 19:13:28] "GET /monitoring/start HTTP/1.1" 200 - 19:13:28 DEBUG "pod": "ds-idrepo-0" 19:13:28 DEBUG "metric": { 19:13:28 DEBUG }, 19:13:28 DEBUG }, 19:13:28 DEBUG { 19:13:28 DEBUG "resultType": "vector", 19:13:28 DEBUG "instance": "10.142.0.107:9100" 19:13:28 DEBUG }, 19:13:28 DEBUG "metric": { 19:13:28 DEBUG 19:13:28 DEBUG { 19:13:28 DEBUG { 19:13:28 DEBUG "value": [ 19:13:28 DEBUG "status": "success", 19:13:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:13:28 DEBUG "metric": { 19:13:28 DEBUG --- http response --- 19:13:28 DEBUG 19:13:28 ERROR --- status code --- 19:13:28 DEBUG "value": [ 19:13:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:13:28 DEBUG "data": { 19:13:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:13:28 DEBUG }, 19:13:28 INFO [http_cmd]: http status code OK 19:13:28 DEBUG "instance": "10.142.0.107:9100" 19:13:28 DEBUG "value": [ 19:13:28 DEBUG "value": [ 19:13:28 DEBUG "metric": { 19:13:28 DEBUG "result": [ 19:13:28 DEBUG }, 19:13:28 DEBUG "value": [ 19:13:28 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:13:28 DEBUG 19:13:28 DEBUG "status": "success", 19:13:28 DEBUG "metric": { 19:13:28 DEBUG 19:13:28 DEBUG "data": { 19:13:28 DEBUG ----- output ----- 19:13:28 DEBUG { 19:13:28 DEBUG 19:13:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:13:28 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:13:28 INFO [http_cmd]: http status code OK 19:13:28 DEBUG 19:13:28 DEBUG }, 19:13:28 DEBUG "resultType": "vector", 19:13:28 DEBUG ----- output ----- 19:13:28 DEBUG "value": [ 19:13:28 DEBUG --- status code --- 19:13:28 DEBUG }, 19:13:28 DEBUG 19:13:28 DEBUG 19:13:28 DEBUG "pod": "ds-idrepo-0" 19:13:28 DEBUG { 19:13:28 DEBUG "value": [ 19:13:28 DEBUG 19:13:28 DEBUG }, 19:13:28 DEBUG [...] 19:13:28 DEBUG "data": { 19:13:28 DEBUG "pod": "ds-idrepo-0" 19:13:28 DEBUG 19:13:28 DEBUG "resultType": "vector", 19:13:28 DEBUG "instance": "10.142.0.107:9100" 19:13:28 DEBUG "status": "success", 19:13:28 DEBUG 19:13:28 DEBUG ----- output ----- 19:13:28 ERROR --- http response --- 19:13:28 DEBUG --- status code --- 19:13:28 DEBUG 19:13:28 DEBUG "value": [ 19:13:28 DEBUG "result": [ 19:13:28 DEBUG { 19:13:28 DEBUG 19:13:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:28 DEBUG "value": [ 19:13:28 DEBUG 19:13:28 DEBUG 19:13:28 DEBUG 19:13:28 DEBUG "metric": { 19:13:28 DEBUG 19:13:28 DEBUG 19:13:28 DEBUG "value": [ 19:13:28 DEBUG 19:13:28 DEBUG "resultType": "vector", 19:13:28 DEBUG }, 19:13:28 DEBUG 19:13:28 DEBUG "result": [ 19:13:28 DEBUG }, 19:13:28 DEBUG "data": { 19:13:28 DEBUG [...] 19:13:28 DEBUG { 19:13:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:28 DEBUG 19:13:28 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:13:28 DEBUG 19:13:28 DEBUG { 19:13:28 DEBUG "status": "success", 19:13:28 DEBUG 19:13:28 DEBUG --- http response --- 19:13:28 DEBUG 19:13:28 DEBUG 19:13:28 DEBUG }, 19:13:28 DEBUG [...] 19:13:28 INFO [http_cmd]: http status code OK 19:13:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:13:28 DEBUG 19:13:28 DEBUG 19:13:28 DEBUG 19:13:28 DEBUG 19:13:28 DEBUG "result": [ 19:13:28 DEBUG "value": [ 19:13:28 DEBUG [...] 19:13:28 DEBUG { 19:13:28 DEBUG "value": [ 19:13:28 DEBUG "resultType": "vector", 19:13:28 DEBUG 19:13:28 DEBUG "status": "success", 19:13:28 DEBUG --- http response --- 19:13:28 DEBUG [...] 19:13:28 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:13:28 DEBUG 19:13:28 DEBUG "metric": { 19:13:28 DEBUG "data": { 19:13:28 DEBUG 19:13:28 DEBUG 19:13:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:13:28 DEBUG [...] 19:13:28 DEBUG "value": [ 19:13:28 DEBUG 19:13:28 DEBUG --- status code --- 19:13:28 DEBUG }, 19:13:28 DEBUG 19:13:28 DEBUG [...] 19:13:28 DEBUG 19:13:28 DEBUG 19:13:28 DEBUG { 19:13:28 DEBUG 19:13:28 DEBUG 19:13:28 DEBUG "metric": { 19:13:28 DEBUG 19:13:28 DEBUG "result": [ 19:13:28 DEBUG 19:13:28 DEBUG "data": { 19:13:28 DEBUG 19:13:28 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:13:28 DEBUG 19:13:28 DEBUG "pod": "ds-cts-0" 19:13:28 DEBUG "resultType": "vector", 19:13:28 DEBUG [...] 19:13:28 DEBUG 19:13:28 DEBUG ----- output ----- 19:13:28 DEBUG 19:13:28 DEBUG 19:13:28 DEBUG 19:13:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:28 DEBUG "value": [ 19:13:28 DEBUG [...] 19:13:28 DEBUG 19:13:28 DEBUG 19:13:28 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:13:28 DEBUG "metric": { 19:13:28 DEBUG "pod": "ds-cts-0" 19:13:28 DEBUG 19:13:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:13:28 DEBUG 19:13:28 DEBUG { 19:13:28 DEBUG 19:13:28 DEBUG "resultType": "vector", 19:13:28 DEBUG 19:13:28 DEBUG [...] 19:13:28 DEBUG }, 19:13:28 DEBUG "value": [ 19:13:28 DEBUG "result": [ 19:13:28 DEBUG [...] 19:13:28 DEBUG { 19:13:28 DEBUG 19:13:28 DEBUG 19:13:28 DEBUG 19:13:28 DEBUG --- http response --- 19:13:28 DEBUG 19:13:28 DEBUG 19:13:28 DEBUG 19:13:28 DEBUG [...] 19:13:28 DEBUG }, 19:13:28 DEBUG }, 19:13:28 DEBUG "value": [ 19:13:28 DEBUG 19:13:28 DEBUG }, 19:13:28 DEBUG 19:13:28 DEBUG "metric": { 19:13:28 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:13:28 DEBUG "result": [ 19:13:28 DEBUG 19:13:28 DEBUG 19:13:28 DEBUG 19:13:28 DEBUG 19:13:28 DEBUG { 19:13:28 DEBUG 19:13:28 DEBUG "status": "success", 19:13:28 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:13:28 DEBUG 19:13:28 DEBUG 19:13:28 DEBUG [...] 19:13:28 DEBUG 19:13:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:13:28 DEBUG ----- output ----- 19:13:28 DEBUG 19:13:28 DEBUG "value": [ 19:13:28 DEBUG 19:13:28 DEBUG 19:13:28 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:13:28 DEBUG "value": [ 19:13:28 DEBUG [...] 19:13:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:13:28 DEBUG }, 19:13:28 DEBUG { 19:13:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:13:28 DEBUG 19:13:28 DEBUG 19:13:28 DEBUG 19:13:28 DEBUG "metric": { 19:13:28 DEBUG 19:13:28 DEBUG "data": { 19:13:28 DEBUG }, 19:13:28 DEBUG "pod": "ds-cts-2" 19:13:28 DEBUG 19:13:28 DEBUG 19:13:28 DEBUG [...] 19:13:28 DEBUG 19:13:28 DEBUG { 19:13:28 DEBUG 19:13:28 DEBUG 1713550407, 19:13:28 DEBUG 19:13:28 DEBUG 19:13:28 DEBUG }, 19:13:28 DEBUG 19:13:28 DEBUG 19:13:28 DEBUG }, 19:13:28 DEBUG "value": [ 19:13:28 DEBUG "metric": { 19:13:28 DEBUG "pod": "ds-idrepo-0" 19:13:28 DEBUG }, 19:13:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:13:28 DEBUG 19:13:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:13:28 DEBUG 19:13:28 DEBUG "resultType": "vector", 19:13:28 DEBUG "value": [ 19:13:28 DEBUG }, 19:13:28 DEBUG 19:13:28 DEBUG 19:13:28 DEBUG 19:13:28 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:13:28 DEBUG "status": "success", 19:13:28 DEBUG 19:13:28 DEBUG "0.00021848436873262879" 19:13:28 DEBUG [...] 19:13:28 DEBUG 19:13:28 DEBUG "value": [ 19:13:28 DEBUG 19:13:28 DEBUG 19:13:28 DEBUG "value": [ 19:13:28 DEBUG 19:13:28 DEBUG 19:13:28 DEBUG }, 19:13:28 DEBUG "value": [ 19:13:28 DEBUG }, 19:13:28 DEBUG [...] 19:13:28 DEBUG }, 19:13:28 DEBUG "instance": "10.142.0.43:9100" 19:13:28 DEBUG "result": [ 19:13:28 DEBUG 1713550407, 19:13:28 DEBUG "value": [ 19:13:28 DEBUG 19:13:28 DEBUG 19:13:28 DEBUG 19:13:28 DEBUG }, 19:13:28 DEBUG "data": { 19:13:28 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:13:28 DEBUG ] 19:13:28 DEBUG 19:13:28 DEBUG 19:13:28 DEBUG 19:13:28 DEBUG 19:13:28 DEBUG 19:13:28 DEBUG 1713550407, 19:13:28 DEBUG 19:13:28 DEBUG 19:13:28 DEBUG "value": [ 19:13:28 DEBUG 19:13:28 DEBUG "value": [ 19:13:28 DEBUG 19:13:28 DEBUG "value": [ 19:13:28 DEBUG }, 19:13:28 DEBUG { 19:13:28 DEBUG "0.02678637659695918" 19:13:28 DEBUG 1713550407, 19:13:28 DEBUG "0" 19:13:28 DEBUG ] 19:13:28 DEBUG 19:13:28 DEBUG "value": [ 19:13:28 DEBUG "resultType": "vector", 19:13:28 DEBUG }, 19:13:28 DEBUG } 19:13:28 DEBUG [...] 19:13:28 DEBUG 1713550407, 19:13:28 DEBUG "pod": "ds-idrepo-2" 19:13:28 DEBUG [...] 19:13:28 DEBUG "instance": "10.142.0.43:9100" 19:13:28 DEBUG "2845605888" 19:13:28 DEBUG 19:13:28 DEBUG "pod": "ds-idrepo-2" 19:13:28 DEBUG 1713550407, 19:13:28 DEBUG 19:13:28 DEBUG 1713550407, 19:13:28 DEBUG 19:13:28 DEBUG 19:13:28 DEBUG "value": [ 19:13:28 DEBUG "metric": { 19:13:28 DEBUG ] 19:13:28 DEBUG "instance": "10.142.0.43:9100" 19:13:28 DEBUG }, 19:13:28 DEBUG } 19:13:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:13:28 DEBUG 1713550407, 19:13:28 DEBUG "result": [ 19:13:28 DEBUG "value": [ 19:13:28 DEBUG ] 19:13:28 DEBUG } 19:13:28 DEBUG "0" 19:13:28 DEBUG }, 19:13:28 DEBUG 19:13:28 DEBUG }, 19:13:28 DEBUG ] 19:13:28 DEBUG [...] 19:13:28 DEBUG }, 19:13:28 DEBUG "value": [ 19:13:28 DEBUG "0" 19:13:28 DEBUG "0" 19:13:28 DEBUG 19:13:28 DEBUG 19:13:28 DEBUG 1713550407, 19:13:28 DEBUG "instance": "10.142.0.107:9100" 19:13:28 DEBUG } 19:13:28 DEBUG "pod": "ds-idrepo-2" 19:13:28 DEBUG "value": [ 19:13:28 DEBUG ] 19:13:28 DEBUG }, 19:13:28 DEBUG "0" 19:13:28 DEBUG { 19:13:28 DEBUG 1713550407, 19:13:28 DEBUG 19:13:28 DEBUG } 19:13:28 DEBUG ] 19:13:28 DEBUG } 19:13:28 DEBUG 19:13:28 DEBUG "value": [ 19:13:28 DEBUG } 19:13:28 DEBUG 19:13:28 DEBUG 19:13:28 DEBUG 1713550407, 19:13:28 DEBUG ] 19:13:28 DEBUG ] 19:13:28 DEBUG "pod": "ds-cts-2" 19:13:28 DEBUG 19:13:28 DEBUG "0.0009999999999990906" 19:13:28 DEBUG }, 19:13:28 DEBUG ] 19:13:28 DEBUG }, 19:13:28 DEBUG 1713550407, 19:13:28 DEBUG } 19:13:28 DEBUG } 19:13:28 DEBUG "value": [ 19:13:28 DEBUG "metric": { 19:13:28 DEBUG "0" 19:13:28 DEBUG 19:13:28 DEBUG 19:13:28 DEBUG "value": [ 19:13:28 DEBUG 19:13:28 DEBUG 1713550407, 19:13:28 DEBUG ] 19:13:28 DEBUG 19:13:28 DEBUG [...] 19:13:28 DEBUG "0" 19:13:28 DEBUG } 19:13:28 DEBUG } 19:13:28 DEBUG }, 19:13:28 DEBUG [...] 19:13:28 DEBUG ] 19:13:28 DEBUG "value": [ 19:13:28 DEBUG } 19:13:28 DEBUG "value": [ 19:13:28 DEBUG "15477.666666666668" 19:13:28 DEBUG ] 19:13:28 DEBUG 1713550408, 19:13:28 DEBUG "instance": "10.142.0.107:9100" 19:13:28 DEBUG ] 19:13:28 DEBUG "pod": "ds-cts-2" 19:13:28 DEBUG }, 19:13:28 DEBUG 1713550407, 19:13:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:13:28 DEBUG "13214" 19:13:28 DEBUG } 19:13:28 DEBUG 19:13:28 DEBUG 19:13:28 DEBUG ] 19:13:28 DEBUG ] 19:13:28 DEBUG ] 19:13:28 DEBUG "value": [ 19:13:28 DEBUG 19:13:28 DEBUG } 19:13:28 DEBUG 19:13:28 DEBUG } 19:13:28 DEBUG 1713550407, 19:13:28 DEBUG ] 19:13:28 DEBUG } 19:13:28 DEBUG "0" 19:13:28 DEBUG }, 19:13:28 DEBUG } 19:13:28 DEBUG ] 19:13:28 DEBUG "value": [ 19:13:28 DEBUG "0" 19:13:28 DEBUG }, 19:13:28 DEBUG ] 19:13:28 DEBUG } 19:13:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:13:28 DEBUG 19:13:28 DEBUG 19:13:28 DEBUG "pod": "ds-cts-2" 19:13:28 DEBUG } 19:13:28 DEBUG 1713550408, 19:13:28 DEBUG "0" 19:13:28 DEBUG 19:13:28 DEBUG 19:13:28 DEBUG "0" 19:13:28 DEBUG } 19:13:28 DEBUG ] 19:13:28 DEBUG ] 19:13:28 DEBUG "value": [ 19:13:28 DEBUG ] 19:13:28 DEBUG } 19:13:28 DEBUG 1713550408, 19:13:28 DEBUG ] 19:13:28 DEBUG "value": [ 19:13:28 DEBUG } 19:13:28 DEBUG }, 19:13:28 DEBUG } 19:13:28 DEBUG } 19:13:28 DEBUG }, 19:13:28 DEBUG } 19:13:28 DEBUG ] 19:13:28 DEBUG ] 19:13:28 DEBUG 19:13:28 DEBUG 19:13:28 DEBUG ] 19:13:28 DEBUG ] 19:13:28 DEBUG } 19:13:28 DEBUG } 19:13:28 DEBUG 19:13:28 DEBUG } 19:13:28 DEBUG } 19:13:28 DEBUG "0" 19:13:28 DEBUG } 19:13:28 DEBUG ] 19:13:28 DEBUG ] 19:13:28 DEBUG "value": [ 19:13:28 DEBUG ] 19:13:28 DEBUG } 19:13:28 DEBUG "value": [ 19:13:28 DEBUG } 19:13:28 DEBUG } 19:13:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:13:28 DEBUG [...] 19:13:28 DEBUG } 19:13:28 DEBUG } 19:13:28 DEBUG } 19:13:28 DEBUG ] 19:13:28 DEBUG 19:13:28 DEBUG } 19:13:28 DEBUG ] 19:13:28 DEBUG 1713550408, 19:13:28 DEBUG } 19:13:28 DEBUG } 19:13:28 DEBUG 1713550408, 19:13:28 DEBUG } 19:13:28 DEBUG 1713550408, 19:13:28 DEBUG } 19:13:28 DEBUG ] 19:13:28 DEBUG }, 19:13:28 DEBUG 19:13:28 DEBUG ] 19:13:28 DEBUG } 19:13:28 DEBUG } 19:13:28 DEBUG 19:13:28 DEBUG [...] 19:13:28 DEBUG "0" 19:13:28 DEBUG } 19:13:28 DEBUG } 19:13:28 DEBUG "2" 19:13:28 DEBUG } 19:13:28 DEBUG "0" 19:13:28 DEBUG } 19:13:28 DEBUG "value": [ 19:13:28 DEBUG 19:13:28 DEBUG } 19:13:28 DEBUG } 19:13:28 DEBUG } 19:13:28 DEBUG 19:13:28 DEBUG ] 19:13:28 DEBUG ] 19:13:28 DEBUG ] 19:13:28 DEBUG } 19:13:28 DEBUG 1713550408, 19:13:28 DEBUG 19:13:28 DEBUG } 19:13:28 DEBUG ] 19:13:28 DEBUG 19:13:28 DEBUG } 19:13:28 DEBUG } 19:13:28 DEBUG } 19:13:28 DEBUG "0.002" 19:13:28 DEBUG "instance": "10.142.0.43:9100" 19:13:28 DEBUG } 19:13:28 DEBUG 19:13:28 DEBUG "instance": "10.142.0.43:9100" 19:13:28 DEBUG ] 19:13:28 DEBUG ] 19:13:28 DEBUG ] 19:13:28 DEBUG }, 19:13:28 DEBUG } 19:13:28 DEBUG ] 19:13:28 DEBUG }, 19:13:28 DEBUG } 19:13:28 DEBUG } 19:13:28 DEBUG } 19:13:28 DEBUG "value": [ 19:13:28 DEBUG } 19:13:28 DEBUG "value": [ 19:13:28 DEBUG } 19:13:28 DEBUG } 19:13:28 DEBUG ] 19:13:28 DEBUG 1713550408, 19:13:28 DEBUG } 19:13:28 DEBUG 1713550408, 19:13:28 DEBUG } 19:13:28 DEBUG "1847726887936" 19:13:28 DEBUG "0.0006999999999673415" 19:13:28 DEBUG } 19:13:28 DEBUG ] 19:13:28 DEBUG ] 19:13:28 DEBUG } 19:13:28 DEBUG } 19:13:28 DEBUG ] 19:13:28 DEBUG ] 19:13:28 DEBUG } 19:13:28 DEBUG } 19:13:28 DEBUG } 19:13:28 DEBUG } 19:14:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713550468" 19:14:28 ERROR [http_cmd]: ERROR 19:14:28 ERROR --- status code --- 19:14:28 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:14:28 ERROR --- http response --- 19:14:28 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:14:28 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:14:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550468" 19:14:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550468" 19:14:28 INFO 19:14:28 INFO [loop_until]: kubectl --namespace=xlou top pods 19:14:28 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:14:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550468" 19:14:28 INFO 19:14:28 INFO [loop_until]: kubectl --namespace=xlou top node 19:14:28 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:14:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550468" 19:14:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713550468" 19:14:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550468" 19:14:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550468" 19:14:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713550468" 19:14:28 INFO [http_cmd]: http status code OK 19:14:28 DEBUG --- status code --- 19:14:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713550468" 19:14:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:28 INFO [http_cmd]: http status code OK 19:14:28 INFO [http_cmd]: http status code OK 19:14:28 DEBUG --- http response --- 19:14:28 DEBUG --- status code --- 19:14:28 DEBUG --- status code --- 19:14:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:14:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:28 DEBUG ----- output ----- 19:14:28 DEBUG --- http response --- 19:14:28 DEBUG --- http response --- 19:14:28 DEBUG { 19:14:28 INFO [http_cmd]: http status code OK 19:14:28 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:14:28 DEBUG "status": "success", 19:14:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:14:28 DEBUG --- status code --- 19:14:28 DEBUG "data": { 19:14:28 DEBUG ----- output ----- 19:14:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:28 DEBUG "resultType": "vector", 19:14:28 DEBUG { 19:14:28 DEBUG --- http response --- 19:14:28 DEBUG "result": [ 19:14:28 DEBUG "status": "success", 19:14:28 DEBUG { 19:14:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:14:28 DEBUG "data": { 19:14:28 DEBUG "metric": { 19:14:28 DEBUG ----- output ----- 19:14:28 DEBUG "resultType": "vector", 19:14:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:14:28 DEBUG { 19:14:28 DEBUG "result": [ 19:14:28 DEBUG }, 19:14:28 DEBUG "status": "success", 19:14:28 DEBUG { 19:14:28 DEBUG "value": [ 19:14:28 DEBUG "data": { 19:14:28 DEBUG "metric": { 19:14:28 DEBUG 19:14:28 DEBUG "resultType": "vector", 19:14:28 INFO [http_cmd]: http status code OK 19:14:28 DEBUG "pod": "ds-idrepo-0" 19:14:28 DEBUG 19:14:28 INFO [http_cmd]: http status code OK 19:14:28 INFO [http_cmd]: http status code OK 19:14:28 INFO [http_cmd]: http status code OK 19:14:28 DEBUG "result": [ 19:14:28 DEBUG --- status code --- 19:14:28 DEBUG }, 19:14:28 DEBUG 19:14:28 DEBUG --- status code --- 19:14:28 DEBUG --- status code --- 19:14:28 DEBUG --- status code --- 19:14:28 DEBUG { 19:14:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:28 DEBUG "value": [ 19:14:28 DEBUG [...] 19:14:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:28 DEBUG "metric": { 19:14:28 DEBUG --- http response --- 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG --- http response --- 19:14:28 DEBUG --- http response --- 19:14:28 DEBUG --- http response --- 19:14:28 DEBUG "pod": "ds-idrepo-0" 19:14:28 DEBUG 19:14:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:14:28 DEBUG 19:14:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:14:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:14:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:14:28 DEBUG }, 19:14:28 DEBUG 19:14:28 DEBUG ----- output ----- 19:14:28 DEBUG 19:14:28 DEBUG ----- output ----- 19:14:28 DEBUG ----- output ----- 19:14:28 DEBUG ----- output ----- 19:14:28 DEBUG "value": [ 19:14:28 DEBUG [...] 19:14:28 DEBUG { 19:14:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:14:28 DEBUG { 19:14:28 DEBUG { 19:14:28 DEBUG { 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG "status": "success", 19:14:28 DEBUG }, 19:14:28 DEBUG "status": "success", 19:14:28 DEBUG "status": "success", 19:14:28 DEBUG "status": "success", 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG "data": { 19:14:28 DEBUG "value": [ 19:14:28 DEBUG "data": { 19:14:28 DEBUG "data": { 19:14:28 DEBUG "data": { 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG "resultType": "vector", 19:14:28 DEBUG 1713550468, 19:14:28 DEBUG "769.9620733333335" 19:14:28 DEBUG "resultType": "vector", 19:14:28 DEBUG "resultType": "vector", 19:14:28 DEBUG [...] 19:14:28 INFO [http_cmd]: http status code OK 19:14:28 DEBUG "pod": "ds-idrepo-2" 19:14:28 DEBUG "result": [ 19:14:28 DEBUG "resultType": "vector", 19:14:28 DEBUG ] 19:14:28 DEBUG "result": [ 19:14:28 DEBUG "result": [ 19:14:28 DEBUG 19:14:28 DEBUG --- status code --- 19:14:28 DEBUG }, 19:14:28 DEBUG { 19:14:28 DEBUG "result": [ 19:14:28 DEBUG } 19:14:28 DEBUG { 19:14:28 DEBUG { 19:14:28 DEBUG 19:14:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:28 DEBUG "value": [ 19:14:28 DEBUG "metric": { 19:14:28 DEBUG { 19:14:28 DEBUG ] 19:14:28 DEBUG "metric": { 19:14:28 DEBUG "metric": { 19:14:28 DEBUG 19:14:28 DEBUG --- http response --- 19:14:28 DEBUG 1713550468, 19:14:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:14:28 DEBUG "metric": { 19:14:28 DEBUG "pod": "ds-idrepo-0" 19:14:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:14:28 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:14:28 DEBUG "pod": "ds-idrepo-2" 19:14:28 DEBUG "0" 19:14:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:14:28 DEBUG }, 19:14:28 DEBUG } 19:14:28 DEBUG }, 19:14:28 DEBUG }, 19:14:28 DEBUG }, 19:14:28 DEBUG }, 19:14:28 DEBUG ] 19:14:28 DEBUG ----- output ----- 19:14:28 DEBUG "value": [ 19:14:28 DEBUG } 19:14:28 DEBUG "value": [ 19:14:28 DEBUG "value": [ 19:14:28 DEBUG "value": [ 19:14:28 DEBUG "value": [ 19:14:28 DEBUG } 19:14:28 DEBUG { 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 1713550468, 19:14:28 DEBUG ] 19:14:28 DEBUG "status": "success", 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG "0" 19:14:28 DEBUG } 19:14:28 DEBUG "data": { 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG [...] 19:14:28 DEBUG ] 19:14:28 DEBUG } 19:14:28 DEBUG "resultType": "vector", 19:14:28 DEBUG [...] 19:14:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550468" 19:14:28 DEBUG [...] 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG } 19:14:28 DEBUG "result": [ 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550468" 19:14:28 DEBUG 19:14:28 DEBUG [...] 19:14:28 DEBUG ] 19:14:28 DEBUG { 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713550468" 19:14:28 DEBUG 19:14:28 DEBUG } 19:14:28 DEBUG "metric": { 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:14:28 DEBUG 19:14:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550468" 19:14:28 DEBUG } 19:14:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:14:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:14:28 DEBUG "pod": "ds-cts-2" 19:14:28 DEBUG }, 19:14:28 DEBUG 19:14:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550468" 19:14:28 DEBUG }, 19:14:28 DEBUG "value": [ 19:14:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550468" 19:14:28 DEBUG }, 19:14:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550468" 19:14:28 DEBUG }, 19:14:28 DEBUG "value": [ 19:14:28 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:14:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550468" 19:14:28 DEBUG 19:14:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550468" 19:14:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550468" 19:14:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550468" 19:14:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550468" 19:14:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550468" 19:14:28 DEBUG "value": [ 19:14:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550468" 19:14:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550468" 19:14:28 DEBUG "value": [ 19:14:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550468" 19:14:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550468" 19:14:28 INFO [http_cmd]: http status code OK 19:14:28 INFO [http_cmd]: http status code OK 19:14:28 DEBUG --- status code --- 19:14:28 INFO [http_cmd]: http status code OK 19:14:28 DEBUG 1713550468, 19:14:28 DEBUG }, 19:14:28 DEBUG 19:14:28 INFO [http_cmd]: http status code OK 19:14:28 INFO [loop_until]: OK (rc = 0) 19:14:28 INFO [http_cmd]: http status code OK 19:14:28 DEBUG 1713550468, 19:14:28 INFO [http_cmd]: http status code OK 19:14:28 INFO [http_cmd]: http status code OK 19:14:28 DEBUG 1713550468, 19:14:28 INFO [http_cmd]: http status code OK 19:14:28 INFO [http_cmd]: http status code OK 19:14:28 INFO [http_cmd]: http status code OK 19:14:28 DEBUG --- status code --- 19:14:28 INFO [http_cmd]: http status code OK 19:14:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:28 DEBUG --- status code --- 19:14:28 DEBUG "0" 19:14:28 DEBUG "value": [ 19:14:28 DEBUG 19:14:28 DEBUG --- status code --- 19:14:28 DEBUG --- stdout --- 19:14:28 DEBUG --- status code --- 19:14:28 INFO [http_cmd]: http status code OK 19:14:28 DEBUG "0" 19:14:28 INFO [http_cmd]: http status code OK 19:14:28 INFO [http_cmd]: http status code OK 19:14:28 DEBUG --- status code --- 19:14:28 DEBUG --- status code --- 19:14:28 DEBUG "0" 19:14:28 DEBUG --- status code --- 19:14:28 DEBUG --- status code --- 19:14:28 DEBUG --- status code --- 19:14:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:28 DEBUG --- status code --- 19:14:28 DEBUG --- http response --- 19:14:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:28 DEBUG ] 19:14:28 DEBUG 1713550468, 19:14:28 DEBUG [...] 19:14:28 DEBUG 19:14:28 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 5622m 3877Mi am-767f6b67b5-qcqz2 6691m 3618Mi am-767f6b67b5-vbltz 4049m 3793Mi ds-cts-0 2809m 10766Mi ds-cts-1 3675m 11048Mi ds-cts-2 3115m 10785Mi ds-idrepo-0 2264m 3441Mi ds-idrepo-1 2370m 3413Mi ds-idrepo-2 2149m 3492Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 2m 61Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 3803m 3078Mi 19:14:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:28 DEBUG --- status code --- 19:14:28 DEBUG ] 19:14:28 DEBUG --- status code --- 19:14:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:28 DEBUG ] 19:14:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:28 DEBUG --- http response --- 19:14:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:14:28 DEBUG ----- output ----- 19:14:28 DEBUG { 19:14:28 DEBUG } 19:14:28 DEBUG 19:14:28 DEBUG --- stderr --- 19:14:28 DEBUG --- http response --- 19:14:28 DEBUG --- status code --- 19:14:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:28 INFO [http_cmd]: http status code OK 19:14:28 INFO [http_cmd]: http status code OK 19:14:28 DEBUG } 19:14:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:28 DEBUG --- http response --- 19:14:28 DEBUG --- http response --- 19:14:28 DEBUG } 19:14:28 DEBUG --- http response --- 19:14:28 DEBUG --- http response --- 19:14:28 DEBUG --- http response --- 19:14:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:14:28 DEBUG --- http response --- 19:14:28 DEBUG --- http response --- 19:14:28 DEBUG "2.8429022729144307" 19:14:28 DEBUG "status": "success", 19:14:28 DEBUG ] 19:14:28 INFO [http_cmd]: http status code OK 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:14:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:28 DEBUG --- http response --- 19:14:28 DEBUG --- http response --- 19:14:28 DEBUG --- status code --- 19:14:28 DEBUG --- status code --- 19:14:28 DEBUG ] 19:14:28 DEBUG --- http response --- 19:14:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:14:28 DEBUG ] 19:14:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:14:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:14:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:14:28 DEBUG ----- output ----- 19:14:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:14:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:14:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:14:28 DEBUG ] 19:14:28 DEBUG "data": { 19:14:28 DEBUG } 19:14:28 DEBUG --- status code --- 19:14:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:14:28 DEBUG ----- output ----- 19:14:28 DEBUG --- http response --- 19:14:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:14:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:14:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:28 INFO [loop_until]: OK (rc = 0) 19:14:28 DEBUG } 19:14:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:14:28 DEBUG ----- output ----- 19:14:28 DEBUG } 19:14:28 DEBUG ----- output ----- 19:14:28 DEBUG ----- output ----- 19:14:28 DEBUG ----- output ----- 19:14:28 DEBUG { 19:14:28 DEBUG ----- output ----- 19:14:28 DEBUG ----- output ----- 19:14:28 DEBUG ----- output ----- 19:14:28 DEBUG } 19:14:28 DEBUG "resultType": "vector", 19:14:28 DEBUG } 19:14:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:28 DEBUG }, 19:14:28 DEBUG { 19:14:28 DEBUG ----- output ----- 19:14:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:14:28 DEBUG ----- output ----- 19:14:28 DEBUG --- http response --- 19:14:28 DEBUG --- http response --- 19:14:28 DEBUG --- stdout --- 19:14:28 DEBUG } 19:14:28 DEBUG ----- output ----- 19:14:28 DEBUG { 19:14:28 DEBUG } 19:14:28 DEBUG { 19:14:28 DEBUG { 19:14:28 DEBUG { 19:14:28 DEBUG "status": "success", 19:14:28 DEBUG { 19:14:28 DEBUG { 19:14:28 DEBUG { 19:14:28 DEBUG ] 19:14:28 DEBUG "result": [ 19:14:28 DEBUG { 19:14:28 DEBUG "value": [ 19:14:28 DEBUG "status": "success", 19:14:28 DEBUG { 19:14:28 DEBUG ----- output ----- 19:14:28 DEBUG { 19:14:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:14:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:14:28 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 68m 0% 6236Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 3424m 11% 7039Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 7899m 26% 6703Mi 6% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 3362Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 3728m 12% 13915Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 163m 0% 5210Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 7316m 24% 6655Mi 5% gke-xlou-cdm-ds-0f601751-cf7b 5634m 18% 3035Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 8200m 27% 6175Mi 5% gke-xlou-cdm-ds-0f601751-j488 3434m 11% 6726Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 3953m 13% 15916Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 3399m 11% 7316Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 3607m 12% 13327Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 4641m 29% 6897Mi 11% 19:14:28 DEBUG { 19:14:28 DEBUG "status": "success", 19:14:28 DEBUG "data": { 19:14:28 DEBUG "resultType": "vector", 19:14:28 DEBUG "status": "success", 19:14:28 DEBUG "data": { 19:14:28 DEBUG "status": "success", 19:14:28 DEBUG "status": "success", 19:14:28 DEBUG "status": "success", 19:14:28 DEBUG } 19:14:28 DEBUG --- http response --- 19:14:28 DEBUG "metric": { 19:14:28 DEBUG 1713550468, 19:14:28 DEBUG "data": { 19:14:28 DEBUG "status": "success", 19:14:28 DEBUG { 19:14:28 DEBUG "status": "success", 19:14:28 DEBUG ----- output ----- 19:14:28 DEBUG ----- output ----- 19:14:28 DEBUG --- stderr --- 19:14:28 DEBUG "status": "success", 19:14:28 DEBUG "status": "success", 19:14:28 DEBUG "status": "success", 19:14:28 DEBUG "result": [ 19:14:28 DEBUG "data": { 19:14:28 DEBUG "resultType": "vector", 19:14:28 DEBUG "data": { 19:14:28 DEBUG "data": { 19:14:28 DEBUG "data": { 19:14:28 DEBUG } 19:14:28 DEBUG "pod": "ds-cts-0" 19:14:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:14:28 DEBUG "0.002" 19:14:28 DEBUG "resultType": "vector", 19:14:28 DEBUG "data": { 19:14:28 DEBUG "status": "success", 19:14:28 DEBUG "data": { 19:14:28 DEBUG { 19:14:28 DEBUG { 19:14:28 DEBUG 19:14:28 DEBUG "data": { 19:14:28 DEBUG "data": { 19:14:28 DEBUG "resultType": "vector", 19:14:28 DEBUG { 19:14:28 DEBUG "resultType": "vector", 19:14:28 DEBUG "result": [ 19:14:28 DEBUG "resultType": "vector", 19:14:28 DEBUG "result": [ 19:14:28 DEBUG "resultType": "vector", 19:14:28 DEBUG }, 19:14:28 DEBUG ----- output ----- 19:14:28 DEBUG ] 19:14:28 DEBUG "result": [ 19:14:28 DEBUG "resultType": "vector", 19:14:28 DEBUG "data": { 19:14:28 DEBUG "resultType": "vector", 19:14:28 DEBUG "status": "success", 19:14:28 DEBUG "status": "success", 19:14:28 DEBUG "resultType": "vector", 19:14:28 DEBUG "data": { 19:14:28 DEBUG "result": [ 19:14:28 DEBUG "metric": { 19:14:28 DEBUG "result": [ 19:14:28 DEBUG { 19:14:28 DEBUG "resultType": "vector", 19:14:28 DEBUG { 19:14:28 DEBUG "result": [ 19:14:28 DEBUG "value": [ 19:14:28 DEBUG { 19:14:28 DEBUG } 19:14:28 DEBUG { 19:14:28 DEBUG "result": [ 19:14:28 DEBUG "resultType": "vector", 19:14:28 DEBUG "result": [ 19:14:28 DEBUG "data": { 19:14:28 DEBUG "data": { 19:14:28 DEBUG "result": [ 19:14:28 DEBUG "resultType": "vector", 19:14:28 DEBUG { 19:14:28 DEBUG "pod": "ds-cts-0" 19:14:28 DEBUG { 19:14:28 DEBUG "metric": { 19:14:28 DEBUG "result": [ 19:14:28 DEBUG "metric": { 19:14:28 DEBUG { 19:14:28 DEBUG 19:14:28 DEBUG "status": "success", 19:14:28 DEBUG ] 19:14:28 DEBUG "metric": { 19:14:28 DEBUG { 19:14:28 DEBUG "result": [ 19:14:28 DEBUG { 19:14:28 DEBUG "resultType": "vector", 19:14:28 DEBUG "resultType": "vector", 19:14:28 DEBUG { 19:14:28 DEBUG "result": [ 19:14:28 DEBUG "metric": { 19:14:28 DEBUG }, 19:14:28 DEBUG "metric": { 19:14:28 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:14:28 DEBUG { 19:14:28 DEBUG "instance": "10.142.0.107:9100" 19:14:28 DEBUG "metric": { 19:14:28 DEBUG 19:14:28 DEBUG "data": { 19:14:28 DEBUG } 19:14:28 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:14:28 DEBUG "metric": { 19:14:28 DEBUG { 19:14:28 DEBUG "metric": { 19:14:28 DEBUG "result": [ 19:14:28 DEBUG "result": [ 19:14:28 DEBUG "metric": { 19:14:28 DEBUG { 19:14:28 DEBUG "instance": "10.142.0.107:9100" 19:14:28 DEBUG "value": [ 19:14:28 DEBUG "pod": "ds-idrepo-0" 19:14:28 DEBUG }, 19:14:28 DEBUG "metric": { 19:14:28 DEBUG }, 19:14:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:14:28 DEBUG 19:14:28 DEBUG "resultType": "vector", 19:14:28 DEBUG } 19:14:28 DEBUG }, 19:14:28 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:14:28 DEBUG "metric": { 19:14:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:14:28 DEBUG { 19:14:28 DEBUG { 19:14:28 DEBUG "pod": "ds-cts-0" 19:14:28 DEBUG "metric": { 19:14:28 DEBUG }, 19:14:28 DEBUG 19:14:28 DEBUG }, 19:14:28 DEBUG "value": [ 19:14:28 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:14:28 DEBUG "value": [ 19:14:28 DEBUG }, 19:14:28 DEBUG [...] 19:14:28 DEBUG "result": [ 19:14:28 DEBUG "value": [ 19:14:28 DEBUG }, 19:14:28 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:14:28 DEBUG }, 19:14:28 DEBUG "metric": { 19:14:28 DEBUG "metric": { 19:14:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:14:28 DEBUG "instance": "10.142.0.107:9100" 19:14:28 DEBUG "value": [ 19:14:28 DEBUG 19:14:28 DEBUG "value": [ 19:14:28 DEBUG 19:14:28 DEBUG }, 19:14:28 DEBUG 19:14:28 DEBUG "value": [ 19:14:28 DEBUG 19:14:28 DEBUG { 19:14:28 DEBUG 19:14:28 DEBUG "value": [ 19:14:28 DEBUG }, 19:14:28 DEBUG "value": [ 19:14:28 DEBUG "instance": "10.142.0.107:9100" 19:14:28 DEBUG }, 19:14:28 DEBUG }, 19:14:28 DEBUG }, 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG "value": [ 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG "metric": { 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG "value": [ 19:14:28 DEBUG 19:14:28 DEBUG }, 19:14:28 DEBUG "value": [ 19:14:28 DEBUG "value": [ 19:14:28 DEBUG "value": [ 19:14:28 DEBUG 19:14:28 DEBUG [...] 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG "instance": "10.142.0.107:9100" 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG "value": [ 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG [...] 19:14:28 DEBUG 19:14:28 DEBUG [...] 19:14:28 DEBUG 19:14:28 DEBUG "pod": "ds-cts-2" 19:14:28 DEBUG }, 19:14:28 DEBUG [...] 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG [...] 19:14:28 DEBUG 19:14:28 DEBUG [...] 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG [...] 19:14:28 DEBUG }, 19:14:28 DEBUG "value": [ 19:14:28 DEBUG 19:14:28 DEBUG [...] 19:14:28 DEBUG 19:14:28 DEBUG [...] 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG [...] 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG "value": [ 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG [...] 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG [...] 19:14:28 DEBUG [...] 19:14:28 DEBUG [...] 19:14:28 DEBUG 19:14:28 DEBUG "pod": "ds-cts-2" 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 1713550468, 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG [...] 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG }, 19:14:28 DEBUG 19:14:28 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 19:14:28 DEBUG 19:14:28 DEBUG "instance": "10.142.0.43:9100" 19:14:28 DEBUG 19:14:28 DEBUG "0" 19:14:28 DEBUG 19:14:28 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG "instance": "10.142.0.43:9100" 19:14:28 DEBUG "value": [ 19:14:28 DEBUG "pod": "ds-idrepo-2" 19:14:28 DEBUG }, 19:14:28 DEBUG 19:14:28 DEBUG }, 19:14:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:14:28 DEBUG ] 19:14:28 DEBUG [...] 19:14:28 DEBUG }, 19:14:28 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 19:14:28 DEBUG 19:14:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG 19:14:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:14:28 DEBUG 19:14:28 DEBUG 1713550468, 19:14:28 DEBUG }, 19:14:28 DEBUG "value": [ 19:14:28 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:14:28 DEBUG "value": [ 19:14:28 DEBUG }, 19:14:28 DEBUG } 19:14:28 DEBUG 19:14:28 DEBUG "value": [ 19:14:28 DEBUG }, 19:14:28 DEBUG "value": [ 19:14:28 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 19:14:28 DEBUG 19:14:28 DEBUG "pod": "ds-cts-2" 19:14:28 DEBUG }, 19:14:28 DEBUG }, 19:14:28 DEBUG "instance": "10.142.0.43:9100" 19:14:28 DEBUG "0" 19:14:28 DEBUG "value": [ 19:14:28 DEBUG 1713550468, 19:14:28 DEBUG }, 19:14:28 DEBUG 1713550468, 19:14:28 DEBUG "value": [ 19:14:28 DEBUG ] 19:14:28 DEBUG 19:14:28 DEBUG 1713550468, 19:14:28 DEBUG }, 19:14:28 DEBUG 1713550468, 19:14:28 DEBUG }, 19:14:28 DEBUG "instance": "10.142.0.43:9100" 19:14:28 DEBUG }, 19:14:28 DEBUG "value": [ 19:14:28 DEBUG "value": [ 19:14:28 DEBUG }, 19:14:28 DEBUG ] 19:14:28 DEBUG 1713550468, 19:14:28 DEBUG "0.000014241092945494208" 19:14:28 DEBUG "value": [ 19:14:28 DEBUG "10909901.2" 19:14:28 DEBUG 1713550468, 19:14:28 DEBUG } 19:14:28 DEBUG } 19:14:28 DEBUG "193273.5507404852" 19:14:28 DEBUG "value": [ 19:14:28 DEBUG "0" 19:14:28 DEBUG "value": [ 19:14:28 DEBUG }, 19:14:28 DEBUG "value": [ 19:14:28 DEBUG 1713550468, 19:14:28 DEBUG 1713550468, 19:14:28 DEBUG "value": [ 19:14:28 DEBUG } 19:14:28 DEBUG "0" 19:14:28 DEBUG ] 19:14:28 DEBUG 1713550468, 19:14:28 DEBUG ] 19:14:28 DEBUG "1260.0666666666666" 19:14:28 DEBUG 19:14:28 DEBUG ] 19:14:28 DEBUG 1713550468, 19:14:28 DEBUG "770.8062433333333" 19:14:28 DEBUG 1713550468, 19:14:28 DEBUG "value": [ 19:14:28 DEBUG 1713550468, 19:14:28 DEBUG "10909894.066666666" 19:14:28 DEBUG "18907.5" 19:14:28 DEBUG 1713550468, 19:14:28 DEBUG ] 19:14:28 DEBUG ] 19:14:28 DEBUG } 19:14:28 DEBUG "3327684608" 19:14:28 DEBUG } 19:14:28 DEBUG ] 19:14:28 DEBUG "instance": "10.142.0.43:9100" 19:14:28 DEBUG } 19:14:28 DEBUG ] 19:14:28 DEBUG ] 19:14:28 DEBUG "0" 19:14:28 DEBUG 1713550468, 19:14:28 DEBUG "1847861097472" 19:14:28 DEBUG ] 19:14:28 DEBUG ] 19:14:28 DEBUG "0.012333333333337275" 19:14:28 DEBUG } 19:14:28 DEBUG } 19:14:28 DEBUG ] 19:14:28 DEBUG ] 19:14:28 DEBUG ] 19:14:28 DEBUG } 19:14:28 DEBUG }, 19:14:28 DEBUG ] 19:14:28 DEBUG } 19:14:28 DEBUG } 19:14:28 DEBUG ] 19:14:28 DEBUG "0" 19:14:28 DEBUG ] 19:14:28 DEBUG } 19:14:28 DEBUG } 19:14:28 DEBUG ] 19:14:28 DEBUG } 19:14:28 DEBUG ] 19:14:28 DEBUG } 19:14:28 DEBUG } 19:14:28 DEBUG } 19:14:28 DEBUG ] 19:14:28 DEBUG "value": [ 19:14:28 DEBUG } 19:14:28 DEBUG } 19:14:28 DEBUG ] 19:14:28 DEBUG } 19:14:28 DEBUG ] 19:14:28 DEBUG } 19:14:28 DEBUG ] 19:14:28 DEBUG } 19:14:28 DEBUG } 19:14:28 DEBUG } 19:14:28 DEBUG } 19:14:28 DEBUG ] 19:14:28 DEBUG } 19:14:28 DEBUG } 19:14:28 DEBUG 1713550468, 19:14:28 DEBUG ] 19:14:28 DEBUG } 19:14:28 DEBUG ] 19:14:28 DEBUG } 19:14:28 DEBUG ] 19:14:28 DEBUG ] 19:14:28 DEBUG } 19:14:28 DEBUG } 19:14:28 DEBUG } 19:14:28 DEBUG } 19:14:28 DEBUG "0.015866666666658112" 19:14:28 DEBUG } 19:14:28 DEBUG } 19:14:28 DEBUG } 19:14:28 DEBUG ] 19:14:28 DEBUG } 19:14:28 DEBUG } 19:14:28 DEBUG ] 19:14:28 DEBUG } 19:14:28 DEBUG ] 19:14:28 DEBUG } 19:14:28 DEBUG } 19:14:28 DEBUG } 19:14:28 DEBUG } 19:14:28 DEBUG } 19:14:28 DEBUG } 19:14:28 DEBUG } 19:14:28 DEBUG } 19:14:28 DEBUG } 19:14:28 DEBUG ] 19:14:28 DEBUG } 19:14:28 DEBUG } 19:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713550528" 19:15:28 ERROR [http_cmd]: ERROR 19:15:28 ERROR --- status code --- 19:15:28 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:15:28 ERROR --- http response --- 19:15:28 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:15:28 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550528" 19:15:28 INFO [http_cmd]: http status code OK 19:15:28 DEBUG --- status code --- 19:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:28 DEBUG --- http response --- 19:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:15:28 DEBUG ----- output ----- 19:15:28 DEBUG { 19:15:28 DEBUG "status": "success", 19:15:28 DEBUG "data": { 19:15:28 DEBUG "resultType": "vector", 19:15:28 DEBUG "result": [ 19:15:28 DEBUG { 19:15:28 DEBUG "metric": { 19:15:28 DEBUG "pod": "ds-idrepo-0" 19:15:28 DEBUG }, 19:15:28 DEBUG "value": [ 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG [...] 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG "pod": "ds-idrepo-2" 19:15:28 DEBUG }, 19:15:28 DEBUG "value": [ 19:15:28 DEBUG 1713550528, 19:15:28 DEBUG "0" 19:15:28 DEBUG ] 19:15:28 DEBUG } 19:15:28 DEBUG ] 19:15:28 DEBUG } 19:15:28 DEBUG } 19:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550528" 19:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550528" 19:15:28 INFO [http_cmd]: http status code OK 19:15:28 DEBUG --- status code --- 19:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:28 DEBUG --- http response --- 19:15:28 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550528" 19:15:28 INFO [http_cmd]: http status code OK 19:15:28 DEBUG --- status code --- 19:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:28 DEBUG --- http response --- 19:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:15:28 DEBUG ----- output ----- 19:15:28 DEBUG { 19:15:28 DEBUG "status": "success", 19:15:28 DEBUG "data": { 19:15:28 DEBUG "resultType": "vector", 19:15:28 DEBUG "result": [ 19:15:28 DEBUG { 19:15:28 DEBUG "metric": { 19:15:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:15:28 DEBUG }, 19:15:28 DEBUG "value": [ 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG [...] 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:15:28 DEBUG }, 19:15:28 DEBUG "value": [ 19:15:28 DEBUG 1713550528, 19:15:28 DEBUG "2347.4" 19:15:28 DEBUG ] 19:15:28 DEBUG } 19:15:28 DEBUG ] 19:15:28 DEBUG } 19:15:28 DEBUG } 19:15:28 INFO [http_cmd]: http status code OK 19:15:28 DEBUG --- status code --- 19:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:28 DEBUG --- http response --- 19:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:15:28 DEBUG ----- output ----- 19:15:28 DEBUG { 19:15:28 DEBUG "status": "success", 19:15:28 DEBUG "data": { 19:15:28 DEBUG "resultType": "vector", 19:15:28 DEBUG "result": [ 19:15:28 DEBUG { 19:15:28 DEBUG "metric": { 19:15:28 DEBUG "pod": "ds-idrepo-0" 19:15:28 DEBUG }, 19:15:28 DEBUG "value": [ 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG [...] 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG "pod": "ds-idrepo-2" 19:15:28 DEBUG }, 19:15:28 DEBUG "value": [ 19:15:28 DEBUG 1713550528, 19:15:28 DEBUG "0" 19:15:28 DEBUG ] 19:15:28 DEBUG } 19:15:28 DEBUG ] 19:15:28 DEBUG } 19:15:28 DEBUG } 19:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550528" 19:15:28 INFO [http_cmd]: http status code OK 19:15:28 DEBUG --- status code --- 19:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:28 DEBUG --- http response --- 19:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:15:28 DEBUG ----- output ----- 19:15:28 DEBUG { 19:15:28 DEBUG "status": "success", 19:15:28 DEBUG "data": { 19:15:28 DEBUG "resultType": "vector", 19:15:28 DEBUG "result": [ 19:15:28 DEBUG { 19:15:28 DEBUG "metric": { 19:15:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:15:28 DEBUG }, 19:15:28 DEBUG "value": [ 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG [...] 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:15:28 DEBUG }, 19:15:28 DEBUG "value": [ 19:15:28 DEBUG 1713550528, 19:15:28 DEBUG "0" 19:15:28 DEBUG ] 19:15:28 DEBUG } 19:15:28 DEBUG ] 19:15:28 DEBUG } 19:15:28 DEBUG } 19:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713550528" 19:15:28 INFO 19:15:28 INFO [loop_until]: kubectl --namespace=xlou top pods 19:15:28 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:15:28 INFO 19:15:28 INFO [loop_until]: kubectl --namespace=xlou top node 19:15:28 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:15:28 INFO [http_cmd]: http status code OK 19:15:28 DEBUG --- status code --- 19:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:28 DEBUG --- http response --- 19:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:15:28 DEBUG ----- output ----- 19:15:28 DEBUG { 19:15:28 DEBUG "status": "success", 19:15:28 DEBUG "data": { 19:15:28 DEBUG "resultType": "vector", 19:15:28 DEBUG "result": [ 19:15:28 DEBUG { 19:15:28 DEBUG "metric": { 19:15:28 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:15:28 DEBUG }, 19:15:28 DEBUG "value": [ 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG [...] 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:15:28 DEBUG }, 19:15:28 DEBUG "value": [ 19:15:28 DEBUG 1713550528, 19:15:28 DEBUG "7.7252151840211365" 19:15:28 DEBUG ] 19:15:28 DEBUG } 19:15:28 DEBUG ] 19:15:28 DEBUG } 19:15:28 DEBUG } 19:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550528" 19:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550528" 19:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713550528" 19:15:28 INFO [http_cmd]: http status code OK 19:15:28 DEBUG --- status code --- 19:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:28 DEBUG --- http response --- 19:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 19:15:28 DEBUG ----- output ----- 19:15:28 DEBUG { 19:15:28 DEBUG "status": "success", 19:15:28 DEBUG "data": { 19:15:28 DEBUG "resultType": "vector", 19:15:28 DEBUG "result": [ 19:15:28 DEBUG { 19:15:28 DEBUG "metric": { 19:15:28 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:15:28 DEBUG }, 19:15:28 DEBUG "value": [ 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG [...] 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG "pod": "ds-cts-0" 19:15:28 DEBUG }, 19:15:28 DEBUG "value": [ 19:15:28 DEBUG 1713550528, 19:15:28 DEBUG "24650491.40485739" 19:15:28 DEBUG ] 19:15:28 DEBUG } 19:15:28 DEBUG ] 19:15:28 DEBUG } 19:15:28 DEBUG } 19:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713550528" 19:15:28 INFO [http_cmd]: http status code OK 19:15:28 INFO [http_cmd]: http status code OK 19:15:28 DEBUG --- status code --- 19:15:28 DEBUG --- status code --- 19:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:28 DEBUG --- http response --- 19:15:28 DEBUG --- http response --- 19:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:15:28 DEBUG ----- output ----- 19:15:28 DEBUG { 19:15:28 DEBUG "status": "success", 19:15:28 DEBUG "data": { 19:15:28 DEBUG "resultType": "vector", 19:15:28 DEBUG "result": [ 19:15:28 DEBUG { 19:15:28 DEBUG "metric": { 19:15:28 DEBUG "pod": "ds-idrepo-0" 19:15:28 DEBUG }, 19:15:28 DEBUG "value": [ 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG [...] 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG "pod": "ds-cts-2" 19:15:28 DEBUG }, 19:15:28 DEBUG "value": [ 19:15:28 DEBUG 1713550528, 19:15:28 DEBUG "0" 19:15:28 DEBUG ] 19:15:28 DEBUG } 19:15:28 DEBUG ] 19:15:28 DEBUG } 19:15:28 DEBUG } 19:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:15:28 DEBUG ----- output ----- 19:15:28 DEBUG { 19:15:28 DEBUG "status": "success", 19:15:28 DEBUG "data": { 19:15:28 DEBUG "resultType": "vector", 19:15:28 DEBUG "result": [ 19:15:28 DEBUG { 19:15:28 DEBUG "metric": { 19:15:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:15:28 DEBUG }, 19:15:28 DEBUG "value": [ 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG [...] 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:15:28 DEBUG }, 19:15:28 DEBUG "value": [ 19:15:28 DEBUG 1713550528, 19:15:28 DEBUG "0" 19:15:28 DEBUG ] 19:15:28 DEBUG } 19:15:28 DEBUG ] 19:15:28 DEBUG } 19:15:28 DEBUG } 19:15:28 INFO [loop_until]: OK (rc = 0) 19:15:28 DEBUG --- stdout --- 19:15:28 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 11310m 4797Mi am-767f6b67b5-qcqz2 11432m 4630Mi am-767f6b67b5-vbltz 11944m 4804Mi ds-cts-0 6116m 10821Mi ds-cts-1 6018m 11075Mi ds-cts-2 6241m 10827Mi ds-idrepo-0 5698m 3487Mi ds-idrepo-1 5528m 3404Mi ds-idrepo-2 5438m 3492Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 2m 61Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7830m 3111Mi 19:15:28 DEBUG --- stderr --- 19:15:28 DEBUG 19:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550528" 19:15:28 INFO [http_cmd]: http status code OK 19:15:28 DEBUG --- status code --- 19:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:28 DEBUG --- http response --- 19:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:15:28 DEBUG ----- output ----- 19:15:28 DEBUG { 19:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550528" 19:15:28 DEBUG "status": "success", 19:15:28 DEBUG "data": { 19:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550528" 19:15:28 DEBUG "resultType": "vector", 19:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550528" 19:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550528" 19:15:28 DEBUG "result": [ 19:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550528" 19:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550528" 19:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550528" 19:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550528" 19:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550528" 19:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550528" 19:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550528" 19:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713550528" 19:15:28 DEBUG { 19:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550528" 19:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550528" 19:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550528" 19:15:28 INFO [loop_until]: OK (rc = 0) 19:15:28 INFO [http_cmd]: http status code OK 19:15:28 INFO [http_cmd]: http status code OK 19:15:28 INFO [http_cmd]: http status code OK 19:15:28 INFO [http_cmd]: http status code OK 19:15:28 DEBUG "metric": { 19:15:28 INFO [http_cmd]: http status code OK 19:15:28 INFO [http_cmd]: http status code OK 19:15:28 INFO [http_cmd]: http status code OK 19:15:28 INFO [http_cmd]: http status code OK 19:15:28 DEBUG --- stdout --- 19:15:28 DEBUG --- status code --- 19:15:28 INFO [http_cmd]: http status code OK 19:15:28 INFO [http_cmd]: http status code OK 19:15:28 INFO [http_cmd]: http status code OK 19:15:28 INFO [http_cmd]: http status code OK 19:15:28 DEBUG --- status code --- 19:15:28 DEBUG --- status code --- 19:15:28 DEBUG --- status code --- 19:15:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:15:28 DEBUG --- status code --- 19:15:28 DEBUG --- status code --- 19:15:28 DEBUG --- status code --- 19:15:28 DEBUG --- status code --- 19:15:28 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 73m 0% 6240Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5531m 18% 7066Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 11283m 37% 7494Mi 6% gke-xlou-cdm-default-pool-1b0769c8-qkns 91m 0% 3370Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 6239m 20% 13940Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 155m 0% 5191Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 11452m 38% 7494Mi 6% gke-xlou-cdm-ds-0f601751-cf7b 12374m 41% 3013Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 13117m 43% 7018Mi 6% gke-xlou-cdm-ds-0f601751-j488 6081m 20% 6718Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 5928m 19% 15980Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 6146m 20% 7306Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 6279m 21% 13393Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 7928m 49% 6928Mi 11% 19:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:28 DEBUG --- status code --- 19:15:28 DEBUG --- status code --- 19:15:28 DEBUG --- status code --- 19:15:28 DEBUG --- status code --- 19:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:28 DEBUG }, 19:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:28 INFO [http_cmd]: http status code OK 19:15:28 DEBUG --- stderr --- 19:15:28 DEBUG --- http response --- 19:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:28 DEBUG --- http response --- 19:15:28 DEBUG --- http response --- 19:15:28 DEBUG --- http response --- 19:15:28 DEBUG "value": [ 19:15:28 DEBUG --- http response --- 19:15:28 DEBUG --- http response --- 19:15:28 DEBUG --- http response --- 19:15:28 DEBUG --- http response --- 19:15:28 DEBUG --- status code --- 19:15:28 DEBUG 19:15:28 DEBUG --- http response --- 19:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:15:28 DEBUG --- http response --- 19:15:28 DEBUG --- http response --- 19:15:28 DEBUG --- http response --- 19:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:15:28 DEBUG 19:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 19:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:28 INFO [http_cmd]: http status code OK 19:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 19:15:28 INFO [http_cmd]: http status code OK 19:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:15:28 DEBUG ----- output ----- 19:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 19:15:28 INFO [http_cmd]: http status code OK 19:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:15:28 DEBUG ----- output ----- 19:15:28 DEBUG ----- output ----- 19:15:28 DEBUG 19:15:28 DEBUG ----- output ----- 19:15:28 DEBUG ----- output ----- 19:15:28 DEBUG ----- output ----- 19:15:28 DEBUG ----- output ----- 19:15:28 DEBUG --- http response --- 19:15:28 DEBUG --- status code --- 19:15:28 DEBUG ----- output ----- 19:15:28 DEBUG --- status code --- 19:15:28 DEBUG ----- output ----- 19:15:28 DEBUG { 19:15:28 DEBUG ----- output ----- 19:15:28 DEBUG ----- output ----- 19:15:28 DEBUG --- status code --- 19:15:28 DEBUG ----- output ----- 19:15:28 DEBUG { 19:15:28 DEBUG { 19:15:28 DEBUG 19:15:28 DEBUG { 19:15:28 DEBUG { 19:15:28 DEBUG { 19:15:28 DEBUG { 19:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:15:28 DEBUG { 19:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:28 DEBUG { 19:15:28 DEBUG "status": "success", 19:15:28 DEBUG { 19:15:28 DEBUG { 19:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:28 DEBUG { 19:15:28 DEBUG "status": "success", 19:15:28 DEBUG "status": "success", 19:15:28 DEBUG [...] 19:15:28 DEBUG "status": "success", 19:15:28 DEBUG "status": "success", 19:15:28 DEBUG "status": "success", 19:15:28 DEBUG "status": "success", 19:15:28 DEBUG --- http response --- 19:15:28 DEBUG ----- output ----- 19:15:28 DEBUG "status": "success", 19:15:28 DEBUG --- http response --- 19:15:28 DEBUG "status": "success", 19:15:28 DEBUG "data": { 19:15:28 DEBUG "status": "success", 19:15:28 DEBUG "status": "success", 19:15:28 DEBUG --- http response --- 19:15:28 DEBUG "status": "success", 19:15:28 DEBUG "data": { 19:15:28 DEBUG "data": { 19:15:28 DEBUG 19:15:28 DEBUG "data": { 19:15:28 DEBUG "data": { 19:15:28 DEBUG "data": { 19:15:28 DEBUG "data": { 19:15:28 DEBUG { 19:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:15:28 DEBUG "data": { 19:15:28 DEBUG "data": { 19:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:15:28 DEBUG "resultType": "vector", 19:15:28 DEBUG "data": { 19:15:28 DEBUG "data": { 19:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:15:28 DEBUG "data": { 19:15:28 DEBUG "resultType": "vector", 19:15:28 DEBUG "resultType": "vector", 19:15:28 DEBUG 19:15:28 DEBUG "resultType": "vector", 19:15:28 DEBUG "resultType": "vector", 19:15:28 DEBUG "resultType": "vector", 19:15:28 DEBUG "resultType": "vector", 19:15:28 DEBUG "status": "success", 19:15:28 DEBUG ----- output ----- 19:15:28 DEBUG "resultType": "vector", 19:15:28 DEBUG "resultType": "vector", 19:15:28 DEBUG ----- output ----- 19:15:28 DEBUG "result": [ 19:15:28 DEBUG "resultType": "vector", 19:15:28 DEBUG "resultType": "vector", 19:15:28 DEBUG ----- output ----- 19:15:28 DEBUG "resultType": "vector", 19:15:28 DEBUG "result": [ 19:15:28 DEBUG "result": [ 19:15:28 DEBUG 19:15:28 DEBUG "result": [ 19:15:28 DEBUG "result": [ 19:15:28 DEBUG "result": [ 19:15:28 DEBUG "result": [ 19:15:28 DEBUG "data": { 19:15:28 DEBUG { 19:15:28 DEBUG "result": [ 19:15:28 DEBUG "result": [ 19:15:28 DEBUG { 19:15:28 DEBUG { 19:15:28 DEBUG "result": [ 19:15:28 DEBUG "result": [ 19:15:28 DEBUG { 19:15:28 DEBUG "result": [ 19:15:28 DEBUG { 19:15:28 DEBUG { 19:15:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:15:28 DEBUG { 19:15:28 DEBUG { 19:15:28 DEBUG { 19:15:28 DEBUG { 19:15:28 DEBUG "resultType": "vector", 19:15:28 DEBUG "status": "success", 19:15:28 DEBUG { 19:15:28 DEBUG { 19:15:28 DEBUG "status": "success", 19:15:28 DEBUG "metric": { 19:15:28 DEBUG { 19:15:28 DEBUG { 19:15:28 DEBUG "status": "success", 19:15:28 DEBUG { 19:15:28 DEBUG "metric": { 19:15:28 DEBUG "metric": { 19:15:28 DEBUG }, 19:15:28 DEBUG "metric": { 19:15:28 DEBUG "metric": { 19:15:28 DEBUG "metric": { 19:15:28 DEBUG "metric": { 19:15:28 DEBUG "result": [ 19:15:28 DEBUG "data": { 19:15:28 DEBUG "metric": { 19:15:28 DEBUG "metric": { 19:15:28 DEBUG "data": { 19:15:28 DEBUG "pod": "ds-cts-0" 19:15:28 DEBUG "metric": { 19:15:28 DEBUG "metric": { 19:15:28 DEBUG "data": { 19:15:28 DEBUG "metric": { 19:15:28 DEBUG "instance": "10.142.0.107:9100" 19:15:28 DEBUG "instance": "10.142.0.107:9100" 19:15:28 DEBUG "value": [ 19:15:28 DEBUG "pod": "ds-cts-0" 19:15:28 DEBUG "instance": "10.142.0.107:9100" 19:15:28 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:15:28 DEBUG "pod": "ds-idrepo-0" 19:15:28 DEBUG { 19:15:28 DEBUG "resultType": "vector", 19:15:28 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:15:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:15:28 DEBUG "resultType": "vector", 19:15:28 DEBUG }, 19:15:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:15:28 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:15:28 DEBUG "resultType": "vector", 19:15:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:15:28 DEBUG }, 19:15:28 DEBUG }, 19:15:28 DEBUG 1713550528, 19:15:28 DEBUG }, 19:15:28 DEBUG }, 19:15:28 DEBUG }, 19:15:28 DEBUG }, 19:15:28 DEBUG "metric": { 19:15:28 DEBUG "result": [ 19:15:28 DEBUG }, 19:15:28 DEBUG }, 19:15:28 DEBUG "result": [ 19:15:28 DEBUG "value": [ 19:15:28 DEBUG }, 19:15:28 DEBUG }, 19:15:28 DEBUG "result": [ 19:15:28 DEBUG }, 19:15:28 DEBUG "value": [ 19:15:28 DEBUG "value": [ 19:15:28 DEBUG "0.002" 19:15:28 DEBUG "value": [ 19:15:28 DEBUG "value": [ 19:15:28 DEBUG "value": [ 19:15:28 DEBUG "value": [ 19:15:28 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:15:28 DEBUG { 19:15:28 DEBUG "value": [ 19:15:28 DEBUG "value": [ 19:15:28 DEBUG { 19:15:28 DEBUG 19:15:28 DEBUG "value": [ 19:15:28 DEBUG "value": [ 19:15:28 DEBUG { 19:15:28 DEBUG "value": [ 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG ] 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG }, 19:15:28 DEBUG "metric": { 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG "metric": { 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG "metric": { 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG } 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG "value": [ 19:15:28 DEBUG "pod": "ds-cts-0" 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG "instance": "10.142.0.107:9100" 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG "instance": "10.142.0.107:9100" 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG ] 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG }, 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG }, 19:15:28 DEBUG [...] 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG }, 19:15:28 DEBUG 19:15:28 DEBUG [...] 19:15:28 DEBUG [...] 19:15:28 DEBUG } 19:15:28 DEBUG [...] 19:15:28 DEBUG [...] 19:15:28 DEBUG [...] 19:15:28 DEBUG [...] 19:15:28 DEBUG 19:15:28 DEBUG "value": [ 19:15:28 DEBUG [...] 19:15:28 DEBUG [...] 19:15:28 DEBUG "value": [ 19:15:28 DEBUG 19:15:28 DEBUG [...] 19:15:28 DEBUG [...] 19:15:28 DEBUG "value": [ 19:15:28 DEBUG [...] 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG } 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG [...] 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG "instance": "10.142.0.43:9100" 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG [...] 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG "instance": "10.142.0.43:9100" 19:15:28 DEBUG "instance": "10.142.0.43:9100" 19:15:28 DEBUG "pod": "ds-cts-2" 19:15:28 DEBUG 19:15:28 DEBUG }, 19:15:28 DEBUG "pod": "ds-idrepo-2" 19:15:28 DEBUG 19:15:28 DEBUG [...] 19:15:28 DEBUG "pod": "ds-cts-0" 19:15:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:15:28 DEBUG "pod": "ds-cts-2" 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG [...] 19:15:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:15:28 DEBUG }, 19:15:28 DEBUG }, 19:15:28 DEBUG }, 19:15:28 DEBUG "pod": "ds-cts-0" 19:15:28 DEBUG "value": [ 19:15:28 DEBUG }, 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG }, 19:15:28 DEBUG }, 19:15:28 DEBUG }, 19:15:28 DEBUG "value": [ 19:15:28 DEBUG "pod": "ds-cts-0" 19:15:28 DEBUG "instance": "10.142.0.43:9100" 19:15:28 DEBUG 19:15:28 DEBUG 19:15:28 DEBUG "value": [ 19:15:28 DEBUG "value": [ 19:15:28 DEBUG "value": [ 19:15:28 DEBUG }, 19:15:28 DEBUG 1713550528, 19:15:28 DEBUG "value": [ 19:15:28 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:15:28 DEBUG 19:15:28 DEBUG "value": [ 19:15:28 DEBUG "value": [ 19:15:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:15:28 DEBUG }, 19:15:28 DEBUG "value": [ 19:15:28 DEBUG }, 19:15:28 DEBUG }, 19:15:28 DEBUG 19:15:28 DEBUG 1713550528, 19:15:28 DEBUG 1713550528, 19:15:28 DEBUG 1713550528, 19:15:28 DEBUG "value": [ 19:15:28 DEBUG "1849710164992" 19:15:28 DEBUG 1713550528, 19:15:28 DEBUG }, 19:15:28 DEBUG 19:15:28 DEBUG 1713550528, 19:15:28 DEBUG 1713550528, 19:15:28 DEBUG 1713550528, 19:15:28 DEBUG 1713550528, 19:15:28 DEBUG }, 19:15:28 DEBUG "value": [ 19:15:28 DEBUG "value": [ 19:15:28 DEBUG "instance": "10.142.0.43:9100" 19:15:28 DEBUG "0.04450000000002395" 19:15:28 DEBUG "55817435.86666667" 19:15:28 DEBUG "0.000019326666666666666" 19:15:28 DEBUG ] 19:15:28 DEBUG ] 19:15:28 DEBUG "0" 19:15:28 DEBUG "value": [ 19:15:28 DEBUG "pod": "ds-cts-2" 19:15:28 DEBUG "3.9308069370697734" 19:15:28 DEBUG "2347.5333333333333" 19:15:28 DEBUG "0" 19:15:28 DEBUG "2347.4666666666667" 19:15:28 DEBUG ] 19:15:28 DEBUG "value": [ 19:15:28 DEBUG 1713550528, 19:15:28 DEBUG }, 19:15:28 DEBUG ] 19:15:28 DEBUG ] 19:15:28 DEBUG 1713550528, 19:15:28 DEBUG } 19:15:28 DEBUG } 19:15:28 DEBUG ] 19:15:28 DEBUG 1713550528, 19:15:28 DEBUG }, 19:15:28 DEBUG ] 19:15:28 DEBUG ] 19:15:28 DEBUG ] 19:15:28 DEBUG 1713550528, 19:15:28 DEBUG } 19:15:28 DEBUG ] 19:15:28 DEBUG "142800" 19:15:28 DEBUG "value": [ 19:15:28 DEBUG } 19:15:28 DEBUG } 19:15:28 DEBUG "279931.40133674396" 19:15:28 DEBUG ] 19:15:28 DEBUG ] 19:15:28 DEBUG } 19:15:28 DEBUG "3357302784" 19:15:28 DEBUG "value": [ 19:15:28 DEBUG } 19:15:28 DEBUG } 19:15:28 DEBUG } 19:15:28 DEBUG "0.030000000000004926" 19:15:28 DEBUG 1713550528, 19:15:28 DEBUG "3900210.1522567873" 19:15:28 DEBUG ] 19:15:28 DEBUG 1713550528, 19:15:28 DEBUG "55815701.133333325" 19:15:28 DEBUG ] 19:15:28 DEBUG ] 19:15:28 DEBUG } 19:15:28 DEBUG } 19:15:28 DEBUG ] 19:15:28 DEBUG ] 19:15:28 DEBUG 1713550528, 19:15:28 DEBUG ] 19:15:28 DEBUG ] 19:15:28 DEBUG ] 19:15:28 DEBUG ] 19:15:28 DEBUG } 19:15:28 DEBUG ] 19:15:28 DEBUG } 19:15:28 DEBUG ] 19:15:28 DEBUG ] 19:15:28 DEBUG } 19:15:28 DEBUG ] 19:15:28 DEBUG } 19:15:28 DEBUG } 19:15:28 DEBUG } 19:15:28 DEBUG } 19:15:28 DEBUG "0" 19:15:28 DEBUG } 19:15:28 DEBUG } 19:15:28 DEBUG } 19:15:28 DEBUG } 19:15:28 DEBUG } 19:15:28 DEBUG } 19:15:28 DEBUG ] 19:15:28 DEBUG } 19:15:28 DEBUG } 19:15:28 DEBUG } 19:15:28 DEBUG } 19:15:28 DEBUG } 19:15:28 DEBUG ] 19:15:28 DEBUG ] 19:15:28 DEBUG } 19:15:28 DEBUG } 19:15:28 DEBUG } 19:15:28 DEBUG ] 19:15:28 DEBUG ] 19:15:28 DEBUG } 19:15:28 DEBUG } 19:15:28 DEBUG } 19:15:28 DEBUG ] 19:15:28 DEBUG } 19:15:28 DEBUG } 19:15:28 DEBUG } 19:15:28 DEBUG ] 19:15:28 DEBUG } 19:15:28 DEBUG } 19:15:28 DEBUG } 19:15:28 DEBUG } 19:15:28 DEBUG } 19:15:28 DEBUG } 19:15:28 DEBUG } 19:15:28 DEBUG } 19:15:28 DEBUG } 19:15:28 DEBUG } 19:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713550588" 19:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550588" 19:16:28 ERROR [http_cmd]: ERROR 19:16:28 ERROR --- status code --- 19:16:28 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:16:28 ERROR --- http response --- 19:16:28 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:16:28 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550588" 19:16:28 INFO [http_cmd]: http status code OK 19:16:28 DEBUG --- status code --- 19:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:28 DEBUG --- http response --- 19:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:16:28 DEBUG ----- output ----- 19:16:28 DEBUG { 19:16:28 DEBUG "status": "success", 19:16:28 DEBUG "data": { 19:16:28 DEBUG "resultType": "vector", 19:16:28 DEBUG "result": [ 19:16:28 DEBUG { 19:16:28 DEBUG "metric": { 19:16:28 DEBUG "pod": "ds-idrepo-0" 19:16:28 DEBUG }, 19:16:28 DEBUG "value": [ 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG [...] 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG "pod": "ds-idrepo-2" 19:16:28 DEBUG }, 19:16:28 DEBUG "value": [ 19:16:28 DEBUG 1713550588, 19:16:28 DEBUG "0" 19:16:28 DEBUG ] 19:16:28 DEBUG } 19:16:28 DEBUG ] 19:16:28 DEBUG } 19:16:28 DEBUG } 19:16:28 INFO [http_cmd]: http status code OK 19:16:28 DEBUG --- status code --- 19:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:28 DEBUG --- http response --- 19:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:16:28 DEBUG ----- output ----- 19:16:28 DEBUG { 19:16:28 DEBUG "status": "success", 19:16:28 DEBUG "data": { 19:16:28 DEBUG "resultType": "vector", 19:16:28 DEBUG "result": [ 19:16:28 DEBUG { 19:16:28 DEBUG "metric": { 19:16:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:16:28 DEBUG }, 19:16:28 DEBUG "value": [ 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG [...] 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:16:28 DEBUG }, 19:16:28 DEBUG "value": [ 19:16:28 DEBUG 1713550588, 19:16:28 DEBUG "2335.9333333333334" 19:16:28 DEBUG ] 19:16:28 DEBUG } 19:16:28 DEBUG ] 19:16:28 DEBUG } 19:16:28 DEBUG } 19:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550588" 19:16:28 INFO [http_cmd]: http status code OK 19:16:28 DEBUG --- status code --- 19:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:28 DEBUG --- http response --- 19:16:28 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550588" 19:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550588" 19:16:28 INFO [http_cmd]: http status code OK 19:16:28 DEBUG --- status code --- 19:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:28 DEBUG --- http response --- 19:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:16:28 DEBUG ----- output ----- 19:16:28 DEBUG { 19:16:28 DEBUG "status": "success", 19:16:28 DEBUG "data": { 19:16:28 DEBUG "resultType": "vector", 19:16:28 DEBUG "result": [ 19:16:28 DEBUG { 19:16:28 DEBUG "metric": { 19:16:28 DEBUG "pod": "ds-idrepo-0" 19:16:28 DEBUG }, 19:16:28 DEBUG "value": [ 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG [...] 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG "pod": "ds-idrepo-2" 19:16:28 DEBUG }, 19:16:28 DEBUG "value": [ 19:16:28 DEBUG 1713550588, 19:16:28 DEBUG "0" 19:16:28 DEBUG ] 19:16:28 DEBUG } 19:16:28 DEBUG ] 19:16:28 DEBUG } 19:16:28 DEBUG } 19:16:28 INFO [http_cmd]: http status code OK 19:16:28 DEBUG --- status code --- 19:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:28 DEBUG --- http response --- 19:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:16:28 DEBUG ----- output ----- 19:16:28 DEBUG { 19:16:28 DEBUG "status": "success", 19:16:28 DEBUG "data": { 19:16:28 DEBUG "resultType": "vector", 19:16:28 DEBUG "result": [ 19:16:28 DEBUG { 19:16:28 DEBUG "metric": { 19:16:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:16:28 DEBUG }, 19:16:28 DEBUG "value": [ 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG [...] 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:16:28 DEBUG }, 19:16:28 DEBUG "value": [ 19:16:28 DEBUG 1713550588, 19:16:28 DEBUG "0" 19:16:28 DEBUG ] 19:16:28 DEBUG } 19:16:28 DEBUG ] 19:16:28 DEBUG } 19:16:28 DEBUG } 19:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713550588" 19:16:28 INFO [http_cmd]: http status code OK 19:16:28 DEBUG --- status code --- 19:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:28 DEBUG --- http response --- 19:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:16:28 DEBUG ----- output ----- 19:16:28 DEBUG { 19:16:28 DEBUG "status": "success", 19:16:28 DEBUG "data": { 19:16:28 DEBUG "resultType": "vector", 19:16:28 DEBUG "result": [ 19:16:28 DEBUG { 19:16:28 DEBUG "metric": { 19:16:28 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:16:28 DEBUG }, 19:16:28 DEBUG "value": [ 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG [...] 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:16:28 DEBUG }, 19:16:28 DEBUG "value": [ 19:16:28 DEBUG 1713550588, 19:16:28 DEBUG "7.74788212136222" 19:16:28 DEBUG ] 19:16:28 DEBUG } 19:16:28 DEBUG ] 19:16:28 DEBUG } 19:16:28 DEBUG } 19:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550588" 19:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713550588" 19:16:28 INFO 19:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550588" 19:16:28 INFO [loop_until]: kubectl --namespace=xlou top pods 19:16:28 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:16:28 INFO [http_cmd]: http status code OK 19:16:28 DEBUG --- status code --- 19:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:28 DEBUG --- http response --- 19:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:16:28 DEBUG ----- output ----- 19:16:28 DEBUG { 19:16:28 DEBUG "status": "success", 19:16:28 DEBUG "data": { 19:16:28 DEBUG "resultType": "vector", 19:16:28 DEBUG "result": [ 19:16:28 DEBUG { 19:16:28 DEBUG "metric": { 19:16:28 DEBUG "pod": "ds-idrepo-1" 19:16:28 DEBUG }, 19:16:28 DEBUG "value": [ 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG [...] 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG "pod": "ds-idrepo-2" 19:16:28 DEBUG }, 19:16:28 DEBUG "value": [ 19:16:28 DEBUG 1713550588, 19:16:28 DEBUG "4075.130392836685" 19:16:28 DEBUG ] 19:16:28 DEBUG } 19:16:28 DEBUG ] 19:16:28 DEBUG } 19:16:28 DEBUG } 19:16:28 INFO [http_cmd]: http status code OK 19:16:28 INFO [http_cmd]: http status code OK 19:16:28 DEBUG --- status code --- 19:16:28 DEBUG --- status code --- 19:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:28 DEBUG --- http response --- 19:16:28 DEBUG --- http response --- 19:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:16:28 DEBUG ----- output ----- 19:16:28 DEBUG { 19:16:28 DEBUG "status": "success", 19:16:28 DEBUG "data": { 19:16:28 DEBUG "resultType": "vector", 19:16:28 DEBUG "result": [ 19:16:28 DEBUG { 19:16:28 DEBUG "metric": { 19:16:28 DEBUG "pod": "ds-idrepo-0" 19:16:28 DEBUG }, 19:16:28 DEBUG "value": [ 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG [...] 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG "pod": "ds-cts-2" 19:16:28 DEBUG }, 19:16:28 DEBUG "value": [ 19:16:28 DEBUG 1713550588, 19:16:28 DEBUG "0" 19:16:28 DEBUG ] 19:16:28 DEBUG } 19:16:28 DEBUG ] 19:16:28 DEBUG } 19:16:28 DEBUG } 19:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:16:28 DEBUG ----- output ----- 19:16:28 DEBUG { 19:16:28 DEBUG "status": "success", 19:16:28 DEBUG "data": { 19:16:28 DEBUG "resultType": "vector", 19:16:28 DEBUG "result": [ 19:16:28 DEBUG { 19:16:28 DEBUG "metric": { 19:16:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:16:28 DEBUG }, 19:16:28 DEBUG "value": [ 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG [...] 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:16:28 DEBUG }, 19:16:28 DEBUG "value": [ 19:16:28 DEBUG 1713550588, 19:16:28 DEBUG "0" 19:16:28 DEBUG ] 19:16:28 DEBUG } 19:16:28 DEBUG ] 19:16:28 DEBUG } 19:16:28 DEBUG } 19:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550588" 19:16:28 INFO 19:16:28 INFO [loop_until]: kubectl --namespace=xlou top node 19:16:28 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:16:28 INFO [http_cmd]: http status code OK 19:16:28 DEBUG --- status code --- 19:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:28 DEBUG --- http response --- 19:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:16:28 DEBUG ----- output ----- 19:16:28 DEBUG { 19:16:28 DEBUG "status": "success", 19:16:28 DEBUG "data": { 19:16:28 DEBUG "resultType": "vector", 19:16:28 DEBUG "result": [ 19:16:28 DEBUG { 19:16:28 DEBUG "metric": { 19:16:28 DEBUG "pod": "ds-cts-0" 19:16:28 DEBUG }, 19:16:28 DEBUG "value": [ 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG [...] 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG "pod": "ds-cts-2" 19:16:28 DEBUG }, 19:16:28 DEBUG "value": [ 19:16:28 DEBUG 1713550588, 19:16:28 DEBUG "0" 19:16:28 DEBUG ] 19:16:28 DEBUG } 19:16:28 DEBUG ] 19:16:28 DEBUG } 19:16:28 DEBUG } 19:16:28 INFO [loop_until]: OK (rc = 0) 19:16:28 DEBUG --- stdout --- 19:16:28 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 11676m 7080Mi am-767f6b67b5-qcqz2 11658m 7068Mi am-767f6b67b5-vbltz 11997m 7142Mi ds-cts-0 6308m 10830Mi ds-cts-1 6107m 11088Mi ds-cts-2 6479m 10833Mi ds-idrepo-0 5672m 3511Mi ds-idrepo-1 5596m 3423Mi ds-idrepo-2 5520m 3492Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 2m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7760m 3117Mi 19:16:28 DEBUG --- stderr --- 19:16:28 DEBUG 19:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713550588" 19:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550588" 19:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550588" 19:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550588" 19:16:28 INFO [http_cmd]: http status code OK 19:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550588" 19:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550588" 19:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550588" 19:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550588" 19:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550588" 19:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550588" 19:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550588" 19:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550588" 19:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550588" 19:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550588" 19:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713550588" 19:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550588" 19:16:28 DEBUG --- status code --- 19:16:28 INFO [http_cmd]: http status code OK 19:16:28 INFO [http_cmd]: http status code OK 19:16:28 INFO [http_cmd]: http status code OK 19:16:28 INFO [http_cmd]: http status code OK 19:16:28 INFO [http_cmd]: http status code OK 19:16:28 INFO [http_cmd]: http status code OK 19:16:28 INFO [http_cmd]: http status code OK 19:16:28 DEBUG --- status code --- 19:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:28 INFO [http_cmd]: http status code OK 19:16:28 DEBUG --- status code --- 19:16:28 DEBUG --- status code --- 19:16:28 DEBUG --- status code --- 19:16:28 DEBUG --- status code --- 19:16:28 DEBUG --- status code --- 19:16:28 INFO [http_cmd]: http status code OK 19:16:28 INFO [http_cmd]: http status code OK 19:16:28 DEBUG --- status code --- 19:16:28 INFO [http_cmd]: http status code OK 19:16:28 INFO [http_cmd]: http status code OK 19:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:28 DEBUG --- http response --- 19:16:28 DEBUG --- status code --- 19:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:28 DEBUG --- http response --- 19:16:28 DEBUG --- status code --- 19:16:28 DEBUG --- status code --- 19:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:28 DEBUG --- status code --- 19:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:28 DEBUG --- http response --- 19:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:28 DEBUG --- http response --- 19:16:28 DEBUG --- http response --- 19:16:28 DEBUG --- http response --- 19:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:28 DEBUG --- http response --- 19:16:28 DEBUG --- status code --- 19:16:28 DEBUG --- http response --- 19:16:28 DEBUG ----- output ----- 19:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:16:28 DEBUG --- http response --- 19:16:28 INFO [http_cmd]: http status code OK 19:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:16:28 DEBUG --- http response --- 19:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:16:28 DEBUG ----- output ----- 19:16:28 DEBUG --- http response --- 19:16:28 DEBUG --- http response --- 19:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:16:28 DEBUG { 19:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 19:16:28 INFO [http_cmd]: http status code OK 19:16:28 DEBUG ----- output ----- 19:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 19:16:28 DEBUG --- status code --- 19:16:28 DEBUG ----- output ----- 19:16:28 INFO [http_cmd]: http status code OK 19:16:28 DEBUG ----- output ----- 19:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:16:28 DEBUG ----- output ----- 19:16:28 DEBUG { 19:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:16:28 DEBUG --- http response --- 19:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:16:28 DEBUG ----- output ----- 19:16:28 DEBUG "status": "success", 19:16:28 DEBUG ----- output ----- 19:16:28 DEBUG --- status code --- 19:16:28 DEBUG { 19:16:28 DEBUG ----- output ----- 19:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:28 DEBUG { 19:16:28 DEBUG --- status code --- 19:16:28 DEBUG { 19:16:28 DEBUG ----- output ----- 19:16:28 DEBUG { 19:16:28 DEBUG "status": "success", 19:16:28 DEBUG ----- output ----- 19:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:16:28 DEBUG ----- output ----- 19:16:28 DEBUG { 19:16:28 DEBUG "data": { 19:16:28 DEBUG { 19:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:28 DEBUG "status": "success", 19:16:28 DEBUG { 19:16:28 DEBUG --- http response --- 19:16:28 DEBUG "status": "success", 19:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:28 DEBUG "status": "success", 19:16:28 DEBUG { 19:16:28 DEBUG "status": "success", 19:16:28 DEBUG "data": { 19:16:28 DEBUG { 19:16:28 DEBUG ----- output ----- 19:16:28 DEBUG { 19:16:28 DEBUG "status": "success", 19:16:28 DEBUG "resultType": "vector", 19:16:28 DEBUG "status": "success", 19:16:28 DEBUG --- http response --- 19:16:28 DEBUG "data": { 19:16:28 DEBUG "status": "success", 19:16:28 DEBUG "data": { 19:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:16:28 DEBUG --- http response --- 19:16:28 DEBUG "data": { 19:16:28 DEBUG "status": "success", 19:16:28 DEBUG "data": { 19:16:28 DEBUG "resultType": "vector", 19:16:28 DEBUG "status": "success", 19:16:28 DEBUG { 19:16:28 DEBUG "status": "success", 19:16:28 DEBUG "data": { 19:16:28 DEBUG "result": [ 19:16:28 DEBUG "data": { 19:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:16:28 DEBUG "resultType": "vector", 19:16:28 DEBUG "data": { 19:16:28 DEBUG "resultType": "vector", 19:16:28 DEBUG ----- output ----- 19:16:28 DEBUG "resultType": "vector", 19:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:16:28 DEBUG "data": { 19:16:28 DEBUG "resultType": "vector", 19:16:28 DEBUG "result": [ 19:16:28 DEBUG "data": { 19:16:28 DEBUG "status": "success", 19:16:28 DEBUG "data": { 19:16:28 DEBUG "resultType": "vector", 19:16:28 DEBUG { 19:16:28 DEBUG "resultType": "vector", 19:16:28 DEBUG ----- output ----- 19:16:28 DEBUG "result": [ 19:16:28 DEBUG "resultType": "vector", 19:16:28 DEBUG "result": [ 19:16:28 DEBUG { 19:16:28 DEBUG "result": [ 19:16:28 DEBUG ----- output ----- 19:16:28 DEBUG "resultType": "vector", 19:16:28 DEBUG "result": [ 19:16:28 DEBUG { 19:16:28 DEBUG "resultType": "vector", 19:16:28 DEBUG "data": { 19:16:28 DEBUG "resultType": "vector", 19:16:28 DEBUG "result": [ 19:16:28 DEBUG "metric": { 19:16:28 DEBUG "result": [ 19:16:28 INFO [loop_until]: OK (rc = 0) 19:16:28 DEBUG { 19:16:28 DEBUG { 19:16:28 DEBUG "result": [ 19:16:28 DEBUG { 19:16:28 DEBUG "status": "success", 19:16:28 DEBUG { 19:16:28 DEBUG { 19:16:28 DEBUG "result": [ 19:16:28 DEBUG { 19:16:28 DEBUG "metric": { 19:16:28 DEBUG "result": [ 19:16:28 DEBUG "resultType": "vector", 19:16:28 DEBUG "result": [ 19:16:28 DEBUG { 19:16:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:16:28 DEBUG { 19:16:28 DEBUG --- stdout --- 19:16:28 DEBUG "status": "success", 19:16:28 DEBUG "metric": { 19:16:28 DEBUG { 19:16:28 DEBUG "metric": { 19:16:28 DEBUG "data": { 19:16:28 DEBUG "metric": { 19:16:28 DEBUG "status": "success", 19:16:28 DEBUG { 19:16:28 DEBUG "metric": { 19:16:28 DEBUG "instance": "10.142.0.107:9100" 19:16:28 DEBUG { 19:16:28 DEBUG "result": [ 19:16:28 DEBUG { 19:16:28 DEBUG "metric": { 19:16:28 DEBUG }, 19:16:28 DEBUG "metric": { 19:16:28 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 69m 0% 6244Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5559m 18% 7103Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 11412m 38% 9766Mi 8% gke-xlou-cdm-default-pool-1b0769c8-qkns 82m 0% 3354Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 6349m 21% 13945Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 154m 0% 5202Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 11596m 38% 9923Mi 8% gke-xlou-cdm-ds-0f601751-cf7b 11937m 39% 3023Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 13212m 44% 9339Mi 8% gke-xlou-cdm-ds-0f601751-j488 5994m 20% 6718Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 5913m 19% 15996Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 6206m 20% 7304Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 6373m 21% 13423Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 7893m 49% 6934Mi 11% 19:16:28 DEBUG "data": { 19:16:28 DEBUG "instance": "10.142.0.107:9100" 19:16:28 DEBUG "metric": { 19:16:28 DEBUG "pod": "ds-cts-0" 19:16:28 DEBUG "resultType": "vector", 19:16:28 DEBUG "instance": "10.142.0.107:9100" 19:16:28 DEBUG "data": { 19:16:28 DEBUG "metric": { 19:16:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:16:28 DEBUG }, 19:16:28 DEBUG "metric": { 19:16:28 DEBUG { 19:16:28 DEBUG "metric": { 19:16:28 DEBUG "pod": "ds-idrepo-0" 19:16:28 DEBUG "value": [ 19:16:28 DEBUG "pod": "ds-idrepo-1" 19:16:28 DEBUG --- stderr --- 19:16:28 DEBUG "resultType": "vector", 19:16:28 DEBUG }, 19:16:28 DEBUG "pod": "ds-idrepo-1" 19:16:28 DEBUG }, 19:16:28 DEBUG "result": [ 19:16:28 DEBUG }, 19:16:28 DEBUG "resultType": "vector", 19:16:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:16:28 DEBUG }, 19:16:28 DEBUG "value": [ 19:16:28 DEBUG "pod": "ds-cts-0" 19:16:28 DEBUG "metric": { 19:16:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:16:28 DEBUG }, 19:16:28 DEBUG 19:16:28 DEBUG }, 19:16:28 DEBUG 19:16:28 DEBUG "result": [ 19:16:28 DEBUG "value": [ 19:16:28 DEBUG }, 19:16:28 DEBUG "value": [ 19:16:28 DEBUG { 19:16:28 DEBUG "value": [ 19:16:28 DEBUG "result": [ 19:16:28 DEBUG }, 19:16:28 DEBUG "value": [ 19:16:28 DEBUG 19:16:28 DEBUG }, 19:16:28 DEBUG "instance": "10.142.0.107:9100" 19:16:28 DEBUG }, 19:16:28 DEBUG "value": [ 19:16:28 DEBUG 19:16:28 DEBUG "value": [ 19:16:28 DEBUG { 19:16:28 DEBUG 19:16:28 DEBUG "value": [ 19:16:28 DEBUG 19:16:28 DEBUG "metric": { 19:16:28 DEBUG 19:16:28 DEBUG { 19:16:28 DEBUG "value": [ 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG "value": [ 19:16:28 DEBUG }, 19:16:28 DEBUG "value": [ 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG "metric": { 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG "pod": "ds-idrepo-1" 19:16:28 DEBUG 19:16:28 DEBUG "metric": { 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG "value": [ 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG [...] 19:16:28 DEBUG 19:16:28 DEBUG "instance": "10.142.0.107:9100" 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG }, 19:16:28 DEBUG 19:16:28 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG [...] 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG }, 19:16:28 DEBUG [...] 19:16:28 DEBUG 19:16:28 DEBUG [...] 19:16:28 DEBUG "value": [ 19:16:28 DEBUG [...] 19:16:28 DEBUG }, 19:16:28 DEBUG 19:16:28 DEBUG [...] 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG [...] 19:16:28 DEBUG 19:16:28 DEBUG [...] 19:16:28 DEBUG "value": [ 19:16:28 DEBUG 19:16:28 DEBUG [...] 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG "value": [ 19:16:28 DEBUG [...] 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG [...] 19:16:28 DEBUG 19:16:28 DEBUG [...] 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG [...] 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG "instance": "10.142.0.43:9100" 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG }, 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG "instance": "10.142.0.43:9100" 19:16:28 DEBUG 19:16:28 DEBUG "pod": "ds-cts-2" 19:16:28 DEBUG [...] 19:16:28 DEBUG "instance": "10.142.0.43:9100" 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:16:28 DEBUG }, 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG 19:16:28 DEBUG "pod": "ds-idrepo-2" 19:16:28 DEBUG "value": [ 19:16:28 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:16:28 DEBUG [...] 19:16:28 DEBUG }, 19:16:28 DEBUG "value": [ 19:16:28 DEBUG }, 19:16:28 DEBUG 19:16:28 DEBUG }, 19:16:28 DEBUG [...] 19:16:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:16:28 DEBUG }, 19:16:28 DEBUG "value": [ 19:16:28 DEBUG "pod": "ds-cts-2" 19:16:28 DEBUG 19:16:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:16:28 DEBUG }, 19:16:28 DEBUG 1713550588, 19:16:28 DEBUG }, 19:16:28 DEBUG 19:16:28 DEBUG 1713550588, 19:16:28 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:16:28 DEBUG "value": [ 19:16:28 DEBUG 19:16:28 DEBUG "value": [ 19:16:28 DEBUG 19:16:28 DEBUG }, 19:16:28 DEBUG "value": [ 19:16:28 DEBUG 1713550588, 19:16:28 DEBUG }, 19:16:28 DEBUG "instance": "10.142.0.43:9100" 19:16:28 DEBUG }, 19:16:28 DEBUG "value": [ 19:16:28 DEBUG "0.002" 19:16:28 DEBUG "value": [ 19:16:28 DEBUG 19:16:28 DEBUG "55845162.2" 19:16:28 DEBUG }, 19:16:28 DEBUG 1713550588, 19:16:28 DEBUG 19:16:28 DEBUG 1713550588, 19:16:28 DEBUG 19:16:28 DEBUG "value": [ 19:16:28 DEBUG 1713550588, 19:16:28 DEBUG "0.04516666666671843" 19:16:28 DEBUG "value": [ 19:16:28 DEBUG }, 19:16:28 DEBUG "value": [ 19:16:28 DEBUG 1713550588, 19:16:28 DEBUG ] 19:16:28 DEBUG } 19:16:28 DEBUG 19:16:28 DEBUG ] 19:16:28 DEBUG "value": [ 19:16:28 DEBUG "0.000019326666666666666" 19:16:28 DEBUG "pod": "ds-idrepo-2" 19:16:28 DEBUG "1852235455488" 19:16:28 DEBUG 19:16:28 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:16:28 DEBUG "2335.9333333333334" 19:16:28 DEBUG ] 19:16:28 DEBUG 1713550588, 19:16:28 DEBUG "value": [ 19:16:28 DEBUG 1713550588, 19:16:28 DEBUG "0" 19:16:28 DEBUG 1713550588, 19:16:28 DEBUG ] 19:16:28 DEBUG "instance": "10.142.0.43:9100" 19:16:28 DEBUG } 19:16:28 DEBUG 1713550588, 19:16:28 DEBUG ] 19:16:28 DEBUG }, 19:16:28 DEBUG ] 19:16:28 DEBUG 1713550588, 19:16:28 DEBUG }, 19:16:28 DEBUG ] 19:16:28 DEBUG } 19:16:28 DEBUG "0" 19:16:28 DEBUG 1713550588, 19:16:28 DEBUG "55847440.66666667" 19:16:28 DEBUG "283287" 19:16:28 DEBUG ] 19:16:28 DEBUG } 19:16:28 DEBUG }, 19:16:28 DEBUG ] 19:16:28 DEBUG "0" 19:16:28 DEBUG } 19:16:28 DEBUG "value": [ 19:16:28 DEBUG } 19:16:28 DEBUG "2335.7666666666664" 19:16:28 DEBUG "value": [ 19:16:28 DEBUG } 19:16:28 DEBUG ] 19:16:28 DEBUG ] 19:16:28 DEBUG ] 19:16:28 DEBUG ] 19:16:28 DEBUG } 19:16:28 DEBUG "0" 19:16:28 DEBUG } 19:16:28 DEBUG "value": [ 19:16:28 DEBUG } 19:16:28 DEBUG ] 19:16:28 DEBUG ] 19:16:28 DEBUG 1713550588, 19:16:28 DEBUG ] 19:16:28 DEBUG ] 19:16:28 DEBUG 1713550588, 19:16:28 DEBUG ] 19:16:28 DEBUG } 19:16:28 DEBUG } 19:16:28 DEBUG } 19:16:28 DEBUG } 19:16:28 DEBUG ] 19:16:28 DEBUG ] 19:16:28 DEBUG 1713550588, 19:16:28 DEBUG } 19:16:28 DEBUG } 19:16:28 DEBUG } 19:16:28 DEBUG "0" 19:16:28 DEBUG } 19:16:28 DEBUG } 19:16:28 DEBUG "3363467264" 19:16:28 DEBUG } 19:16:28 DEBUG } 19:16:28 DEBUG ] 19:16:28 DEBUG ] 19:16:28 DEBUG ] 19:16:28 DEBUG } 19:16:28 DEBUG } 19:16:28 DEBUG "0.03166666666666439" 19:16:28 DEBUG ] 19:16:28 DEBUG } 19:16:28 DEBUG ] 19:16:28 DEBUG } 19:16:28 DEBUG ] 19:16:28 DEBUG ] 19:16:28 DEBUG } 19:16:28 DEBUG } 19:16:28 DEBUG } 19:16:28 DEBUG } 19:16:28 DEBUG } 19:16:28 DEBUG ] 19:16:28 DEBUG ] 19:16:28 DEBUG } 19:16:28 DEBUG } 19:16:28 DEBUG } 19:16:28 DEBUG } 19:16:28 DEBUG } 19:16:28 DEBUG ] 19:16:28 DEBUG } 19:16:28 DEBUG } 19:16:28 DEBUG } 19:16:28 DEBUG } 19:16:28 DEBUG ] 19:16:28 DEBUG } 19:16:28 DEBUG } 19:16:28 DEBUG } 19:16:28 DEBUG } 19:16:28 DEBUG ] 19:16:28 DEBUG } 19:16:28 DEBUG } 19:16:28 DEBUG } 19:16:28 DEBUG } 19:16:28 DEBUG } 19:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713550648" 19:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550648" 19:17:28 ERROR [http_cmd]: ERROR 19:17:28 ERROR --- status code --- 19:17:28 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:17:28 ERROR --- http response --- 19:17:28 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:17:28 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550648" 19:17:28 INFO [http_cmd]: http status code OK 19:17:28 DEBUG --- status code --- 19:17:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:17:28 DEBUG --- http response --- 19:17:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:17:28 DEBUG ----- output ----- 19:17:28 DEBUG { 19:17:28 DEBUG "status": "success", 19:17:28 DEBUG "data": { 19:17:28 DEBUG "resultType": "vector", 19:17:28 DEBUG "result": [ 19:17:28 DEBUG { 19:17:28 DEBUG "metric": { 19:17:28 DEBUG "pod": "ds-idrepo-0" 19:17:28 DEBUG }, 19:17:28 DEBUG "value": [ 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG [...] 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG "pod": "ds-idrepo-2" 19:17:28 DEBUG }, 19:17:28 DEBUG "value": [ 19:17:28 DEBUG 1713550648, 19:17:28 DEBUG "0" 19:17:28 DEBUG ] 19:17:28 DEBUG } 19:17:28 DEBUG ] 19:17:28 DEBUG } 19:17:28 DEBUG } 19:17:28 INFO [http_cmd]: http status code OK 19:17:28 DEBUG --- status code --- 19:17:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:17:28 DEBUG --- http response --- 19:17:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:17:28 DEBUG ----- output ----- 19:17:28 DEBUG { 19:17:28 DEBUG "status": "success", 19:17:28 DEBUG "data": { 19:17:28 DEBUG "resultType": "vector", 19:17:28 DEBUG "result": [ 19:17:28 DEBUG { 19:17:28 DEBUG "metric": { 19:17:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:17:28 DEBUG }, 19:17:28 DEBUG "value": [ 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG [...] 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:17:28 DEBUG }, 19:17:28 DEBUG "value": [ 19:17:28 DEBUG 1713550648, 19:17:28 DEBUG "2337.8333333333335" 19:17:28 DEBUG ] 19:17:28 DEBUG } 19:17:28 DEBUG ] 19:17:28 DEBUG } 19:17:28 DEBUG } 19:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550648" 19:17:28 INFO [http_cmd]: http status code OK 19:17:28 DEBUG --- status code --- 19:17:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:17:28 DEBUG --- http response --- 19:17:28 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550648" 19:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550648" 19:17:28 INFO [http_cmd]: http status code OK 19:17:28 DEBUG --- status code --- 19:17:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:17:28 DEBUG --- http response --- 19:17:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:17:28 DEBUG ----- output ----- 19:17:28 DEBUG { 19:17:28 DEBUG "status": "success", 19:17:28 DEBUG "data": { 19:17:28 DEBUG "resultType": "vector", 19:17:28 DEBUG "result": [ 19:17:28 DEBUG { 19:17:28 DEBUG "metric": { 19:17:28 DEBUG "pod": "ds-idrepo-0" 19:17:28 DEBUG }, 19:17:28 DEBUG "value": [ 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG [...] 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG "pod": "ds-idrepo-2" 19:17:28 DEBUG }, 19:17:28 DEBUG "value": [ 19:17:28 DEBUG 1713550648, 19:17:28 DEBUG "0" 19:17:28 DEBUG ] 19:17:28 DEBUG } 19:17:28 DEBUG ] 19:17:28 DEBUG } 19:17:28 DEBUG } 19:17:28 INFO [http_cmd]: http status code OK 19:17:28 DEBUG --- status code --- 19:17:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:17:28 DEBUG --- http response --- 19:17:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:17:28 DEBUG ----- output ----- 19:17:28 DEBUG { 19:17:28 DEBUG "status": "success", 19:17:28 DEBUG "data": { 19:17:28 DEBUG "resultType": "vector", 19:17:28 DEBUG "result": [ 19:17:28 DEBUG { 19:17:28 DEBUG "metric": { 19:17:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:17:28 DEBUG }, 19:17:28 DEBUG "value": [ 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG [...] 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:17:28 DEBUG }, 19:17:28 DEBUG "value": [ 19:17:28 DEBUG 1713550648, 19:17:28 DEBUG "768.5570033333333" 19:17:28 DEBUG ] 19:17:28 DEBUG } 19:17:28 DEBUG ] 19:17:28 DEBUG } 19:17:28 DEBUG } 19:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713550648" 19:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550648" 19:17:28 INFO [http_cmd]: http status code OK 19:17:28 DEBUG --- status code --- 19:17:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:17:28 DEBUG --- http response --- 19:17:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:17:28 DEBUG ----- output ----- 19:17:28 DEBUG { 19:17:28 DEBUG "status": "success", 19:17:28 DEBUG "data": { 19:17:28 DEBUG "resultType": "vector", 19:17:28 DEBUG "result": [ 19:17:28 DEBUG { 19:17:28 DEBUG "metric": { 19:17:28 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:17:28 DEBUG }, 19:17:28 DEBUG "value": [ 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG [...] 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:17:28 DEBUG }, 19:17:28 DEBUG "value": [ 19:17:28 DEBUG 1713550648, 19:17:28 DEBUG "7.731737344079791" 19:17:28 DEBUG ] 19:17:28 DEBUG } 19:17:28 DEBUG ] 19:17:28 DEBUG } 19:17:28 DEBUG } 19:17:28 INFO [http_cmd]: http status code OK 19:17:28 DEBUG --- status code --- 19:17:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:17:28 DEBUG --- http response --- 19:17:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:17:28 DEBUG ----- output ----- 19:17:28 DEBUG { 19:17:28 DEBUG "status": "success", 19:17:28 DEBUG "data": { 19:17:28 DEBUG "resultType": "vector", 19:17:28 DEBUG "result": [ 19:17:28 DEBUG { 19:17:28 DEBUG "metric": { 19:17:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:17:28 DEBUG }, 19:17:28 DEBUG "value": [ 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG [...] 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:17:28 DEBUG }, 19:17:28 DEBUG "value": [ 19:17:28 DEBUG 1713550648, 19:17:28 DEBUG "0" 19:17:28 DEBUG ] 19:17:28 DEBUG } 19:17:28 DEBUG ] 19:17:28 DEBUG } 19:17:28 DEBUG } 19:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550648" 19:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550648" 19:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713550648" 19:17:28 INFO [http_cmd]: http status code OK 19:17:28 DEBUG --- status code --- 19:17:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:17:28 DEBUG --- http response --- 19:17:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:17:28 DEBUG ----- output ----- 19:17:28 DEBUG { 19:17:28 DEBUG "status": "success", 19:17:28 DEBUG "data": { 19:17:28 DEBUG "resultType": "vector", 19:17:28 DEBUG "result": [ 19:17:28 DEBUG { 19:17:28 DEBUG "metric": { 19:17:28 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:17:28 DEBUG }, 19:17:28 DEBUG "value": [ 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG [...] 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:17:28 DEBUG }, 19:17:28 DEBUG "value": [ 19:17:28 DEBUG 1713550648, 19:17:28 DEBUG "0" 19:17:28 DEBUG ] 19:17:28 DEBUG } 19:17:28 DEBUG ] 19:17:28 DEBUG } 19:17:28 DEBUG } 19:17:28 INFO [http_cmd]: http status code OK 19:17:28 DEBUG --- status code --- 19:17:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:17:28 INFO [http_cmd]: http status code OK 19:17:28 DEBUG --- http response --- 19:17:28 DEBUG --- status code --- 19:17:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:17:28 DEBUG --- http response --- 19:17:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:17:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:17:28 DEBUG ----- output ----- 19:17:28 DEBUG ----- output ----- 19:17:28 DEBUG { 19:17:28 DEBUG { 19:17:28 DEBUG "status": "success", 19:17:28 DEBUG "status": "success", 19:17:28 DEBUG "data": { 19:17:28 DEBUG "data": { 19:17:28 DEBUG "resultType": "vector", 19:17:28 DEBUG "resultType": "vector", 19:17:28 DEBUG "result": [ 19:17:28 DEBUG "result": [ 19:17:28 DEBUG { 19:17:28 DEBUG { 19:17:28 DEBUG "metric": { 19:17:28 DEBUG "metric": { 19:17:28 DEBUG "pod": "ds-idrepo-0" 19:17:28 DEBUG "pod": "ds-cts-0" 19:17:28 DEBUG }, 19:17:28 DEBUG }, 19:17:28 DEBUG "value": [ 19:17:28 DEBUG "value": [ 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG [...] 19:17:28 DEBUG [...] 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG "pod": "ds-cts-2" 19:17:28 DEBUG "pod": "ds-cts-2" 19:17:28 DEBUG }, 19:17:28 DEBUG }, 19:17:28 DEBUG "value": [ 19:17:28 DEBUG "value": [ 19:17:28 DEBUG 1713550648, 19:17:28 DEBUG 1713550648, 19:17:28 DEBUG "0" 19:17:28 DEBUG "0" 19:17:28 DEBUG ] 19:17:28 DEBUG ] 19:17:28 DEBUG } 19:17:28 DEBUG } 19:17:28 DEBUG ] 19:17:28 DEBUG ] 19:17:28 DEBUG } 19:17:28 DEBUG } 19:17:28 DEBUG } 19:17:28 DEBUG } 19:17:28 INFO 19:17:28 INFO [loop_until]: kubectl --namespace=xlou top pods 19:17:28 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:17:28 INFO [loop_until]: OK (rc = 0) 19:17:28 DEBUG --- stdout --- 19:17:28 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 13176m 7221Mi am-767f6b67b5-qcqz2 12887m 7194Mi am-767f6b67b5-vbltz 14043m 7291Mi ds-cts-0 9698m 10837Mi ds-cts-1 9444m 11094Mi ds-cts-2 10522m 10838Mi ds-idrepo-0 6065m 3539Mi ds-idrepo-1 5835m 3438Mi ds-idrepo-2 5992m 3492Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 2m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7606m 3119Mi 19:17:28 DEBUG --- stderr --- 19:17:28 DEBUG 19:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550648" 19:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550648" 19:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550648" 19:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550648" 19:17:28 INFO [http_cmd]: http status code OK 19:17:28 DEBUG --- status code --- 19:17:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:17:28 DEBUG --- http response --- 19:17:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:17:28 DEBUG ----- output ----- 19:17:28 DEBUG { 19:17:28 DEBUG "status": "success", 19:17:28 DEBUG "data": { 19:17:28 DEBUG "resultType": "vector", 19:17:28 DEBUG "result": [ 19:17:28 DEBUG { 19:17:28 DEBUG "metric": { 19:17:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:17:28 DEBUG }, 19:17:28 DEBUG "value": [ 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG [...] 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:17:28 DEBUG }, 19:17:28 DEBUG "value": [ 19:17:28 DEBUG 1713550648, 19:17:28 DEBUG "2338.133333333333" 19:17:28 DEBUG ] 19:17:28 DEBUG } 19:17:28 DEBUG ] 19:17:28 DEBUG } 19:17:28 DEBUG } 19:17:28 INFO [http_cmd]: http status code OK 19:17:28 DEBUG --- status code --- 19:17:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:17:28 DEBUG --- http response --- 19:17:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:17:28 DEBUG ----- output ----- 19:17:28 DEBUG { 19:17:28 DEBUG "status": "success", 19:17:28 DEBUG "data": { 19:17:28 DEBUG "resultType": "vector", 19:17:28 DEBUG "result": [ 19:17:28 DEBUG { 19:17:28 DEBUG "metric": { 19:17:28 DEBUG "instance": "10.142.0.107:9100" 19:17:28 DEBUG }, 19:17:28 DEBUG "value": [ 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG [...] 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG "instance": "10.142.0.43:9100" 19:17:28 DEBUG }, 19:17:28 DEBUG "value": [ 19:17:28 DEBUG 1713550648, 19:17:28 DEBUG "1854800079872" 19:17:28 DEBUG ] 19:17:28 DEBUG } 19:17:28 DEBUG ] 19:17:28 DEBUG } 19:17:28 DEBUG } 19:17:28 INFO [http_cmd]: http status code OK 19:17:28 DEBUG --- status code --- 19:17:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:17:28 DEBUG --- http response --- 19:17:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:17:28 DEBUG ----- output ----- 19:17:28 DEBUG { 19:17:28 DEBUG "status": "success", 19:17:28 DEBUG "data": { 19:17:28 DEBUG "resultType": "vector", 19:17:28 DEBUG "result": [ 19:17:28 DEBUG { 19:17:28 DEBUG "metric": { 19:17:28 DEBUG "instance": "10.142.0.107:9100" 19:17:28 DEBUG }, 19:17:28 DEBUG "value": [ 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG [...] 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG "instance": "10.142.0.43:9100" 19:17:28 DEBUG }, 19:17:28 DEBUG "value": [ 19:17:28 DEBUG 1713550648, 19:17:28 DEBUG "0.034333333333326735" 19:17:28 DEBUG ] 19:17:28 DEBUG } 19:17:28 DEBUG ] 19:17:28 DEBUG } 19:17:28 DEBUG } 19:17:28 INFO 19:17:28 INFO [loop_until]: kubectl --namespace=xlou top node 19:17:28 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550648" 19:17:28 INFO [http_cmd]: http status code OK 19:17:28 DEBUG --- status code --- 19:17:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:17:28 DEBUG --- http response --- 19:17:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:17:28 DEBUG ----- output ----- 19:17:28 DEBUG { 19:17:28 DEBUG "status": "success", 19:17:28 DEBUG "data": { 19:17:28 DEBUG "resultType": "vector", 19:17:28 DEBUG "result": [ 19:17:28 DEBUG { 19:17:28 DEBUG "metric": { 19:17:28 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:17:28 DEBUG }, 19:17:28 DEBUG "value": [ 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG [...] 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:17:28 DEBUG }, 19:17:28 DEBUG "value": [ 19:17:28 DEBUG 1713550648, 19:17:28 DEBUG "40137.51936789969" 19:17:28 DEBUG ] 19:17:28 DEBUG } 19:17:28 DEBUG ] 19:17:28 DEBUG } 19:17:28 DEBUG } 19:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713550648" 19:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550648" 19:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550648" 19:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550648" 19:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550648" 19:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550648" 19:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550648" 19:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550648" 19:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550648" 19:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713550648" 19:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550648" 19:17:28 INFO [http_cmd]: http status code OK 19:17:28 INFO [http_cmd]: http status code OK 19:17:28 INFO [http_cmd]: http status code OK 19:17:28 INFO [http_cmd]: http status code OK 19:17:28 INFO [http_cmd]: http status code OK 19:17:28 DEBUG --- status code --- 19:17:28 INFO [http_cmd]: http status code OK 19:17:28 DEBUG --- status code --- 19:17:28 INFO [http_cmd]: http status code OK 19:17:28 INFO [http_cmd]: http status code OK 19:17:28 DEBUG --- status code --- 19:17:28 DEBUG --- status code --- 19:17:28 DEBUG --- status code --- 19:17:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:17:28 DEBUG --- status code --- 19:17:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:17:28 DEBUG --- status code --- 19:17:28 DEBUG --- status code --- 19:17:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:17:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:17:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:17:28 DEBUG --- http response --- 19:17:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:17:28 INFO [http_cmd]: http status code OK 19:17:28 DEBUG --- http response --- 19:17:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:17:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:17:28 DEBUG --- http response --- 19:17:28 DEBUG --- http response --- 19:17:28 DEBUG --- http response --- 19:17:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:17:28 DEBUG --- http response --- 19:17:28 DEBUG --- status code --- 19:17:28 DEBUG --- http response --- 19:17:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:17:28 DEBUG --- http response --- 19:17:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:17:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:17:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:17:28 DEBUG ----- output ----- 19:17:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:17:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:17:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:17:28 DEBUG ----- output ----- 19:17:28 DEBUG ----- output ----- 19:17:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:17:28 INFO [http_cmd]: http status code OK 19:17:28 DEBUG ----- output ----- 19:17:28 DEBUG ----- output ----- 19:17:28 INFO [http_cmd]: http status code OK 19:17:28 DEBUG { 19:17:28 DEBUG --- http response --- 19:17:28 INFO [http_cmd]: http status code OK 19:17:28 DEBUG ----- output ----- 19:17:28 DEBUG ----- output ----- 19:17:28 DEBUG { 19:17:28 DEBUG { 19:17:28 DEBUG ----- output ----- 19:17:28 DEBUG --- status code --- 19:17:28 DEBUG { 19:17:28 DEBUG { 19:17:28 DEBUG --- status code --- 19:17:28 DEBUG "status": "success", 19:17:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:17:28 DEBUG --- status code --- 19:17:28 DEBUG { 19:17:28 DEBUG { 19:17:28 DEBUG "status": "success", 19:17:28 DEBUG "status": "success", 19:17:28 DEBUG { 19:17:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:17:28 DEBUG "status": "success", 19:17:28 DEBUG "status": "success", 19:17:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:17:28 DEBUG "data": { 19:17:28 DEBUG ----- output ----- 19:17:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:17:28 DEBUG "status": "success", 19:17:28 DEBUG "status": "success", 19:17:28 DEBUG "data": { 19:17:28 DEBUG "data": { 19:17:28 DEBUG "status": "success", 19:17:28 DEBUG --- http response --- 19:17:28 DEBUG "data": { 19:17:28 DEBUG "data": { 19:17:28 DEBUG --- http response --- 19:17:28 DEBUG "resultType": "vector", 19:17:28 DEBUG { 19:17:28 DEBUG --- http response --- 19:17:28 DEBUG "data": { 19:17:28 DEBUG "data": { 19:17:28 DEBUG "resultType": "vector", 19:17:28 DEBUG "resultType": "vector", 19:17:28 DEBUG "data": { 19:17:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:17:28 DEBUG "resultType": "vector", 19:17:28 DEBUG "resultType": "vector", 19:17:28 DEBUG "result": [ 19:17:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:17:28 DEBUG "status": "success", 19:17:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:17:28 DEBUG "resultType": "vector", 19:17:28 DEBUG "resultType": "vector", 19:17:28 DEBUG "result": [ 19:17:28 DEBUG "result": [ 19:17:28 DEBUG "resultType": "vector", 19:17:28 DEBUG ----- output ----- 19:17:28 DEBUG "result": [ 19:17:28 DEBUG "result": [ 19:17:28 DEBUG { 19:17:28 DEBUG ----- output ----- 19:17:28 DEBUG "data": { 19:17:28 DEBUG ----- output ----- 19:17:28 DEBUG "result": [ 19:17:28 DEBUG "result": [ 19:17:28 DEBUG { 19:17:28 DEBUG { 19:17:28 DEBUG "result": [ 19:17:28 DEBUG { 19:17:28 DEBUG { 19:17:28 DEBUG { 19:17:28 DEBUG "metric": { 19:17:28 DEBUG { 19:17:28 DEBUG "resultType": "vector", 19:17:28 DEBUG { 19:17:28 DEBUG { 19:17:28 DEBUG { 19:17:28 DEBUG "metric": { 19:17:28 DEBUG "metric": { 19:17:28 DEBUG { 19:17:28 DEBUG "status": "success", 19:17:28 DEBUG "metric": { 19:17:28 DEBUG "metric": { 19:17:28 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:17:28 DEBUG "status": "success", 19:17:28 DEBUG "result": [ 19:17:28 DEBUG "status": "success", 19:17:28 DEBUG "metric": { 19:17:28 DEBUG "metric": { 19:17:28 DEBUG "instance": "10.142.0.107:9100" 19:17:28 DEBUG "instance": "10.142.0.107:9100" 19:17:28 DEBUG "metric": { 19:17:28 DEBUG "data": { 19:17:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:17:28 DEBUG "pod": "ds-cts-0" 19:17:28 DEBUG }, 19:17:28 DEBUG "data": { 19:17:28 DEBUG { 19:17:28 DEBUG "data": { 19:17:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:17:28 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:17:28 DEBUG }, 19:17:28 DEBUG }, 19:17:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:17:28 DEBUG "resultType": "vector", 19:17:28 DEBUG }, 19:17:28 DEBUG }, 19:17:28 DEBUG "value": [ 19:17:28 DEBUG "resultType": "vector", 19:17:28 DEBUG "metric": { 19:17:28 DEBUG "resultType": "vector", 19:17:28 DEBUG }, 19:17:28 DEBUG }, 19:17:28 DEBUG "value": [ 19:17:28 DEBUG "value": [ 19:17:28 DEBUG }, 19:17:28 DEBUG "result": [ 19:17:28 DEBUG "value": [ 19:17:28 DEBUG "value": [ 19:17:28 DEBUG 19:17:28 DEBUG "result": [ 19:17:28 DEBUG "pod": "ds-idrepo-0" 19:17:28 DEBUG "result": [ 19:17:28 DEBUG "value": [ 19:17:28 DEBUG "value": [ 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG "value": [ 19:17:28 DEBUG { 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG { 19:17:28 DEBUG }, 19:17:28 DEBUG { 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG "metric": { 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG "metric": { 19:17:28 DEBUG "value": [ 19:17:28 DEBUG "metric": { 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG "pod": "ds-cts-0" 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG [...] 19:17:28 DEBUG "instance": "10.142.0.107:9100" 19:17:28 DEBUG 19:17:28 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG [...] 19:17:28 DEBUG [...] 19:17:28 DEBUG 19:17:28 DEBUG }, 19:17:28 DEBUG [...] 19:17:28 DEBUG [...] 19:17:28 DEBUG 19:17:28 DEBUG }, 19:17:28 DEBUG 19:17:28 DEBUG }, 19:17:28 DEBUG [...] 19:17:28 DEBUG [...] 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG [...] 19:17:28 DEBUG "value": [ 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG "value": [ 19:17:28 DEBUG 19:17:28 DEBUG "value": [ 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG [...] 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG "instance": "10.142.0.43:9100" 19:17:28 DEBUG "instance": "10.142.0.43:9100" 19:17:28 DEBUG }, 19:17:28 DEBUG "value": [ 19:17:28 DEBUG 19:17:28 DEBUG "pod": "ds-cts-2" 19:17:28 DEBUG }, 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:17:28 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:17:28 DEBUG }, 19:17:28 DEBUG 19:17:28 DEBUG 1713550648, 19:17:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:17:28 DEBUG [...] 19:17:28 DEBUG }, 19:17:28 DEBUG "value": [ 19:17:28 DEBUG [...] 19:17:28 DEBUG 19:17:28 DEBUG [...] 19:17:28 DEBUG }, 19:17:28 DEBUG }, 19:17:28 DEBUG "value": [ 19:17:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:17:28 DEBUG "0.04663333333332104" 19:17:28 DEBUG }, 19:17:28 DEBUG 19:17:28 DEBUG "value": [ 19:17:28 DEBUG 1713550648, 19:17:28 DEBUG 19:17:28 DEBUG "pod": "ds-idrepo-2" 19:17:28 DEBUG 19:17:28 DEBUG "value": [ 19:17:28 DEBUG "value": [ 19:17:28 DEBUG 1713550648, 19:17:28 DEBUG }, 19:17:28 DEBUG ] 19:17:28 DEBUG "value": [ 19:17:28 DEBUG 19:17:28 DEBUG 19:17:28 DEBUG 1713550648, 19:17:28 DEBUG 19:17:28 DEBUG }, 19:17:28 DEBUG 19:17:28 DEBUG 1713550648, 19:17:28 DEBUG 1713550648, 19:17:28 DEBUG "58916661.33333333" 19:17:28 DEBUG "value": [ 19:17:28 DEBUG } 19:17:28 DEBUG 1713550648, 19:17:28 DEBUG "0" 19:17:28 DEBUG "pod": "ds-cts-2" 19:17:28 DEBUG "0.0001133" 19:17:28 DEBUG 19:17:28 DEBUG "value": [ 19:17:28 DEBUG 19:17:28 DEBUG "2338.6" 19:17:28 DEBUG "5.093258802748054" 19:17:28 DEBUG ] 19:17:28 DEBUG 1713550648, 19:17:28 DEBUG ] 19:17:28 DEBUG "0.002" 19:17:28 DEBUG ] 19:17:28 DEBUG }, 19:17:28 DEBUG ] 19:17:28 DEBUG "instance": "10.142.0.43:9100" 19:17:28 DEBUG 1713550648, 19:17:28 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:17:28 DEBUG ] 19:17:28 DEBUG ] 19:17:28 DEBUG } 19:17:28 DEBUG "404830.5" 19:17:28 DEBUG } 19:17:28 DEBUG ] 19:17:28 DEBUG } 19:17:28 DEBUG "value": [ 19:17:28 DEBUG } 19:17:28 DEBUG }, 19:17:28 DEBUG "0" 19:17:28 DEBUG }, 19:17:28 DEBUG } 19:17:28 DEBUG } 19:17:28 DEBUG ] 19:17:28 DEBUG ] 19:17:28 DEBUG } 19:17:28 DEBUG } 19:17:28 DEBUG ] 19:17:28 DEBUG 1713550648, 19:17:28 DEBUG ] 19:17:28 DEBUG "value": [ 19:17:28 DEBUG ] 19:17:28 DEBUG "value": [ 19:17:28 DEBUG ] 19:17:28 DEBUG ] 19:17:28 DEBUG } 19:17:28 DEBUG } 19:17:28 DEBUG ] 19:17:28 DEBUG ] 19:17:28 DEBUG } 19:17:28 DEBUG "0" 19:17:28 DEBUG } 19:17:28 DEBUG 1713550648, 19:17:28 DEBUG } 19:17:28 DEBUG 1713550648, 19:17:28 DEBUG } 19:17:28 DEBUG } 19:17:28 DEBUG } 19:17:28 DEBUG } 19:17:28 DEBUG } 19:17:28 DEBUG } 19:17:28 DEBUG ] 19:17:28 DEBUG } 19:17:28 DEBUG "58919076.36666667" 19:17:28 DEBUG ] 19:17:28 DEBUG "3368722432" 19:17:28 DEBUG ] 19:17:28 DEBUG } 19:17:28 DEBUG } 19:17:28 DEBUG } 19:17:28 DEBUG } 19:17:28 DEBUG ] 19:17:28 DEBUG ] 19:17:28 DEBUG } 19:17:28 DEBUG } 19:17:28 DEBUG } 19:17:28 DEBUG } 19:17:28 DEBUG } 19:17:28 DEBUG ] 19:17:28 DEBUG ] 19:17:28 DEBUG } 19:17:28 DEBUG } 19:17:28 DEBUG } 19:17:28 DEBUG } 19:17:28 DEBUG } 19:17:28 DEBUG } 19:17:28 INFO [loop_until]: OK (rc = 0) 19:17:28 DEBUG --- stdout --- 19:17:28 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 70m 0% 6223Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 6126m 20% 7101Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13212m 44% 9930Mi 8% gke-xlou-cdm-default-pool-1b0769c8-qkns 84m 0% 3344Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 11683m 39% 13878Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 165m 0% 5219Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 13354m 44% 10081Mi 9% gke-xlou-cdm-ds-0f601751-cf7b 12298m 41% 3015Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 15126m 50% 9491Mi 8% gke-xlou-cdm-ds-0f601751-j488 6370m 21% 6730Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 9024m 30% 15940Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 6688m 22% 7341Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 11617m 38% 13353Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 7635m 48% 6936Mi 11% 19:17:28 DEBUG --- stderr --- 19:17:28 DEBUG 19:18:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713550708" 19:18:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550708" 19:18:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550708" 19:18:28 ERROR [http_cmd]: ERROR 19:18:28 INFO [http_cmd]: http status code OK 19:18:28 ERROR --- status code --- 19:18:28 DEBUG --- status code --- 19:18:28 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:18:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:18:28 ERROR --- http response --- 19:18:28 DEBUG --- http response --- 19:18:28 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:18:28 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:18:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:18:28 DEBUG ----- output ----- 19:18:28 DEBUG { 19:18:28 DEBUG "status": "success", 19:18:28 DEBUG "data": { 19:18:28 DEBUG "resultType": "vector", 19:18:28 DEBUG "result": [ 19:18:28 DEBUG { 19:18:28 DEBUG "metric": { 19:18:28 DEBUG "pod": "ds-idrepo-0" 19:18:28 DEBUG }, 19:18:28 DEBUG "value": [ 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG [...] 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG "pod": "ds-idrepo-2" 19:18:28 DEBUG }, 19:18:28 DEBUG "value": [ 19:18:28 DEBUG 1713550708, 19:18:28 DEBUG "0" 19:18:28 DEBUG ] 19:18:28 DEBUG } 19:18:28 DEBUG ] 19:18:28 DEBUG } 19:18:28 DEBUG } 19:18:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550708" 19:18:28 INFO [http_cmd]: http status code OK 19:18:28 DEBUG --- status code --- 19:18:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:18:28 DEBUG --- http response --- 19:18:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:18:28 DEBUG ----- output ----- 19:18:28 DEBUG { 19:18:28 DEBUG "status": "success", 19:18:28 DEBUG "data": { 19:18:28 DEBUG "resultType": "vector", 19:18:28 DEBUG "result": [ 19:18:28 DEBUG { 19:18:28 DEBUG "metric": { 19:18:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:18:28 DEBUG }, 19:18:28 DEBUG "value": [ 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG [...] 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:18:28 DEBUG }, 19:18:28 DEBUG "value": [ 19:18:28 DEBUG 1713550708, 19:18:28 DEBUG "2335.0333333333333" 19:18:28 DEBUG ] 19:18:28 DEBUG } 19:18:28 DEBUG ] 19:18:28 DEBUG } 19:18:28 DEBUG } 19:18:28 INFO [http_cmd]: http status code OK 19:18:28 DEBUG --- status code --- 19:18:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:18:28 DEBUG --- http response --- 19:18:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:18:28 DEBUG ----- output ----- 19:18:28 DEBUG { 19:18:28 DEBUG "status": "success", 19:18:28 DEBUG "data": { 19:18:28 DEBUG "resultType": "vector", 19:18:28 DEBUG "result": [ 19:18:28 DEBUG { 19:18:28 DEBUG "metric": { 19:18:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:18:28 DEBUG }, 19:18:28 DEBUG "value": [ 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG [...] 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:18:28 DEBUG }, 19:18:28 DEBUG "value": [ 19:18:28 DEBUG 1713550708, 19:18:28 DEBUG "2347" 19:18:28 DEBUG ] 19:18:28 DEBUG } 19:18:28 DEBUG ] 19:18:28 DEBUG } 19:18:28 DEBUG } 19:18:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550708" 19:18:28 INFO [http_cmd]: http status code OK 19:18:28 DEBUG --- status code --- 19:18:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:18:28 DEBUG --- http response --- 19:18:28 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:18:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550708" 19:18:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713550708" 19:18:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550708" 19:18:28 INFO [http_cmd]: http status code OK 19:18:28 DEBUG --- status code --- 19:18:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:18:28 DEBUG --- http response --- 19:18:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:18:28 DEBUG ----- output ----- 19:18:28 DEBUG { 19:18:28 DEBUG "status": "success", 19:18:28 DEBUG "data": { 19:18:28 DEBUG "resultType": "vector", 19:18:28 DEBUG "result": [ 19:18:28 DEBUG { 19:18:28 DEBUG "metric": { 19:18:28 DEBUG "pod": "ds-idrepo-0" 19:18:28 DEBUG }, 19:18:28 DEBUG "value": [ 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG [...] 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG "pod": "ds-idrepo-2" 19:18:28 DEBUG }, 19:18:28 DEBUG "value": [ 19:18:28 DEBUG 1713550708, 19:18:28 DEBUG "0" 19:18:28 DEBUG ] 19:18:28 DEBUG } 19:18:28 DEBUG ] 19:18:28 DEBUG } 19:18:28 DEBUG } 19:18:28 INFO [http_cmd]: http status code OK 19:18:28 DEBUG --- status code --- 19:18:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:18:28 DEBUG --- http response --- 19:18:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:18:28 DEBUG ----- output ----- 19:18:28 DEBUG { 19:18:28 DEBUG "status": "success", 19:18:28 DEBUG "data": { 19:18:28 DEBUG "resultType": "vector", 19:18:28 DEBUG "result": [ 19:18:28 DEBUG { 19:18:28 DEBUG "metric": { 19:18:28 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:18:28 DEBUG }, 19:18:28 DEBUG "value": [ 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG [...] 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:18:28 DEBUG }, 19:18:28 DEBUG "value": [ 19:18:28 DEBUG 1713550708, 19:18:28 DEBUG "7.710908510340239" 19:18:28 DEBUG ] 19:18:28 DEBUG } 19:18:28 DEBUG ] 19:18:28 DEBUG } 19:18:28 DEBUG } 19:18:28 INFO [http_cmd]: http status code OK 19:18:28 DEBUG --- status code --- 19:18:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:18:28 DEBUG --- http response --- 19:18:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550708" 19:18:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:18:28 DEBUG ----- output ----- 19:18:28 DEBUG { 19:18:28 DEBUG "status": "success", 19:18:28 DEBUG "data": { 19:18:28 DEBUG "resultType": "vector", 19:18:28 DEBUG "result": [ 19:18:28 DEBUG { 19:18:28 DEBUG "metric": { 19:18:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:18:28 DEBUG }, 19:18:28 DEBUG "value": [ 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG [...] 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:18:28 DEBUG }, 19:18:28 DEBUG "value": [ 19:18:28 DEBUG 1713550708, 19:18:28 DEBUG "0" 19:18:28 DEBUG ] 19:18:28 DEBUG } 19:18:28 DEBUG ] 19:18:28 DEBUG } 19:18:28 DEBUG } 19:18:28 INFO [http_cmd]: http status code OK 19:18:28 DEBUG --- status code --- 19:18:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:18:28 DEBUG --- http response --- 19:18:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:18:28 DEBUG ----- output ----- 19:18:28 DEBUG { 19:18:28 DEBUG "status": "success", 19:18:28 DEBUG "data": { 19:18:28 DEBUG "resultType": "vector", 19:18:28 DEBUG "result": [ 19:18:28 DEBUG { 19:18:28 DEBUG "metric": { 19:18:28 DEBUG "pod": "ds-cts-0" 19:18:28 DEBUG }, 19:18:28 DEBUG "value": [ 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG [...] 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG "pod": "ds-cts-2" 19:18:28 DEBUG }, 19:18:28 DEBUG "value": [ 19:18:28 DEBUG 1713550708, 19:18:28 DEBUG "0" 19:18:28 DEBUG ] 19:18:28 DEBUG } 19:18:28 DEBUG ] 19:18:28 DEBUG } 19:18:28 DEBUG } 19:18:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550708" 19:18:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713550708" 19:18:28 INFO [http_cmd]: http status code OK 19:18:28 DEBUG --- status code --- 19:18:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:18:28 DEBUG --- http response --- 19:18:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:18:28 DEBUG ----- output ----- 19:18:28 DEBUG { 19:18:28 DEBUG "status": "success", 19:18:28 DEBUG "data": { 19:18:28 DEBUG "resultType": "vector", 19:18:28 DEBUG "result": [ 19:18:28 DEBUG { 19:18:28 DEBUG "metric": { 19:18:28 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:18:28 DEBUG }, 19:18:28 DEBUG "value": [ 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG [...] 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:18:28 DEBUG }, 19:18:28 DEBUG "value": [ 19:18:28 DEBUG 1713550708, 19:18:28 DEBUG "0" 19:18:28 DEBUG ] 19:18:28 DEBUG } 19:18:28 DEBUG ] 19:18:28 DEBUG } 19:18:28 DEBUG } 19:18:28 INFO [http_cmd]: http status code OK 19:18:28 DEBUG --- status code --- 19:18:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:18:28 DEBUG --- http response --- 19:18:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:18:28 DEBUG ----- output ----- 19:18:28 DEBUG { 19:18:28 DEBUG "status": "success", 19:18:28 DEBUG "data": { 19:18:28 DEBUG "resultType": "vector", 19:18:28 DEBUG "result": [ 19:18:28 DEBUG { 19:18:28 DEBUG "metric": { 19:18:28 DEBUG "pod": "ds-idrepo-0" 19:18:28 DEBUG }, 19:18:28 DEBUG "value": [ 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG [...] 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG "pod": "ds-cts-2" 19:18:28 DEBUG }, 19:18:28 DEBUG "value": [ 19:18:28 DEBUG 1713550708, 19:18:28 DEBUG "0" 19:18:28 DEBUG ] 19:18:28 DEBUG } 19:18:28 DEBUG ] 19:18:28 DEBUG } 19:18:28 DEBUG } 19:18:28 INFO 19:18:28 INFO [loop_until]: kubectl --namespace=xlou top pods 19:18:28 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:18:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550708" 19:18:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550708" 19:18:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550708" 19:18:28 INFO [http_cmd]: http status code OK 19:18:28 INFO [http_cmd]: http status code OK 19:18:28 DEBUG --- status code --- 19:18:28 DEBUG --- status code --- 19:18:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550708" 19:18:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:18:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:18:28 DEBUG --- http response --- 19:18:28 DEBUG --- http response --- 19:18:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:18:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:18:28 DEBUG ----- output ----- 19:18:28 DEBUG ----- output ----- 19:18:28 DEBUG { 19:18:28 DEBUG { 19:18:28 DEBUG "status": "success", 19:18:28 DEBUG "status": "success", 19:18:28 DEBUG "data": { 19:18:28 DEBUG "data": { 19:18:28 DEBUG "resultType": "vector", 19:18:28 DEBUG "resultType": "vector", 19:18:28 DEBUG "result": [ 19:18:28 DEBUG "result": [ 19:18:28 DEBUG { 19:18:28 DEBUG { 19:18:28 DEBUG "metric": { 19:18:28 DEBUG "metric": { 19:18:28 DEBUG "instance": "10.142.0.107:9100" 19:18:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:18:28 DEBUG }, 19:18:28 DEBUG }, 19:18:28 DEBUG "value": [ 19:18:28 DEBUG "value": [ 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG [...] 19:18:28 DEBUG [...] 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 INFO [http_cmd]: http status code OK 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG --- status code --- 19:18:28 DEBUG 19:18:28 DEBUG "instance": "10.142.0.43:9100" 19:18:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:18:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:18:28 DEBUG }, 19:18:28 DEBUG --- http response --- 19:18:28 DEBUG }, 19:18:28 DEBUG "value": [ 19:18:28 DEBUG "value": [ 19:18:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:18:28 DEBUG 1713550708, 19:18:28 DEBUG 1713550708, 19:18:28 DEBUG ----- output ----- 19:18:28 DEBUG "1857661148160" 19:18:28 DEBUG "2335" 19:18:28 DEBUG { 19:18:28 DEBUG ] 19:18:28 DEBUG ] 19:18:28 DEBUG "status": "success", 19:18:28 DEBUG } 19:18:28 DEBUG } 19:18:28 DEBUG "data": { 19:18:28 DEBUG ] 19:18:28 DEBUG ] 19:18:28 DEBUG "resultType": "vector", 19:18:28 DEBUG } 19:18:28 DEBUG } 19:18:28 DEBUG "result": [ 19:18:28 DEBUG } 19:18:28 DEBUG } 19:18:28 DEBUG { 19:18:28 DEBUG "metric": { 19:18:28 DEBUG "instance": "10.142.0.107:9100" 19:18:28 DEBUG }, 19:18:28 DEBUG "value": [ 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG [...] 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG "instance": "10.142.0.43:9100" 19:18:28 DEBUG }, 19:18:28 DEBUG "value": [ 19:18:28 DEBUG 1713550708, 19:18:28 DEBUG "0.07066666666666303" 19:18:28 DEBUG ] 19:18:28 DEBUG } 19:18:28 DEBUG ] 19:18:28 DEBUG } 19:18:28 DEBUG } 19:18:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550708" 19:18:28 INFO [http_cmd]: http status code OK 19:18:28 DEBUG --- status code --- 19:18:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:18:28 DEBUG --- http response --- 19:18:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:18:28 DEBUG ----- output ----- 19:18:28 DEBUG { 19:18:28 DEBUG "status": "success", 19:18:28 DEBUG "data": { 19:18:28 DEBUG "resultType": "vector", 19:18:28 DEBUG "result": [ 19:18:28 DEBUG { 19:18:28 DEBUG "metric": { 19:18:28 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:18:28 DEBUG }, 19:18:28 DEBUG "value": [ 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG [...] 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:18:28 DEBUG }, 19:18:28 DEBUG "value": [ 19:18:28 DEBUG 1713550708, 19:18:28 DEBUG "29144.403143544507" 19:18:28 DEBUG ] 19:18:28 DEBUG } 19:18:28 DEBUG ] 19:18:28 DEBUG } 19:18:28 DEBUG } 19:18:28 INFO [loop_until]: OK (rc = 0) 19:18:28 DEBUG --- stdout --- 19:18:28 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 14850m 7228Mi am-767f6b67b5-qcqz2 14595m 7204Mi am-767f6b67b5-vbltz 15598m 7306Mi ds-cts-0 14973m 10817Mi ds-cts-1 13528m 11078Mi ds-cts-2 15309m 10819Mi ds-idrepo-0 6550m 3560Mi ds-idrepo-1 6255m 3460Mi ds-idrepo-2 6343m 3492Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 4m 1217Mi lodemon-f8459c597-6tj6f 2m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7721m 3123Mi 19:18:28 DEBUG --- stderr --- 19:18:28 DEBUG 19:18:28 INFO [http_cmd]: http status code OK 19:18:28 DEBUG --- status code --- 19:18:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:18:28 DEBUG --- http response --- 19:18:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:18:28 DEBUG ----- output ----- 19:18:28 DEBUG { 19:18:28 DEBUG "status": "success", 19:18:28 DEBUG "data": { 19:18:28 DEBUG "resultType": "vector", 19:18:28 DEBUG "result": [ 19:18:28 DEBUG { 19:18:28 DEBUG "metric": { 19:18:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:18:28 DEBUG }, 19:18:28 DEBUG "value": [ 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG [...] 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:18:28 DEBUG }, 19:18:28 DEBUG "value": [ 19:18:28 DEBUG 1713550708, 19:18:28 DEBUG "421317" 19:18:28 DEBUG ] 19:18:28 DEBUG } 19:18:28 DEBUG ] 19:18:28 DEBUG } 19:18:28 DEBUG } 19:18:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713550708" 19:18:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550708" 19:18:28 INFO [http_cmd]: http status code OK 19:18:28 DEBUG --- status code --- 19:18:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:18:28 DEBUG --- http response --- 19:18:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:18:28 DEBUG ----- output ----- 19:18:28 DEBUG { 19:18:28 DEBUG "status": "success", 19:18:28 DEBUG "data": { 19:18:28 DEBUG "resultType": "vector", 19:18:28 DEBUG "result": [ 19:18:28 DEBUG { 19:18:28 DEBUG "metric": { 19:18:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:18:28 DEBUG }, 19:18:28 DEBUG "value": [ 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG [...] 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:18:28 DEBUG }, 19:18:28 DEBUG "value": [ 19:18:28 DEBUG 1713550708, 19:18:28 DEBUG "0.003" 19:18:28 DEBUG ] 19:18:28 DEBUG } 19:18:28 DEBUG ] 19:18:28 DEBUG } 19:18:28 DEBUG } 19:18:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550708" 19:18:28 INFO [http_cmd]: http status code OK 19:18:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550708" 19:18:28 DEBUG --- status code --- 19:18:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:18:28 DEBUG --- http response --- 19:18:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550708" 19:18:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550708" 19:18:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:18:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550708" 19:18:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550708" 19:18:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550708" 19:18:28 DEBUG ----- output ----- 19:18:28 DEBUG { 19:18:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550708" 19:18:28 INFO 19:18:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713550708" 19:18:28 DEBUG "status": "success", 19:18:28 INFO [loop_until]: kubectl --namespace=xlou top node 19:18:28 INFO [http_cmd]: http status code OK 19:18:28 INFO [http_cmd]: http status code OK 19:18:28 DEBUG "data": { 19:18:28 INFO [http_cmd]: http status code OK 19:18:28 INFO [http_cmd]: http status code OK 19:18:28 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:18:28 INFO [http_cmd]: http status code OK 19:18:28 DEBUG --- status code --- 19:18:28 DEBUG --- status code --- 19:18:28 DEBUG "resultType": "vector", 19:18:28 DEBUG --- status code --- 19:18:28 DEBUG --- status code --- 19:18:28 DEBUG --- status code --- 19:18:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:18:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:18:28 DEBUG "result": [ 19:18:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:18:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:18:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:18:28 INFO [http_cmd]: http status code OK 19:18:28 INFO [http_cmd]: http status code OK 19:18:28 INFO [http_cmd]: http status code OK 19:18:28 DEBUG --- http response --- 19:18:28 DEBUG --- http response --- 19:18:28 DEBUG { 19:18:28 DEBUG --- http response --- 19:18:28 DEBUG --- http response --- 19:18:28 DEBUG --- http response --- 19:18:28 DEBUG --- status code --- 19:18:28 DEBUG --- status code --- 19:18:28 DEBUG --- status code --- 19:18:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:18:28 DEBUG "metric": { 19:18:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:18:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:18:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:18:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:18:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:18:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:18:28 INFO [http_cmd]: http status code OK 19:18:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:18:28 DEBUG ----- output ----- 19:18:28 DEBUG "instance": "10.142.0.107:9100" 19:18:28 DEBUG ----- output ----- 19:18:28 DEBUG ----- output ----- 19:18:28 DEBUG ----- output ----- 19:18:28 DEBUG --- http response --- 19:18:28 DEBUG ----- output ----- 19:18:28 DEBUG --- http response --- 19:18:28 DEBUG --- status code --- 19:18:28 DEBUG --- http response --- 19:18:28 DEBUG { 19:18:28 DEBUG }, 19:18:28 DEBUG { 19:18:28 DEBUG { 19:18:28 DEBUG { 19:18:28 DEBUG { 19:18:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:18:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:18:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:18:28 DEBUG "status": "success", 19:18:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:18:28 DEBUG "value": [ 19:18:28 DEBUG "status": "success", 19:18:28 DEBUG "status": "success", 19:18:28 DEBUG "status": "success", 19:18:28 DEBUG "status": "success", 19:18:28 DEBUG ----- output ----- 19:18:28 DEBUG ----- output ----- 19:18:28 DEBUG --- http response --- 19:18:28 DEBUG "data": { 19:18:28 DEBUG ----- output ----- 19:18:28 DEBUG 19:18:28 DEBUG "data": { 19:18:28 DEBUG "data": { 19:18:28 DEBUG "data": { 19:18:28 DEBUG "data": { 19:18:28 DEBUG { 19:18:28 DEBUG { 19:18:28 DEBUG "resultType": "vector", 19:18:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:18:28 DEBUG { 19:18:28 DEBUG 19:18:28 DEBUG "resultType": "vector", 19:18:28 DEBUG "resultType": "vector", 19:18:28 DEBUG "resultType": "vector", 19:18:28 DEBUG "resultType": "vector", 19:18:28 DEBUG "status": "success", 19:18:28 DEBUG "status": "success", 19:18:28 DEBUG "result": [ 19:18:28 DEBUG ----- output ----- 19:18:28 DEBUG "status": "success", 19:18:28 DEBUG 19:18:28 DEBUG "result": [ 19:18:28 DEBUG "result": [ 19:18:28 DEBUG "result": [ 19:18:28 DEBUG "result": [ 19:18:28 DEBUG "data": { 19:18:28 DEBUG "data": { 19:18:28 DEBUG { 19:18:28 DEBUG { 19:18:28 DEBUG "data": { 19:18:28 DEBUG [...] 19:18:28 DEBUG { 19:18:28 DEBUG { 19:18:28 DEBUG { 19:18:28 DEBUG { 19:18:28 DEBUG "resultType": "vector", 19:18:28 DEBUG "resultType": "vector", 19:18:28 DEBUG "metric": { 19:18:28 DEBUG "status": "success", 19:18:28 DEBUG "resultType": "vector", 19:18:28 DEBUG 19:18:28 DEBUG "metric": { 19:18:28 DEBUG "metric": { 19:18:28 DEBUG "metric": { 19:18:28 DEBUG "metric": { 19:18:28 DEBUG "result": [ 19:18:28 DEBUG "result": [ 19:18:28 DEBUG "instance": "10.142.0.107:9100" 19:18:28 DEBUG "data": { 19:18:28 DEBUG "result": [ 19:18:28 DEBUG 19:18:28 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:18:28 DEBUG "pod": "ds-cts-0" 19:18:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:18:28 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:18:28 DEBUG { 19:18:28 DEBUG { 19:18:28 DEBUG }, 19:18:28 DEBUG "resultType": "vector", 19:18:28 DEBUG { 19:18:28 DEBUG 19:18:28 DEBUG }, 19:18:28 DEBUG }, 19:18:28 DEBUG }, 19:18:28 DEBUG }, 19:18:28 DEBUG "metric": { 19:18:28 DEBUG "metric": { 19:18:28 DEBUG "value": [ 19:18:28 DEBUG "result": [ 19:18:28 DEBUG "metric": { 19:18:28 DEBUG "instance": "10.142.0.43:9100" 19:18:28 DEBUG "value": [ 19:18:28 DEBUG "value": [ 19:18:28 DEBUG "value": [ 19:18:28 DEBUG "value": [ 19:18:28 DEBUG "pod": "ds-idrepo-0" 19:18:28 DEBUG "pod": "ds-cts-0" 19:18:28 DEBUG 19:18:28 DEBUG { 19:18:28 DEBUG "instance": "10.142.0.107:9100" 19:18:28 DEBUG }, 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG }, 19:18:28 DEBUG }, 19:18:28 DEBUG 19:18:28 DEBUG "metric": { 19:18:28 DEBUG }, 19:18:28 DEBUG "value": [ 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG "value": [ 19:18:28 DEBUG "value": [ 19:18:28 DEBUG 19:18:28 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:18:28 DEBUG "value": [ 19:18:28 DEBUG 1713550708, 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG [...] 19:18:28 DEBUG }, 19:18:28 DEBUG 19:18:28 DEBUG "70773507" 19:18:28 DEBUG [...] 19:18:28 DEBUG [...] 19:18:28 DEBUG [...] 19:18:28 DEBUG [...] 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG "value": [ 19:18:28 DEBUG 19:18:28 DEBUG ] 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG } 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG [...] 19:18:28 DEBUG [...] 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG [...] 19:18:28 DEBUG ] 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG "instance": "10.142.0.43:9100" 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG } 19:18:28 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:18:28 DEBUG "pod": "ds-cts-2" 19:18:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:18:28 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG }, 19:18:28 DEBUG [...] 19:18:28 DEBUG 19:18:28 DEBUG } 19:18:28 DEBUG }, 19:18:28 DEBUG "value": [ 19:18:28 DEBUG }, 19:18:28 DEBUG }, 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG "value": [ 19:18:28 DEBUG 19:18:28 DEBUG 19:18:28 DEBUG }, 19:18:28 DEBUG 1713550708, 19:18:28 DEBUG "value": [ 19:18:28 DEBUG "value": [ 19:18:28 DEBUG 1713550708, 19:18:28 DEBUG "pod": "ds-cts-2" 19:18:28 DEBUG 1713550708, 19:18:28 DEBUG 19:18:28 DEBUG "instance": "10.142.0.43:9100" 19:18:28 DEBUG "value": [ 19:18:28 DEBUG "0" 19:18:28 DEBUG 1713550708, 19:18:28 DEBUG "5.159149161598144" 19:18:28 DEBUG "pod": "ds-idrepo-2" 19:18:28 DEBUG }, 19:18:28 DEBUG "0.15849999999997239" 19:18:28 DEBUG 19:18:28 DEBUG }, 19:18:28 DEBUG 1713550708, 19:18:28 DEBUG "2335.1" 19:18:28 DEBUG "0" 19:18:28 DEBUG ] 19:18:28 DEBUG }, 19:18:28 DEBUG "value": [ 19:18:28 DEBUG ] 19:18:28 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:18:28 DEBUG "value": [ 19:18:28 DEBUG ] 19:18:28 DEBUG ] 19:18:28 DEBUG ] 19:18:28 DEBUG } 19:18:28 DEBUG "value": [ 19:18:28 DEBUG 1713550708, 19:18:28 DEBUG } 19:18:28 DEBUG }, 19:18:28 DEBUG 1713550708, 19:18:28 DEBUG } 19:18:28 DEBUG } 19:18:28 DEBUG } 19:18:28 DEBUG ] 19:18:28 DEBUG 1713550708, 19:18:28 DEBUG "0" 19:18:28 DEBUG ] 19:18:28 DEBUG "value": [ 19:18:28 DEBUG "70777772.43333334" 19:18:28 DEBUG ] 19:18:28 DEBUG ] 19:18:28 DEBUG ] 19:18:28 DEBUG } 19:18:28 DEBUG "0" 19:18:28 DEBUG ] 19:18:28 DEBUG } 19:18:28 DEBUG 1713550708, 19:18:28 DEBUG ] 19:18:28 DEBUG } 19:18:28 DEBUG } 19:18:28 DEBUG } 19:18:28 DEBUG } 19:18:28 DEBUG ] 19:18:28 DEBUG } 19:18:28 DEBUG } 19:18:28 DEBUG "3372523520" 19:18:28 DEBUG } 19:18:28 DEBUG } 19:18:28 DEBUG } 19:18:28 DEBUG } 19:18:28 DEBUG } 19:18:28 DEBUG ] 19:18:28 DEBUG ] 19:18:28 DEBUG ] 19:18:28 DEBUG ] 19:18:28 DEBUG } 19:18:28 DEBUG } 19:18:28 DEBUG } 19:18:28 DEBUG } 19:18:28 DEBUG } 19:18:28 DEBUG ] 19:18:28 DEBUG } 19:18:28 DEBUG } 19:18:28 DEBUG } 19:18:28 DEBUG } 19:18:28 INFO [loop_until]: OK (rc = 0) 19:18:28 DEBUG --- stdout --- 19:18:28 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 68m 0% 6250Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 6457m 21% 7139Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 14376m 48% 9936Mi 8% gke-xlou-cdm-default-pool-1b0769c8-qkns 79m 0% 3350Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 15034m 50% 13874Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 153m 0% 5191Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 14601m 48% 10070Mi 9% gke-xlou-cdm-ds-0f601751-cf7b 12371m 41% 3014Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 16493m 55% 9517Mi 8% gke-xlou-cdm-ds-0f601751-j488 6971m 23% 6715Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 13421m 44% 15925Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 6963m 23% 7361Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 15376m 51% 13335Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 7905m 49% 6936Mi 11% 19:18:28 DEBUG --- stderr --- 19:18:28 DEBUG 19:19:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713550768" 19:19:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550768" 19:19:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550768" 19:19:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550768" 19:19:28 INFO [http_cmd]: http status code OK 19:19:28 ERROR [http_cmd]: ERROR 19:19:28 DEBUG --- status code --- 19:19:28 INFO [http_cmd]: http status code OK 19:19:28 ERROR --- status code --- 19:19:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:28 DEBUG --- status code --- 19:19:28 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:19:28 DEBUG --- http response --- 19:19:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:28 ERROR --- http response --- 19:19:28 DEBUG --- http response --- 19:19:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:19:28 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:19:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:19:28 DEBUG ----- output ----- 19:19:28 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:19:28 DEBUG ----- output ----- 19:19:28 DEBUG { 19:19:28 DEBUG { 19:19:28 DEBUG "status": "success", 19:19:28 DEBUG "status": "success", 19:19:28 DEBUG "data": { 19:19:28 DEBUG "data": { 19:19:28 DEBUG "resultType": "vector", 19:19:28 DEBUG "resultType": "vector", 19:19:28 DEBUG "result": [ 19:19:28 DEBUG "result": [ 19:19:28 DEBUG { 19:19:28 DEBUG { 19:19:28 DEBUG "metric": { 19:19:28 DEBUG "metric": { 19:19:28 DEBUG "pod": "ds-idrepo-0" 19:19:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:19:28 DEBUG }, 19:19:28 DEBUG }, 19:19:28 DEBUG "value": [ 19:19:28 DEBUG "value": [ 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG [...] 19:19:28 DEBUG [...] 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG "pod": "ds-idrepo-2" 19:19:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:19:28 DEBUG }, 19:19:28 DEBUG }, 19:19:28 DEBUG "value": [ 19:19:28 DEBUG "value": [ 19:19:28 DEBUG 1713550768, 19:19:28 DEBUG 1713550768, 19:19:28 DEBUG "0" 19:19:28 DEBUG "2336.6" 19:19:28 DEBUG ] 19:19:28 DEBUG ] 19:19:28 DEBUG } 19:19:28 DEBUG } 19:19:28 DEBUG ] 19:19:28 DEBUG ] 19:19:28 DEBUG } 19:19:28 DEBUG } 19:19:28 DEBUG } 19:19:28 DEBUG } 19:19:28 INFO [http_cmd]: http status code OK 19:19:28 DEBUG --- status code --- 19:19:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:28 DEBUG --- http response --- 19:19:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:19:28 DEBUG ----- output ----- 19:19:28 DEBUG { 19:19:28 DEBUG "status": "success", 19:19:28 DEBUG "data": { 19:19:28 DEBUG "resultType": "vector", 19:19:28 DEBUG "result": [ 19:19:28 DEBUG { 19:19:28 DEBUG "metric": { 19:19:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:19:28 DEBUG }, 19:19:28 DEBUG "value": [ 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG [...] 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:19:28 DEBUG }, 19:19:28 DEBUG "value": [ 19:19:28 DEBUG 1713550768, 19:19:28 DEBUG "2332.7666666666664" 19:19:28 DEBUG ] 19:19:28 DEBUG } 19:19:28 DEBUG ] 19:19:28 DEBUG } 19:19:28 DEBUG } 19:19:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550768" 19:19:28 INFO [http_cmd]: http status code OK 19:19:28 DEBUG --- status code --- 19:19:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:28 DEBUG --- http response --- 19:19:28 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:19:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550768" 19:19:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713550768" 19:19:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550768" 19:19:28 INFO [http_cmd]: http status code OK 19:19:28 DEBUG --- status code --- 19:19:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:28 DEBUG --- http response --- 19:19:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:19:28 DEBUG ----- output ----- 19:19:28 DEBUG { 19:19:28 DEBUG "status": "success", 19:19:28 DEBUG "data": { 19:19:28 DEBUG "resultType": "vector", 19:19:28 DEBUG "result": [ 19:19:28 DEBUG { 19:19:28 DEBUG "metric": { 19:19:28 DEBUG "pod": "ds-idrepo-0" 19:19:28 DEBUG }, 19:19:28 DEBUG "value": [ 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG [...] 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG "pod": "ds-idrepo-2" 19:19:28 DEBUG }, 19:19:28 DEBUG "value": [ 19:19:28 DEBUG 1713550768, 19:19:28 DEBUG "0" 19:19:28 DEBUG ] 19:19:28 DEBUG } 19:19:28 DEBUG ] 19:19:28 DEBUG } 19:19:28 DEBUG } 19:19:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550768" 19:19:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713550768" 19:19:28 INFO [http_cmd]: http status code OK 19:19:28 DEBUG --- status code --- 19:19:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:28 DEBUG --- http response --- 19:19:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:19:28 DEBUG ----- output ----- 19:19:28 DEBUG { 19:19:28 DEBUG "status": "success", 19:19:28 DEBUG "data": { 19:19:28 DEBUG "resultType": "vector", 19:19:28 DEBUG "result": [ 19:19:28 DEBUG { 19:19:28 DEBUG "metric": { 19:19:28 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:19:28 DEBUG }, 19:19:28 DEBUG "value": [ 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG [...] 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:19:28 DEBUG }, 19:19:28 DEBUG "value": [ 19:19:28 DEBUG 1713550768, 19:19:28 DEBUG "7.772386732328593" 19:19:28 DEBUG ] 19:19:28 DEBUG } 19:19:28 DEBUG ] 19:19:28 DEBUG } 19:19:28 DEBUG } 19:19:28 INFO [http_cmd]: http status code OK 19:19:28 DEBUG --- status code --- 19:19:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:28 DEBUG --- http response --- 19:19:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:19:28 DEBUG ----- output ----- 19:19:28 DEBUG { 19:19:28 DEBUG "status": "success", 19:19:28 DEBUG "data": { 19:19:28 DEBUG "resultType": "vector", 19:19:28 DEBUG "result": [ 19:19:28 DEBUG { 19:19:28 DEBUG "metric": { 19:19:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:19:28 DEBUG }, 19:19:28 DEBUG "value": [ 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG [...] 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:19:28 DEBUG }, 19:19:28 DEBUG "value": [ 19:19:28 DEBUG 1713550768, 19:19:28 DEBUG "0" 19:19:28 DEBUG ] 19:19:28 DEBUG } 19:19:28 DEBUG ] 19:19:28 DEBUG } 19:19:28 DEBUG } 19:19:28 INFO [http_cmd]: http status code OK 19:19:28 DEBUG --- status code --- 19:19:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:28 DEBUG --- http response --- 19:19:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:19:28 DEBUG ----- output ----- 19:19:28 DEBUG { 19:19:28 DEBUG "status": "success", 19:19:28 DEBUG "data": { 19:19:28 DEBUG "resultType": "vector", 19:19:28 DEBUG "result": [ 19:19:28 DEBUG { 19:19:28 DEBUG "metric": { 19:19:28 DEBUG "pod": "ds-cts-0" 19:19:28 DEBUG }, 19:19:28 DEBUG "value": [ 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG [...] 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG "pod": "ds-cts-2" 19:19:28 DEBUG }, 19:19:28 DEBUG "value": [ 19:19:28 DEBUG 1713550768, 19:19:28 DEBUG "0" 19:19:28 DEBUG ] 19:19:28 DEBUG } 19:19:28 DEBUG ] 19:19:28 DEBUG } 19:19:28 DEBUG } 19:19:28 INFO [http_cmd]: http status code OK 19:19:28 DEBUG --- status code --- 19:19:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:28 DEBUG --- http response --- 19:19:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:19:28 DEBUG ----- output ----- 19:19:28 DEBUG { 19:19:28 DEBUG "status": "success", 19:19:28 DEBUG "data": { 19:19:28 DEBUG "resultType": "vector", 19:19:28 DEBUG "result": [ 19:19:28 DEBUG { 19:19:28 DEBUG "metric": { 19:19:28 DEBUG "pod": "ds-idrepo-0" 19:19:28 DEBUG }, 19:19:28 DEBUG "value": [ 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG [...] 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG "pod": "ds-cts-2" 19:19:28 DEBUG }, 19:19:28 DEBUG "value": [ 19:19:28 DEBUG 1713550768, 19:19:28 DEBUG "0" 19:19:28 DEBUG ] 19:19:28 DEBUG } 19:19:28 DEBUG ] 19:19:28 DEBUG } 19:19:28 DEBUG } 19:19:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550768" 19:19:28 INFO [http_cmd]: http status code OK 19:19:28 DEBUG --- status code --- 19:19:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:28 DEBUG --- http response --- 19:19:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:19:28 DEBUG ----- output ----- 19:19:28 DEBUG { 19:19:28 DEBUG "status": "success", 19:19:28 DEBUG "data": { 19:19:28 DEBUG "resultType": "vector", 19:19:28 DEBUG "result": [ 19:19:28 DEBUG { 19:19:28 DEBUG "metric": { 19:19:28 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:19:28 DEBUG }, 19:19:28 DEBUG "value": [ 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG [...] 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:19:28 DEBUG }, 19:19:28 DEBUG "value": [ 19:19:28 DEBUG 1713550768, 19:19:28 DEBUG "0" 19:19:28 DEBUG ] 19:19:28 DEBUG } 19:19:28 DEBUG ] 19:19:28 DEBUG } 19:19:28 DEBUG } 19:19:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550768" 19:19:28 INFO [http_cmd]: http status code OK 19:19:28 DEBUG --- status code --- 19:19:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:28 DEBUG --- http response --- 19:19:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:19:28 DEBUG ----- output ----- 19:19:28 DEBUG { 19:19:28 DEBUG "status": "success", 19:19:28 DEBUG "data": { 19:19:28 DEBUG "resultType": "vector", 19:19:28 DEBUG "result": [ 19:19:28 DEBUG { 19:19:28 DEBUG "metric": { 19:19:28 DEBUG "instance": "10.142.0.107:9100" 19:19:28 DEBUG }, 19:19:28 DEBUG "value": [ 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG [...] 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG "instance": "10.142.0.43:9100" 19:19:28 DEBUG }, 19:19:28 DEBUG "value": [ 19:19:28 DEBUG 1713550768, 19:19:28 DEBUG "1860563387392" 19:19:28 DEBUG ] 19:19:28 DEBUG } 19:19:28 DEBUG ] 19:19:28 DEBUG } 19:19:28 DEBUG } 19:19:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550768" 19:19:28 INFO 19:19:28 INFO [loop_until]: kubectl --namespace=xlou top pods 19:19:28 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:19:28 INFO [http_cmd]: http status code OK 19:19:28 DEBUG --- status code --- 19:19:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:28 DEBUG --- http response --- 19:19:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:19:28 DEBUG ----- output ----- 19:19:28 DEBUG { 19:19:28 DEBUG "status": "success", 19:19:28 DEBUG "data": { 19:19:28 DEBUG "resultType": "vector", 19:19:28 DEBUG "result": [ 19:19:28 DEBUG { 19:19:28 DEBUG "metric": { 19:19:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:19:28 DEBUG }, 19:19:28 DEBUG "value": [ 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG [...] 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:19:28 DEBUG }, 19:19:28 DEBUG "value": [ 19:19:28 DEBUG 1713550768, 19:19:28 DEBUG "2332.6666666666665" 19:19:28 DEBUG ] 19:19:28 DEBUG } 19:19:28 DEBUG ] 19:19:28 DEBUG } 19:19:28 DEBUG } 19:19:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550768" 19:19:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550768" 19:19:28 INFO [http_cmd]: http status code OK 19:19:28 DEBUG --- status code --- 19:19:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:28 DEBUG --- http response --- 19:19:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:19:28 DEBUG ----- output ----- 19:19:28 DEBUG { 19:19:28 DEBUG "status": "success", 19:19:28 DEBUG "data": { 19:19:28 DEBUG "resultType": "vector", 19:19:28 DEBUG "result": [ 19:19:28 DEBUG { 19:19:28 DEBUG "metric": { 19:19:28 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:19:28 DEBUG }, 19:19:28 DEBUG "value": [ 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG [...] 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:19:28 DEBUG }, 19:19:28 DEBUG "value": [ 19:19:28 DEBUG 1713550768, 19:19:28 DEBUG "0" 19:19:28 DEBUG ] 19:19:28 DEBUG } 19:19:28 DEBUG ] 19:19:28 DEBUG } 19:19:28 DEBUG } 19:19:28 INFO [http_cmd]: http status code OK 19:19:28 DEBUG --- status code --- 19:19:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:28 DEBUG --- http response --- 19:19:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:19:28 DEBUG ----- output ----- 19:19:28 DEBUG { 19:19:28 DEBUG "status": "success", 19:19:28 DEBUG "data": { 19:19:28 DEBUG "resultType": "vector", 19:19:28 DEBUG "result": [ 19:19:28 DEBUG { 19:19:28 DEBUG "metric": { 19:19:28 DEBUG "instance": "10.142.0.107:9100" 19:19:28 DEBUG }, 19:19:28 DEBUG "value": [ 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG [...] 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG "instance": "10.142.0.43:9100" 19:19:28 DEBUG }, 19:19:28 DEBUG "value": [ 19:19:28 DEBUG 1713550768, 19:19:28 DEBUG "0.06966666666665637" 19:19:28 DEBUG ] 19:19:28 DEBUG } 19:19:28 DEBUG ] 19:19:28 DEBUG } 19:19:28 DEBUG } 19:19:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550768" 19:19:28 INFO [http_cmd]: http status code OK 19:19:28 DEBUG --- status code --- 19:19:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:28 DEBUG --- http response --- 19:19:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:19:28 DEBUG ----- output ----- 19:19:28 DEBUG { 19:19:28 DEBUG "status": "success", 19:19:28 DEBUG "data": { 19:19:28 DEBUG "resultType": "vector", 19:19:28 DEBUG "result": [ 19:19:28 DEBUG { 19:19:28 DEBUG "metric": { 19:19:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:19:28 DEBUG }, 19:19:28 DEBUG "value": [ 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG [...] 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:19:28 DEBUG }, 19:19:28 DEBUG "value": [ 19:19:28 DEBUG 1713550768, 19:19:28 DEBUG "420862.5" 19:19:28 DEBUG ] 19:19:28 DEBUG } 19:19:28 DEBUG ] 19:19:28 DEBUG } 19:19:28 DEBUG } 19:19:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713550768" 19:19:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550768" 19:19:28 INFO [loop_until]: OK (rc = 0) 19:19:28 DEBUG --- stdout --- 19:19:28 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 13997m 8809Mi am-767f6b67b5-qcqz2 14155m 8820Mi am-767f6b67b5-vbltz 14974m 8825Mi ds-cts-0 14617m 10846Mi ds-cts-1 13878m 11083Mi ds-cts-2 15491m 10826Mi ds-idrepo-0 6584m 3570Mi ds-idrepo-1 6360m 3462Mi ds-idrepo-2 6346m 3493Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 2m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7552m 3125Mi 19:19:28 DEBUG --- stderr --- 19:19:28 DEBUG 19:19:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550768" 19:19:28 INFO [http_cmd]: http status code OK 19:19:28 DEBUG --- status code --- 19:19:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:28 DEBUG --- http response --- 19:19:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:19:28 DEBUG ----- output ----- 19:19:28 DEBUG { 19:19:28 DEBUG "status": "success", 19:19:28 DEBUG "data": { 19:19:28 DEBUG "resultType": "vector", 19:19:28 DEBUG "result": [ 19:19:28 DEBUG { 19:19:28 DEBUG "metric": { 19:19:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:19:28 DEBUG }, 19:19:28 DEBUG "value": [ 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG [...] 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:19:28 DEBUG }, 19:19:28 DEBUG "value": [ 19:19:28 DEBUG 1713550768, 19:19:28 DEBUG "0.003" 19:19:28 DEBUG ] 19:19:28 DEBUG } 19:19:28 DEBUG ] 19:19:28 DEBUG } 19:19:28 DEBUG } 19:19:28 INFO [http_cmd]: http status code OK 19:19:28 DEBUG --- status code --- 19:19:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:28 DEBUG --- http response --- 19:19:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:19:28 DEBUG ----- output ----- 19:19:28 DEBUG { 19:19:28 DEBUG "status": "success", 19:19:28 DEBUG "data": { 19:19:28 DEBUG "resultType": "vector", 19:19:28 DEBUG "result": [ 19:19:28 DEBUG { 19:19:28 DEBUG "metric": { 19:19:28 DEBUG "pod": "ds-idrepo-0" 19:19:28 DEBUG }, 19:19:28 DEBUG "value": [ 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG [...] 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG "pod": "ds-idrepo-2" 19:19:28 DEBUG }, 19:19:28 DEBUG "value": [ 19:19:28 DEBUG 1713550768, 19:19:28 DEBUG "0" 19:19:28 DEBUG ] 19:19:28 DEBUG } 19:19:28 DEBUG ] 19:19:28 DEBUG } 19:19:28 DEBUG } 19:19:28 INFO [http_cmd]: http status code OK 19:19:28 DEBUG --- status code --- 19:19:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:28 DEBUG --- http response --- 19:19:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:19:28 DEBUG ----- output ----- 19:19:28 DEBUG { 19:19:28 DEBUG "status": "success", 19:19:28 DEBUG "data": { 19:19:28 DEBUG "resultType": "vector", 19:19:28 DEBUG "result": [ 19:19:28 DEBUG { 19:19:28 DEBUG "metric": { 19:19:28 DEBUG "instance": "10.142.0.107:9100" 19:19:28 DEBUG }, 19:19:28 DEBUG "value": [ 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG [...] 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG "instance": "10.142.0.43:9100" 19:19:28 DEBUG }, 19:19:28 DEBUG "value": [ 19:19:28 DEBUG 1713550768, 19:19:28 DEBUG "0.160599999999936" 19:19:28 DEBUG ] 19:19:28 DEBUG } 19:19:28 DEBUG ] 19:19:28 DEBUG } 19:19:28 DEBUG } 19:19:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550768" 19:19:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550768" 19:19:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550768" 19:19:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550768" 19:19:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713550768" 19:19:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550768" 19:19:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550768" 19:19:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550768" 19:19:28 INFO [http_cmd]: http status code OK 19:19:28 INFO [http_cmd]: http status code OK 19:19:28 INFO [http_cmd]: http status code OK 19:19:28 DEBUG --- status code --- 19:19:28 DEBUG --- status code --- 19:19:28 DEBUG --- status code --- 19:19:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:28 DEBUG --- http response --- 19:19:28 DEBUG --- http response --- 19:19:28 DEBUG --- http response --- 19:19:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:19:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:19:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:19:28 INFO [http_cmd]: http status code OK 19:19:28 DEBUG ----- output ----- 19:19:28 DEBUG ----- output ----- 19:19:28 DEBUG ----- output ----- 19:19:28 DEBUG --- status code --- 19:19:28 DEBUG { 19:19:28 DEBUG { 19:19:28 DEBUG { 19:19:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:28 DEBUG "status": "success", 19:19:28 DEBUG "status": "success", 19:19:28 DEBUG "status": "success", 19:19:28 DEBUG --- http response --- 19:19:28 DEBUG "data": { 19:19:28 DEBUG "data": { 19:19:28 DEBUG "data": { 19:19:28 DEBUG "resultType": "vector", 19:19:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:19:28 DEBUG "resultType": "vector", 19:19:28 DEBUG "resultType": "vector", 19:19:28 DEBUG "result": [ 19:19:28 DEBUG ----- output ----- 19:19:28 DEBUG "result": [ 19:19:28 DEBUG "result": [ 19:19:28 DEBUG { 19:19:28 DEBUG { 19:19:28 DEBUG { 19:19:28 DEBUG { 19:19:28 DEBUG "metric": { 19:19:28 DEBUG "status": "success", 19:19:28 DEBUG "metric": { 19:19:28 DEBUG "metric": { 19:19:28 DEBUG "instance": "10.142.0.107:9100" 19:19:28 DEBUG "data": { 19:19:28 INFO [http_cmd]: http status code OK 19:19:28 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:19:28 INFO [http_cmd]: http status code OK 19:19:28 INFO [http_cmd]: http status code OK 19:19:28 DEBUG "instance": "10.142.0.107:9100" 19:19:28 DEBUG }, 19:19:28 DEBUG "resultType": "vector", 19:19:28 INFO [http_cmd]: http status code OK 19:19:28 DEBUG --- status code --- 19:19:28 DEBUG }, 19:19:28 DEBUG --- status code --- 19:19:28 DEBUG --- status code --- 19:19:28 DEBUG }, 19:19:28 DEBUG "value": [ 19:19:28 DEBUG "result": [ 19:19:28 DEBUG --- status code --- 19:19:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:28 DEBUG "value": [ 19:19:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:28 DEBUG "value": [ 19:19:28 DEBUG 19:19:28 DEBUG { 19:19:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:28 DEBUG --- http response --- 19:19:28 DEBUG 19:19:28 DEBUG --- http response --- 19:19:28 DEBUG --- http response --- 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG "metric": { 19:19:28 DEBUG --- http response --- 19:19:28 DEBUG 19:19:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:19:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:19:28 DEBUG 19:19:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:19:28 DEBUG 19:19:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:19:28 DEBUG 19:19:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:19:28 DEBUG ----- output ----- 19:19:28 DEBUG ----- output ----- 19:19:28 DEBUG 19:19:28 DEBUG ----- output ----- 19:19:28 DEBUG [...] 19:19:28 DEBUG }, 19:19:28 DEBUG [...] 19:19:28 DEBUG ----- output ----- 19:19:28 DEBUG { 19:19:28 DEBUG { 19:19:28 DEBUG [...] 19:19:28 DEBUG { 19:19:28 DEBUG 19:19:28 DEBUG "value": [ 19:19:28 DEBUG 19:19:28 DEBUG { 19:19:28 DEBUG "status": "success", 19:19:28 DEBUG "status": "success", 19:19:28 DEBUG 19:19:28 DEBUG "status": "success", 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG "status": "success", 19:19:28 DEBUG "data": { 19:19:28 DEBUG "data": { 19:19:28 DEBUG 19:19:28 DEBUG "data": { 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG "data": { 19:19:28 DEBUG "resultType": "vector", 19:19:28 DEBUG "resultType": "vector", 19:19:28 DEBUG 19:19:28 DEBUG "resultType": "vector", 19:19:28 DEBUG "instance": "10.142.0.43:9100" 19:19:28 DEBUG 19:19:28 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:19:28 DEBUG "resultType": "vector", 19:19:28 DEBUG "result": [ 19:19:28 DEBUG "result": [ 19:19:28 DEBUG "instance": "10.142.0.43:9100" 19:19:28 DEBUG "result": [ 19:19:28 DEBUG }, 19:19:28 DEBUG [...] 19:19:28 DEBUG }, 19:19:28 DEBUG "result": [ 19:19:28 DEBUG { 19:19:28 DEBUG { 19:19:28 DEBUG }, 19:19:28 DEBUG { 19:19:28 DEBUG "value": [ 19:19:28 DEBUG 19:19:28 DEBUG "value": [ 19:19:28 DEBUG { 19:19:28 DEBUG "metric": { 19:19:28 DEBUG "metric": { 19:19:28 DEBUG "value": [ 19:19:28 DEBUG "metric": { 19:19:28 DEBUG 1713550768, 19:19:28 DEBUG 19:19:28 DEBUG 1713550768, 19:19:28 DEBUG "metric": { 19:19:28 DEBUG "pod": "ds-cts-0" 19:19:28 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:19:28 DEBUG 1713550768, 19:19:28 DEBUG "pod": "ds-cts-0" 19:19:28 DEBUG "70690908.13333333" 19:19:28 DEBUG 19:19:28 DEBUG "0" 19:19:28 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:19:28 DEBUG }, 19:19:28 DEBUG }, 19:19:28 DEBUG "70693218" 19:19:28 DEBUG }, 19:19:28 DEBUG ] 19:19:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:19:28 DEBUG ] 19:19:28 DEBUG }, 19:19:28 DEBUG "value": [ 19:19:28 DEBUG "value": [ 19:19:28 DEBUG ] 19:19:28 DEBUG "value": [ 19:19:28 DEBUG } 19:19:28 DEBUG }, 19:19:28 DEBUG } 19:19:28 DEBUG "value": [ 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG } 19:19:28 DEBUG 19:19:28 DEBUG ] 19:19:28 DEBUG "value": [ 19:19:28 DEBUG ] 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG ] 19:19:28 DEBUG 19:19:28 DEBUG } 19:19:28 DEBUG 1713550768, 19:19:28 DEBUG } 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG } 19:19:28 DEBUG 19:19:28 DEBUG } 19:19:28 DEBUG "2333.0666666666666" 19:19:28 DEBUG } 19:19:28 DEBUG 19:19:28 DEBUG [...] 19:19:28 DEBUG [...] 19:19:28 DEBUG } 19:19:28 DEBUG [...] 19:19:28 DEBUG ] 19:19:28 DEBUG [...] 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG } 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG ] 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG 19:19:28 DEBUG } 19:19:28 DEBUG 19:19:28 DEBUG "pod": "ds-cts-2" 19:19:28 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:19:28 DEBUG "pod": "ds-cts-2" 19:19:28 DEBUG } 19:19:28 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:19:28 DEBUG }, 19:19:28 DEBUG }, 19:19:28 DEBUG }, 19:19:28 DEBUG }, 19:19:28 DEBUG "value": [ 19:19:28 DEBUG "value": [ 19:19:28 DEBUG "value": [ 19:19:28 DEBUG "value": [ 19:19:28 DEBUG 1713550768, 19:19:28 DEBUG 1713550768, 19:19:28 DEBUG 1713550768, 19:19:28 DEBUG 1713550768, 19:19:28 DEBUG "0.00040053999999999996" 19:19:28 DEBUG "0" 19:19:28 DEBUG "0" 19:19:28 DEBUG "3374510080" 19:19:28 DEBUG ] 19:19:28 DEBUG ] 19:19:28 DEBUG ] 19:19:28 DEBUG ] 19:19:28 DEBUG } 19:19:28 DEBUG } 19:19:28 DEBUG } 19:19:28 DEBUG } 19:19:28 DEBUG ] 19:19:28 DEBUG ] 19:19:28 DEBUG ] 19:19:28 DEBUG ] 19:19:28 DEBUG } 19:19:28 DEBUG } 19:19:28 DEBUG } 19:19:28 DEBUG } 19:19:28 DEBUG } 19:19:28 DEBUG } 19:19:28 DEBUG } 19:19:28 DEBUG } 19:19:28 INFO 19:19:28 INFO [loop_until]: kubectl --namespace=xlou top node 19:19:28 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:19:28 INFO [loop_until]: OK (rc = 0) 19:19:28 DEBUG --- stdout --- 19:19:28 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 67m 0% 6231Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 6468m 21% 7148Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13695m 45% 11510Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 79m 0% 3354Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 14699m 49% 13883Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 154m 0% 5199Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 14261m 47% 11687Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12321m 41% 3013Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 15653m 52% 11030Mi 9% gke-xlou-cdm-ds-0f601751-j488 7027m 23% 6733Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 13532m 45% 15915Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 6934m 23% 7357Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 15285m 51% 13336Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 7864m 49% 6936Mi 11% 19:19:28 DEBUG --- stderr --- 19:19:28 DEBUG 19:20:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713550828" 19:20:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550828" 19:20:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550828" 19:20:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550828" 19:20:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550828" 19:20:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550828" 19:20:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550828" 19:20:28 ERROR [http_cmd]: ERROR 19:20:28 ERROR --- status code --- 19:20:28 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:20:28 ERROR --- http response --- 19:20:28 INFO [http_cmd]: http status code OK 19:20:28 INFO [http_cmd]: http status code OK 19:20:28 INFO [http_cmd]: http status code OK 19:20:28 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:20:28 DEBUG --- status code --- 19:20:28 DEBUG --- status code --- 19:20:28 DEBUG --- status code --- 19:20:28 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:20:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:28 DEBUG --- http response --- 19:20:28 DEBUG --- http response --- 19:20:28 DEBUG --- http response --- 19:20:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:20:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:20:28 INFO [http_cmd]: http status code OK 19:20:28 INFO [http_cmd]: http status code OK 19:20:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:20:28 DEBUG ----- output ----- 19:20:28 DEBUG ----- output ----- 19:20:28 DEBUG --- status code --- 19:20:28 DEBUG --- status code --- 19:20:28 DEBUG ----- output ----- 19:20:28 DEBUG { 19:20:28 DEBUG { 19:20:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:28 DEBUG { 19:20:28 DEBUG "status": "success", 19:20:28 DEBUG "status": "success", 19:20:28 DEBUG --- http response --- 19:20:28 DEBUG --- http response --- 19:20:28 DEBUG "status": "success", 19:20:28 DEBUG "data": { 19:20:28 DEBUG "data": { 19:20:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:20:28 DEBUG "data": { 19:20:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:20:28 DEBUG "resultType": "vector", 19:20:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713550828" 19:20:28 DEBUG "resultType": "vector", 19:20:28 DEBUG ----- output ----- 19:20:28 DEBUG "resultType": "vector", 19:20:28 DEBUG ----- output ----- 19:20:28 DEBUG "result": [ 19:20:28 DEBUG "result": [ 19:20:28 DEBUG { 19:20:28 DEBUG "result": [ 19:20:28 DEBUG { 19:20:28 DEBUG { 19:20:28 DEBUG { 19:20:28 DEBUG "status": "success", 19:20:28 DEBUG { 19:20:28 DEBUG "status": "success", 19:20:28 DEBUG "metric": { 19:20:28 DEBUG "metric": { 19:20:28 DEBUG "data": { 19:20:28 DEBUG "metric": { 19:20:28 DEBUG "data": { 19:20:28 DEBUG "pod": "ds-idrepo-0" 19:20:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:20:28 DEBUG "resultType": "vector", 19:20:28 DEBUG "pod": "ds-idrepo-0" 19:20:28 DEBUG "resultType": "vector", 19:20:28 DEBUG }, 19:20:28 DEBUG }, 19:20:28 DEBUG "result": [ 19:20:28 DEBUG }, 19:20:28 DEBUG "result": [ 19:20:28 DEBUG "value": [ 19:20:28 DEBUG "value": [ 19:20:28 DEBUG { 19:20:28 DEBUG "value": [ 19:20:28 DEBUG { 19:20:28 DEBUG 19:20:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550828" 19:20:28 DEBUG 19:20:28 DEBUG "metric": { 19:20:28 DEBUG 19:20:28 DEBUG "metric": { 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:20:28 DEBUG 19:20:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG }, 19:20:28 DEBUG 19:20:28 DEBUG }, 19:20:28 DEBUG [...] 19:20:28 DEBUG [...] 19:20:28 DEBUG "value": [ 19:20:28 DEBUG [...] 19:20:28 DEBUG "value": [ 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 INFO [http_cmd]: http status code OK 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG --- status code --- 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG "pod": "ds-idrepo-2" 19:20:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:20:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:28 DEBUG [...] 19:20:28 DEBUG "pod": "ds-idrepo-2" 19:20:28 DEBUG [...] 19:20:28 DEBUG }, 19:20:28 DEBUG }, 19:20:28 DEBUG --- http response --- 19:20:28 DEBUG 19:20:28 DEBUG }, 19:20:28 DEBUG 19:20:28 DEBUG "value": [ 19:20:28 DEBUG "value": [ 19:20:28 DEBUG 19:20:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:20:28 DEBUG "value": [ 19:20:28 DEBUG 19:20:28 DEBUG 1713550828, 19:20:28 DEBUG 1713550828, 19:20:28 DEBUG 19:20:28 DEBUG ----- output ----- 19:20:28 DEBUG 1713550828, 19:20:28 DEBUG 19:20:28 DEBUG "0" 19:20:28 DEBUG "2327.7" 19:20:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:20:28 DEBUG { 19:20:28 DEBUG "0" 19:20:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:20:28 DEBUG ] 19:20:28 DEBUG ] 19:20:28 DEBUG }, 19:20:28 DEBUG "status": "success", 19:20:28 DEBUG ] 19:20:28 DEBUG }, 19:20:28 DEBUG } 19:20:28 DEBUG } 19:20:28 DEBUG "value": [ 19:20:28 DEBUG "data": { 19:20:28 DEBUG } 19:20:28 DEBUG "value": [ 19:20:28 DEBUG ] 19:20:28 DEBUG ] 19:20:28 DEBUG 1713550828, 19:20:28 DEBUG "resultType": "vector", 19:20:28 DEBUG ] 19:20:28 DEBUG 1713550828, 19:20:28 DEBUG } 19:20:28 DEBUG } 19:20:28 DEBUG "0" 19:20:28 DEBUG "result": [ 19:20:28 DEBUG } 19:20:28 DEBUG "2337.9666666666667" 19:20:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713550828" 19:20:28 DEBUG } 19:20:28 DEBUG } 19:20:28 DEBUG ] 19:20:28 DEBUG { 19:20:28 DEBUG } 19:20:28 DEBUG ] 19:20:28 DEBUG } 19:20:28 INFO [http_cmd]: http status code OK 19:20:28 DEBUG "metric": { 19:20:28 DEBUG } 19:20:28 DEBUG ] 19:20:28 DEBUG --- status code --- 19:20:28 DEBUG "pod": "ds-cts-0" 19:20:28 DEBUG ] 19:20:28 DEBUG } 19:20:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:28 DEBUG }, 19:20:28 DEBUG } 19:20:28 DEBUG } 19:20:28 DEBUG --- http response --- 19:20:28 DEBUG "value": [ 19:20:28 DEBUG } 19:20:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:20:28 DEBUG 19:20:28 DEBUG ----- output ----- 19:20:28 DEBUG 19:20:28 DEBUG { 19:20:28 DEBUG 19:20:28 DEBUG "status": "success", 19:20:28 INFO [http_cmd]: http status code OK 19:20:28 DEBUG [...] 19:20:28 DEBUG "data": { 19:20:28 DEBUG --- status code --- 19:20:28 DEBUG 19:20:28 DEBUG "resultType": "vector", 19:20:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:28 DEBUG 19:20:28 DEBUG "result": [ 19:20:28 DEBUG --- http response --- 19:20:28 DEBUG 19:20:28 DEBUG { 19:20:28 DEBUG "pod": "ds-cts-2" 19:20:28 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:20:28 DEBUG "metric": { 19:20:28 DEBUG }, 19:20:28 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:20:28 DEBUG "value": [ 19:20:28 DEBUG }, 19:20:28 DEBUG 1713550828, 19:20:28 DEBUG "value": [ 19:20:28 DEBUG "0" 19:20:28 DEBUG 19:20:28 DEBUG ] 19:20:28 DEBUG 19:20:28 DEBUG } 19:20:28 DEBUG 19:20:28 DEBUG ] 19:20:28 DEBUG [...] 19:20:28 DEBUG } 19:20:28 DEBUG 19:20:28 DEBUG } 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:20:28 DEBUG }, 19:20:28 DEBUG "value": [ 19:20:28 DEBUG 1713550828, 19:20:28 DEBUG "7.708330035422991" 19:20:28 DEBUG ] 19:20:28 DEBUG } 19:20:28 DEBUG ] 19:20:28 DEBUG } 19:20:28 DEBUG } 19:20:28 INFO [http_cmd]: http status code OK 19:20:28 DEBUG --- status code --- 19:20:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:28 DEBUG --- http response --- 19:20:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:20:28 DEBUG ----- output ----- 19:20:28 DEBUG { 19:20:28 DEBUG "status": "success", 19:20:28 DEBUG "data": { 19:20:28 DEBUG "resultType": "vector", 19:20:28 DEBUG "result": [ 19:20:28 DEBUG { 19:20:28 DEBUG "metric": { 19:20:28 DEBUG "pod": "ds-idrepo-0" 19:20:28 DEBUG }, 19:20:28 DEBUG "value": [ 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG [...] 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG "pod": "ds-cts-2" 19:20:28 DEBUG }, 19:20:28 DEBUG "value": [ 19:20:28 DEBUG 1713550828, 19:20:28 DEBUG "0" 19:20:28 DEBUG ] 19:20:28 DEBUG } 19:20:28 DEBUG ] 19:20:28 DEBUG } 19:20:28 DEBUG } 19:20:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550828" 19:20:28 INFO [http_cmd]: http status code OK 19:20:28 DEBUG --- status code --- 19:20:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:28 DEBUG --- http response --- 19:20:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:20:28 DEBUG ----- output ----- 19:20:28 DEBUG { 19:20:28 DEBUG "status": "success", 19:20:28 DEBUG "data": { 19:20:28 DEBUG "resultType": "vector", 19:20:28 DEBUG "result": [ 19:20:28 DEBUG { 19:20:28 DEBUG "metric": { 19:20:28 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:20:28 DEBUG }, 19:20:28 DEBUG "value": [ 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG [...] 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:20:28 DEBUG }, 19:20:28 DEBUG "value": [ 19:20:28 DEBUG 1713550828, 19:20:28 DEBUG "0" 19:20:28 DEBUG ] 19:20:28 DEBUG } 19:20:28 DEBUG ] 19:20:28 DEBUG } 19:20:28 DEBUG } 19:20:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550828" 19:20:28 INFO [http_cmd]: http status code OK 19:20:28 DEBUG --- status code --- 19:20:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:28 DEBUG --- http response --- 19:20:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:20:28 DEBUG ----- output ----- 19:20:28 DEBUG { 19:20:28 DEBUG "status": "success", 19:20:28 DEBUG "data": { 19:20:28 DEBUG "resultType": "vector", 19:20:28 DEBUG "result": [ 19:20:28 DEBUG { 19:20:28 DEBUG "metric": { 19:20:28 DEBUG "instance": "10.142.0.107:9100" 19:20:28 DEBUG }, 19:20:28 DEBUG "value": [ 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG [...] 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG "instance": "10.142.0.43:9100" 19:20:28 DEBUG }, 19:20:28 DEBUG "value": [ 19:20:28 DEBUG 1713550828, 19:20:28 DEBUG "1863534984192" 19:20:28 DEBUG ] 19:20:28 DEBUG } 19:20:28 DEBUG ] 19:20:28 DEBUG } 19:20:28 DEBUG } 19:20:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550828" 19:20:28 INFO [http_cmd]: http status code OK 19:20:28 DEBUG --- status code --- 19:20:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:28 DEBUG --- http response --- 19:20:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:20:28 DEBUG ----- output ----- 19:20:28 DEBUG { 19:20:28 DEBUG "status": "success", 19:20:28 DEBUG "data": { 19:20:28 DEBUG "resultType": "vector", 19:20:28 DEBUG "result": [ 19:20:28 DEBUG { 19:20:28 DEBUG "metric": { 19:20:28 DEBUG "instance": "10.142.0.107:9100" 19:20:28 DEBUG }, 19:20:28 DEBUG "value": [ 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG [...] 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG "instance": "10.142.0.43:9100" 19:20:28 DEBUG }, 19:20:28 DEBUG "value": [ 19:20:28 DEBUG 1713550828, 19:20:28 DEBUG "0.03933333333334303" 19:20:28 DEBUG ] 19:20:28 DEBUG } 19:20:28 DEBUG ] 19:20:28 DEBUG } 19:20:28 DEBUG } 19:20:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550828" 19:20:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550828" 19:20:28 INFO [http_cmd]: http status code OK 19:20:28 INFO [http_cmd]: http status code OK 19:20:28 DEBUG --- status code --- 19:20:28 DEBUG --- status code --- 19:20:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:28 DEBUG --- http response --- 19:20:28 DEBUG --- http response --- 19:20:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:20:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:20:28 DEBUG ----- output ----- 19:20:28 DEBUG ----- output ----- 19:20:28 DEBUG { 19:20:28 DEBUG { 19:20:28 DEBUG "status": "success", 19:20:28 DEBUG "status": "success", 19:20:28 DEBUG "data": { 19:20:28 DEBUG "data": { 19:20:28 DEBUG "resultType": "vector", 19:20:28 DEBUG "resultType": "vector", 19:20:28 DEBUG "result": [ 19:20:28 DEBUG "result": [ 19:20:28 DEBUG { 19:20:28 DEBUG { 19:20:28 DEBUG "metric": { 19:20:28 DEBUG "metric": { 19:20:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:20:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:20:28 DEBUG }, 19:20:28 DEBUG }, 19:20:28 DEBUG "value": [ 19:20:28 DEBUG "value": [ 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG [...] 19:20:28 DEBUG [...] 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:20:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:20:28 DEBUG }, 19:20:28 DEBUG }, 19:20:28 DEBUG "value": [ 19:20:28 DEBUG "value": [ 19:20:28 DEBUG 1713550828, 19:20:28 DEBUG 1713550828, 19:20:28 DEBUG "2327.0666666666666" 19:20:28 DEBUG "420427" 19:20:28 DEBUG ] 19:20:28 DEBUG ] 19:20:28 DEBUG } 19:20:28 DEBUG } 19:20:28 DEBUG ] 19:20:28 DEBUG ] 19:20:28 DEBUG } 19:20:28 DEBUG } 19:20:28 DEBUG } 19:20:28 DEBUG } 19:20:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550828" 19:20:28 INFO [http_cmd]: http status code OK 19:20:28 DEBUG --- status code --- 19:20:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:28 DEBUG --- http response --- 19:20:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:20:28 DEBUG ----- output ----- 19:20:28 DEBUG { 19:20:28 DEBUG "status": "success", 19:20:28 DEBUG "data": { 19:20:28 DEBUG "resultType": "vector", 19:20:28 DEBUG "result": [ 19:20:28 DEBUG { 19:20:28 DEBUG "metric": { 19:20:28 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:20:28 DEBUG }, 19:20:28 DEBUG "value": [ 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG [...] 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:20:28 DEBUG }, 19:20:28 DEBUG "value": [ 19:20:28 DEBUG 1713550828, 19:20:28 DEBUG "17594.698385491065" 19:20:28 DEBUG ] 19:20:28 DEBUG } 19:20:28 DEBUG ] 19:20:28 DEBUG } 19:20:28 DEBUG } 19:20:28 INFO 19:20:28 INFO [loop_until]: kubectl --namespace=xlou top pods 19:20:28 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:20:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713550828" 19:20:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550828" 19:20:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550828" 19:20:28 INFO [http_cmd]: http status code OK 19:20:28 INFO [http_cmd]: http status code OK 19:20:28 DEBUG --- status code --- 19:20:28 DEBUG --- status code --- 19:20:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:28 DEBUG --- http response --- 19:20:28 DEBUG --- http response --- 19:20:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:20:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:20:28 DEBUG ----- output ----- 19:20:28 DEBUG ----- output ----- 19:20:28 DEBUG { 19:20:28 DEBUG { 19:20:28 DEBUG "status": "success", 19:20:28 DEBUG "status": "success", 19:20:28 DEBUG "data": { 19:20:28 DEBUG "data": { 19:20:28 DEBUG "resultType": "vector", 19:20:28 DEBUG "resultType": "vector", 19:20:28 DEBUG "result": [ 19:20:28 DEBUG "result": [ 19:20:28 DEBUG { 19:20:28 DEBUG { 19:20:28 DEBUG "metric": { 19:20:28 DEBUG "metric": { 19:20:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:20:28 DEBUG "pod": "ds-idrepo-0" 19:20:28 DEBUG }, 19:20:28 DEBUG }, 19:20:28 DEBUG "value": [ 19:20:28 DEBUG "value": [ 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG [...] 19:20:28 DEBUG [...] 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:20:28 DEBUG "pod": "ds-idrepo-2" 19:20:28 DEBUG }, 19:20:28 DEBUG }, 19:20:28 DEBUG "value": [ 19:20:28 DEBUG "value": [ 19:20:28 DEBUG 1713550828, 19:20:28 DEBUG 1713550828, 19:20:28 DEBUG "0.003" 19:20:28 DEBUG "0" 19:20:28 DEBUG ] 19:20:28 DEBUG ] 19:20:28 DEBUG } 19:20:28 DEBUG } 19:20:28 DEBUG ] 19:20:28 DEBUG ] 19:20:28 DEBUG } 19:20:28 DEBUG } 19:20:28 DEBUG } 19:20:28 DEBUG } 19:20:28 INFO [http_cmd]: http status code OK 19:20:28 DEBUG --- status code --- 19:20:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:28 DEBUG --- http response --- 19:20:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:20:28 DEBUG ----- output ----- 19:20:28 DEBUG { 19:20:28 DEBUG "status": "success", 19:20:28 DEBUG "data": { 19:20:28 DEBUG "resultType": "vector", 19:20:28 DEBUG "result": [ 19:20:28 DEBUG { 19:20:28 DEBUG "metric": { 19:20:28 DEBUG "instance": "10.142.0.107:9100" 19:20:28 DEBUG }, 19:20:28 DEBUG "value": [ 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG [...] 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG "instance": "10.142.0.43:9100" 19:20:28 DEBUG }, 19:20:28 DEBUG "value": [ 19:20:28 DEBUG 1713550828, 19:20:28 DEBUG "0.07843333333337246" 19:20:28 DEBUG ] 19:20:28 DEBUG } 19:20:28 DEBUG ] 19:20:28 DEBUG } 19:20:28 DEBUG } 19:20:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550828" 19:20:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550828" 19:20:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550828" 19:20:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550828" 19:20:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550828" 19:20:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713550828" 19:20:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550828" 19:20:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550828" 19:20:28 INFO [http_cmd]: http status code OK 19:20:28 INFO [http_cmd]: http status code OK 19:20:28 DEBUG --- status code --- 19:20:28 DEBUG --- status code --- 19:20:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:28 DEBUG --- http response --- 19:20:28 INFO [http_cmd]: http status code OK 19:20:28 DEBUG --- http response --- 19:20:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:20:28 DEBUG --- status code --- 19:20:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:20:28 DEBUG ----- output ----- 19:20:28 INFO [http_cmd]: http status code OK 19:20:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:28 DEBUG ----- output ----- 19:20:28 DEBUG { 19:20:28 DEBUG --- status code --- 19:20:28 DEBUG --- http response --- 19:20:28 DEBUG { 19:20:28 DEBUG "status": "success", 19:20:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:28 DEBUG "status": "success", 19:20:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:20:28 DEBUG "data": { 19:20:28 DEBUG --- http response --- 19:20:28 DEBUG "data": { 19:20:28 DEBUG ----- output ----- 19:20:28 DEBUG "resultType": "vector", 19:20:28 DEBUG "resultType": "vector", 19:20:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:20:28 DEBUG { 19:20:28 DEBUG "result": [ 19:20:28 INFO [http_cmd]: http status code OK 19:20:28 DEBUG "result": [ 19:20:28 INFO [http_cmd]: http status code OK 19:20:28 DEBUG ----- output ----- 19:20:28 DEBUG "status": "success", 19:20:28 DEBUG { 19:20:28 DEBUG --- status code --- 19:20:28 DEBUG { 19:20:28 DEBUG --- status code --- 19:20:28 DEBUG { 19:20:28 DEBUG "data": { 19:20:28 DEBUG "metric": { 19:20:28 INFO [http_cmd]: http status code OK 19:20:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:28 DEBUG "metric": { 19:20:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:28 DEBUG "status": "success", 19:20:28 DEBUG "resultType": "vector", 19:20:28 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:20:28 DEBUG --- status code --- 19:20:28 DEBUG --- http response --- 19:20:28 DEBUG "instance": "10.142.0.107:9100" 19:20:28 DEBUG --- http response --- 19:20:28 DEBUG "data": { 19:20:28 DEBUG "result": [ 19:20:28 DEBUG }, 19:20:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:28 DEBUG }, 19:20:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:20:28 DEBUG "resultType": "vector", 19:20:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:20:28 DEBUG { 19:20:28 DEBUG "value": [ 19:20:28 DEBUG --- http response --- 19:20:28 DEBUG "value": [ 19:20:28 DEBUG ----- output ----- 19:20:28 DEBUG "result": [ 19:20:28 DEBUG ----- output ----- 19:20:28 DEBUG "metric": { 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:20:28 DEBUG { 19:20:28 INFO [http_cmd]: http status code OK 19:20:28 DEBUG { 19:20:28 DEBUG { 19:20:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG ----- output ----- 19:20:28 DEBUG "status": "success", 19:20:28 DEBUG --- status code --- 19:20:28 DEBUG "metric": { 19:20:28 DEBUG "status": "success", 19:20:28 DEBUG }, 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG { 19:20:28 DEBUG "data": { 19:20:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:28 DEBUG "instance": "10.142.0.107:9100" 19:20:28 DEBUG "data": { 19:20:28 DEBUG "value": [ 19:20:28 DEBUG [...] 19:20:28 DEBUG [...] 19:20:28 DEBUG "status": "success", 19:20:28 DEBUG "resultType": "vector", 19:20:28 DEBUG --- http response --- 19:20:28 DEBUG }, 19:20:28 DEBUG "resultType": "vector", 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG "data": { 19:20:28 DEBUG "result": [ 19:20:28 DEBUG "value": [ 19:20:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:20:28 DEBUG "result": [ 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG "resultType": "vector", 19:20:28 DEBUG { 19:20:28 DEBUG 19:20:28 DEBUG ----- output ----- 19:20:28 DEBUG { 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG "result": [ 19:20:28 DEBUG "metric": { 19:20:28 DEBUG 19:20:28 DEBUG { 19:20:28 DEBUG "metric": { 19:20:28 DEBUG [...] 19:20:28 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:20:28 DEBUG "instance": "10.142.0.43:9100" 19:20:28 DEBUG { 19:20:28 DEBUG "pod": "ds-cts-0" 19:20:28 DEBUG 19:20:28 DEBUG "status": "success", 19:20:28 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:20:28 DEBUG 19:20:28 DEBUG }, 19:20:28 DEBUG }, 19:20:28 DEBUG "metric": { 19:20:28 DEBUG }, 19:20:28 DEBUG [...] 19:20:28 DEBUG "data": { 19:20:28 DEBUG }, 19:20:28 DEBUG 19:20:28 DEBUG "value": [ 19:20:28 DEBUG "value": [ 19:20:28 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:20:28 DEBUG "value": [ 19:20:28 DEBUG 19:20:28 DEBUG "resultType": "vector", 19:20:28 DEBUG "value": [ 19:20:28 DEBUG 19:20:28 DEBUG 1713550828, 19:20:28 DEBUG 1713550828, 19:20:28 DEBUG }, 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG "result": [ 19:20:28 DEBUG 19:20:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:20:28 DEBUG "3.3404141326453547" 19:20:28 DEBUG "70671453.43333334" 19:20:28 DEBUG "value": [ 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG { 19:20:28 DEBUG 19:20:28 DEBUG }, 19:20:28 DEBUG ] 19:20:28 DEBUG ] 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG "instance": "10.142.0.43:9100" 19:20:28 DEBUG "metric": { 19:20:28 DEBUG 19:20:28 DEBUG "value": [ 19:20:28 DEBUG } 19:20:28 DEBUG } 19:20:28 DEBUG 19:20:28 DEBUG [...] 19:20:28 DEBUG }, 19:20:28 DEBUG "pod": "ds-cts-0" 19:20:28 DEBUG [...] 19:20:28 DEBUG 1713550828, 19:20:28 DEBUG ] 19:20:28 DEBUG ] 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG "value": [ 19:20:28 DEBUG }, 19:20:28 DEBUG 19:20:28 DEBUG "2327.5333333333333" 19:20:28 DEBUG } 19:20:28 DEBUG } 19:20:28 DEBUG [...] 19:20:28 DEBUG 19:20:28 DEBUG 1713550828, 19:20:28 DEBUG "value": [ 19:20:28 DEBUG 19:20:28 DEBUG ] 19:20:28 DEBUG } 19:20:28 DEBUG } 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG "70673566.03333333" 19:20:28 DEBUG 19:20:28 DEBUG 19:20:28 DEBUG } 19:20:28 DEBUG 19:20:28 DEBUG "pod": "ds-cts-2" 19:20:28 DEBUG ] 19:20:28 DEBUG 19:20:28 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:20:28 DEBUG ] 19:20:28 DEBUG 19:20:28 DEBUG }, 19:20:28 DEBUG } 19:20:28 DEBUG 19:20:28 DEBUG }, 19:20:28 DEBUG } 19:20:28 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:20:28 DEBUG "value": [ 19:20:28 DEBUG ] 19:20:28 DEBUG [...] 19:20:28 DEBUG "value": [ 19:20:28 DEBUG } 19:20:28 DEBUG }, 19:20:28 DEBUG 1713550828, 19:20:28 DEBUG } 19:20:28 INFO [loop_until]: OK (rc = 0) 19:20:28 DEBUG 19:20:28 DEBUG 1713550828, 19:20:28 DEBUG "value": [ 19:20:28 DEBUG "0.00033989999999999997" 19:20:28 DEBUG } 19:20:28 DEBUG --- stdout --- 19:20:28 DEBUG 19:20:28 DEBUG "0" 19:20:28 DEBUG 1713550828, 19:20:28 DEBUG ] 19:20:28 DEBUG 19:20:28 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 13982m 8841Mi am-767f6b67b5-qcqz2 14332m 8849Mi am-767f6b67b5-vbltz 15408m 8858Mi ds-cts-0 15088m 10833Mi ds-cts-1 13671m 11086Mi ds-cts-2 15760m 10828Mi ds-idrepo-0 6250m 3575Mi ds-idrepo-1 6229m 3467Mi ds-idrepo-2 6244m 3493Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 2m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7653m 3128Mi 19:20:28 DEBUG ] 19:20:28 DEBUG "3377508352" 19:20:28 DEBUG } 19:20:28 DEBUG "pod": "ds-cts-2" 19:20:28 DEBUG --- stderr --- 19:20:28 DEBUG } 19:20:28 DEBUG ] 19:20:28 DEBUG ] 19:20:28 DEBUG }, 19:20:28 DEBUG 19:20:28 DEBUG ] 19:20:28 DEBUG } 19:20:28 DEBUG } 19:20:28 DEBUG "value": [ 19:20:28 DEBUG } 19:20:28 DEBUG ] 19:20:28 DEBUG } 19:20:28 DEBUG 1713550828, 19:20:28 DEBUG } 19:20:28 DEBUG } 19:20:28 DEBUG "0" 19:20:28 DEBUG } 19:20:28 DEBUG ] 19:20:28 DEBUG } 19:20:28 DEBUG ] 19:20:28 DEBUG } 19:20:28 DEBUG } 19:20:28 INFO 19:20:28 INFO [loop_until]: kubectl --namespace=xlou top node 19:20:28 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:20:29 INFO [loop_until]: OK (rc = 0) 19:20:29 DEBUG --- stdout --- 19:20:29 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 74m 0% 6235Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 6425m 21% 7165Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 14118m 47% 11562Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 78m 0% 3352Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 15194m 50% 13879Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 179m 0% 5221Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 14355m 48% 11712Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12268m 41% 3002Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 16262m 54% 11076Mi 9% gke-xlou-cdm-ds-0f601751-j488 6931m 23% 6726Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 13702m 45% 15923Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 7048m 23% 7355Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 15589m 52% 13336Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 7888m 49% 6937Mi 11% 19:20:29 DEBUG --- stderr --- 19:20:29 DEBUG 19:21:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550888" 19:21:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713550888" 19:21:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550888" 19:21:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713550888" 19:21:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550888" 19:21:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713550888" 19:21:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550888" 19:21:28 INFO [http_cmd]: http status code OK 19:21:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550888" 19:21:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550888" 19:21:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550888" 19:21:28 INFO [http_cmd]: http status code OK 19:21:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550888" 19:21:28 INFO [http_cmd]: http status code OK 19:21:28 DEBUG --- status code --- 19:21:28 INFO [http_cmd]: http status code OK 19:21:28 DEBUG --- status code --- 19:21:28 INFO [http_cmd]: http status code OK 19:21:28 ERROR [http_cmd]: ERROR 19:21:28 DEBUG --- status code --- 19:21:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:28 DEBUG --- status code --- 19:21:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:28 DEBUG --- status code --- 19:21:28 ERROR --- status code --- 19:21:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:28 DEBUG --- http response --- 19:21:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:28 DEBUG --- http response --- 19:21:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:28 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:21:28 DEBUG --- http response --- 19:21:28 DEBUG --- http response --- 19:21:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:21:28 DEBUG --- http response --- 19:21:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:21:28 ERROR --- http response --- 19:21:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:21:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:21:28 DEBUG ----- output ----- 19:21:28 INFO [http_cmd]: http status code OK 19:21:28 INFO [http_cmd]: http status code OK 19:21:28 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:21:28 INFO [http_cmd]: http status code OK 19:21:28 DEBUG ----- output ----- 19:21:28 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:21:28 DEBUG ----- output ----- 19:21:28 DEBUG ----- output ----- 19:21:28 DEBUG { 19:21:28 DEBUG --- status code --- 19:21:28 DEBUG --- status code --- 19:21:28 DEBUG --- status code --- 19:21:28 DEBUG { 19:21:28 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:21:28 DEBUG { 19:21:28 DEBUG { 19:21:28 DEBUG "status": "success", 19:21:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:28 DEBUG "status": "success", 19:21:28 DEBUG "status": "success", 19:21:28 DEBUG "status": "success", 19:21:28 DEBUG "data": { 19:21:28 DEBUG --- http response --- 19:21:28 DEBUG --- http response --- 19:21:28 DEBUG --- http response --- 19:21:28 DEBUG "data": { 19:21:28 DEBUG "data": { 19:21:28 DEBUG "data": { 19:21:28 DEBUG "resultType": "vector", 19:21:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:21:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:21:28 DEBUG "resultType": "vector", 19:21:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:21:28 DEBUG "resultType": "vector", 19:21:28 DEBUG "resultType": "vector", 19:21:28 DEBUG "result": [ 19:21:28 DEBUG ----- output ----- 19:21:28 DEBUG ----- output ----- 19:21:28 DEBUG "result": [ 19:21:28 DEBUG ----- output ----- 19:21:28 DEBUG "result": [ 19:21:28 DEBUG "result": [ 19:21:28 DEBUG { 19:21:28 DEBUG { 19:21:28 DEBUG { 19:21:28 DEBUG { 19:21:28 DEBUG { 19:21:28 DEBUG { 19:21:28 DEBUG { 19:21:28 DEBUG "metric": { 19:21:28 DEBUG "status": "success", 19:21:28 DEBUG "status": "success", 19:21:28 DEBUG "metric": { 19:21:28 DEBUG "status": "success", 19:21:28 DEBUG "metric": { 19:21:28 DEBUG "metric": { 19:21:28 DEBUG "pod": "ds-idrepo-0" 19:21:28 DEBUG "data": { 19:21:28 DEBUG "data": { 19:21:28 DEBUG "pod": "ds-idrepo-0" 19:21:28 DEBUG "data": { 19:21:28 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:21:28 INFO [http_cmd]: http status code OK 19:21:28 DEBUG "pod": "ds-idrepo-0" 19:21:28 DEBUG }, 19:21:28 DEBUG "resultType": "vector", 19:21:28 DEBUG "resultType": "vector", 19:21:28 DEBUG }, 19:21:28 DEBUG "resultType": "vector", 19:21:28 DEBUG }, 19:21:28 DEBUG --- status code --- 19:21:28 DEBUG }, 19:21:28 DEBUG "value": [ 19:21:28 DEBUG "result": [ 19:21:28 DEBUG "result": [ 19:21:28 DEBUG "value": [ 19:21:28 DEBUG "result": [ 19:21:28 DEBUG "value": [ 19:21:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:28 DEBUG "value": [ 19:21:28 DEBUG 19:21:28 DEBUG { 19:21:28 DEBUG { 19:21:28 DEBUG 19:21:28 DEBUG { 19:21:28 DEBUG 19:21:28 DEBUG --- http response --- 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG "metric": { 19:21:28 DEBUG "metric": { 19:21:28 DEBUG 19:21:28 DEBUG "metric": { 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:21:28 DEBUG 19:21:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:21:28 DEBUG "pod": "ds-cts-0" 19:21:28 DEBUG 19:21:28 DEBUG "pod": "ds-idrepo-1" 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG ----- output ----- 19:21:28 DEBUG [...] 19:21:28 DEBUG }, 19:21:28 DEBUG }, 19:21:28 DEBUG [...] 19:21:28 DEBUG }, 19:21:28 DEBUG [...] 19:21:28 DEBUG [...] 19:21:28 DEBUG { 19:21:28 DEBUG 19:21:28 DEBUG "value": [ 19:21:28 DEBUG "value": [ 19:21:28 DEBUG 19:21:28 DEBUG "value": [ 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG "status": "success", 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG "data": { 19:21:28 INFO [http_cmd]: http status code OK 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG "resultType": "vector", 19:21:28 DEBUG --- status code --- 19:21:28 DEBUG "pod": "ds-idrepo-2" 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG "pod": "ds-idrepo-2" 19:21:28 DEBUG 19:21:28 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:21:28 DEBUG "pod": "ds-cts-2" 19:21:28 DEBUG "result": [ 19:21:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:28 DEBUG }, 19:21:28 DEBUG [...] 19:21:28 DEBUG [...] 19:21:28 DEBUG }, 19:21:28 DEBUG [...] 19:21:28 DEBUG }, 19:21:28 DEBUG }, 19:21:28 DEBUG { 19:21:28 DEBUG --- http response --- 19:21:28 DEBUG "value": [ 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG "value": [ 19:21:28 DEBUG 19:21:28 DEBUG "value": [ 19:21:28 DEBUG "value": [ 19:21:28 DEBUG "metric": { 19:21:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:21:28 DEBUG 1713550888, 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG 1713550888, 19:21:28 DEBUG 19:21:28 DEBUG 1713550888, 19:21:28 DEBUG 1713550888, 19:21:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:21:28 DEBUG ----- output ----- 19:21:28 DEBUG "0" 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG "0" 19:21:28 DEBUG 19:21:28 DEBUG "7.720354460458098" 19:21:28 DEBUG "0" 19:21:28 DEBUG }, 19:21:28 DEBUG { 19:21:28 DEBUG ] 19:21:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:21:28 DEBUG "pod": "ds-cts-2" 19:21:28 DEBUG ] 19:21:28 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:21:28 DEBUG ] 19:21:28 DEBUG ] 19:21:28 DEBUG "value": [ 19:21:28 DEBUG "status": "success", 19:21:28 DEBUG } 19:21:28 DEBUG }, 19:21:28 DEBUG }, 19:21:28 DEBUG } 19:21:28 DEBUG }, 19:21:28 DEBUG } 19:21:28 DEBUG } 19:21:28 DEBUG 19:21:28 DEBUG "data": { 19:21:28 DEBUG ] 19:21:28 DEBUG "value": [ 19:21:28 DEBUG "value": [ 19:21:28 DEBUG ] 19:21:28 DEBUG "value": [ 19:21:28 DEBUG ] 19:21:28 DEBUG ] 19:21:28 DEBUG 19:21:28 DEBUG "resultType": "vector", 19:21:28 DEBUG } 19:21:28 DEBUG 1713550888, 19:21:28 DEBUG 1713550888, 19:21:28 DEBUG } 19:21:28 DEBUG 1713550888, 19:21:28 DEBUG } 19:21:28 DEBUG } 19:21:28 DEBUG 19:21:28 DEBUG "result": [ 19:21:28 DEBUG } 19:21:28 DEBUG "2335.0333333333333" 19:21:28 DEBUG "0" 19:21:28 DEBUG } 19:21:28 DEBUG "0" 19:21:28 DEBUG } 19:21:28 DEBUG } 19:21:28 DEBUG [...] 19:21:28 DEBUG { 19:21:28 DEBUG ] 19:21:28 DEBUG ] 19:21:28 DEBUG ] 19:21:28 DEBUG 19:21:28 DEBUG "metric": { 19:21:28 DEBUG } 19:21:28 DEBUG } 19:21:28 DEBUG } 19:21:28 DEBUG 19:21:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:21:28 DEBUG ] 19:21:28 DEBUG ] 19:21:28 DEBUG ] 19:21:28 DEBUG 19:21:28 DEBUG }, 19:21:28 DEBUG } 19:21:28 DEBUG } 19:21:28 DEBUG } 19:21:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:21:28 DEBUG "value": [ 19:21:28 DEBUG } 19:21:28 DEBUG } 19:21:28 DEBUG } 19:21:28 DEBUG }, 19:21:28 DEBUG 19:21:28 DEBUG "value": [ 19:21:28 DEBUG 19:21:28 DEBUG 1713550888, 19:21:28 DEBUG 19:21:28 DEBUG "2324.6" 19:21:28 DEBUG [...] 19:21:28 DEBUG ] 19:21:28 DEBUG 19:21:28 DEBUG } 19:21:28 DEBUG 19:21:28 DEBUG ] 19:21:28 DEBUG 19:21:28 DEBUG } 19:21:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:21:28 DEBUG } 19:21:28 DEBUG }, 19:21:28 DEBUG "value": [ 19:21:28 DEBUG 1713550888, 19:21:28 DEBUG "0" 19:21:28 DEBUG ] 19:21:28 DEBUG } 19:21:28 DEBUG ] 19:21:28 DEBUG } 19:21:28 DEBUG } 19:21:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550888" 19:21:28 INFO [http_cmd]: http status code OK 19:21:28 DEBUG --- status code --- 19:21:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:28 DEBUG --- http response --- 19:21:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:21:28 DEBUG ----- output ----- 19:21:28 DEBUG { 19:21:28 DEBUG "status": "success", 19:21:28 DEBUG "data": { 19:21:28 DEBUG "resultType": "vector", 19:21:28 DEBUG "result": [ 19:21:28 DEBUG { 19:21:28 DEBUG "metric": { 19:21:28 DEBUG "instance": "10.142.0.107:9100" 19:21:28 DEBUG }, 19:21:28 DEBUG "value": [ 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG [...] 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG "instance": "10.142.0.43:9100" 19:21:28 DEBUG }, 19:21:28 DEBUG "value": [ 19:21:28 DEBUG 1713550888, 19:21:28 DEBUG "1866752642048" 19:21:28 DEBUG ] 19:21:28 DEBUG } 19:21:28 DEBUG ] 19:21:28 DEBUG } 19:21:28 DEBUG } 19:21:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550888" 19:21:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550888" 19:21:28 INFO [http_cmd]: http status code OK 19:21:28 DEBUG --- status code --- 19:21:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:28 DEBUG --- http response --- 19:21:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:21:28 DEBUG ----- output ----- 19:21:28 DEBUG { 19:21:28 DEBUG "status": "success", 19:21:28 DEBUG "data": { 19:21:28 DEBUG "resultType": "vector", 19:21:28 DEBUG "result": [ 19:21:28 DEBUG { 19:21:28 DEBUG "metric": { 19:21:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:21:28 DEBUG }, 19:21:28 DEBUG "value": [ 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG [...] 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:21:28 DEBUG }, 19:21:28 DEBUG "value": [ 19:21:28 DEBUG 1713550888, 19:21:28 DEBUG "420138" 19:21:28 DEBUG ] 19:21:28 DEBUG } 19:21:28 DEBUG ] 19:21:28 DEBUG } 19:21:28 DEBUG } 19:21:28 INFO [http_cmd]: http status code OK 19:21:28 DEBUG --- status code --- 19:21:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:28 DEBUG --- http response --- 19:21:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:21:28 DEBUG ----- output ----- 19:21:28 DEBUG { 19:21:28 DEBUG "status": "success", 19:21:28 DEBUG "data": { 19:21:28 DEBUG "resultType": "vector", 19:21:28 DEBUG "result": [ 19:21:28 DEBUG { 19:21:28 DEBUG "metric": { 19:21:28 DEBUG "instance": "10.142.0.107:9100" 19:21:28 DEBUG }, 19:21:28 DEBUG "value": [ 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG [...] 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG "instance": "10.142.0.43:9100" 19:21:28 DEBUG }, 19:21:28 DEBUG "value": [ 19:21:28 DEBUG 1713550888, 19:21:28 DEBUG "0.08800000000000335" 19:21:28 DEBUG ] 19:21:28 DEBUG } 19:21:28 DEBUG ] 19:21:28 DEBUG } 19:21:28 DEBUG } 19:21:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550888" 19:21:28 INFO [http_cmd]: http status code OK 19:21:28 DEBUG --- status code --- 19:21:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:28 DEBUG --- http response --- 19:21:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:21:28 DEBUG ----- output ----- 19:21:28 DEBUG { 19:21:28 DEBUG "status": "success", 19:21:28 DEBUG "data": { 19:21:28 DEBUG "resultType": "vector", 19:21:28 DEBUG "result": [ 19:21:28 DEBUG { 19:21:28 DEBUG "metric": { 19:21:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:21:28 DEBUG }, 19:21:28 DEBUG "value": [ 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG [...] 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:21:28 DEBUG }, 19:21:28 DEBUG "value": [ 19:21:28 DEBUG 1713550888, 19:21:28 DEBUG "2325.2666666666664" 19:21:28 DEBUG ] 19:21:28 DEBUG } 19:21:28 DEBUG ] 19:21:28 DEBUG } 19:21:28 DEBUG } 19:21:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550888" 19:21:28 INFO [http_cmd]: http status code OK 19:21:28 DEBUG --- status code --- 19:21:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:28 DEBUG --- http response --- 19:21:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:21:28 DEBUG ----- output ----- 19:21:28 DEBUG { 19:21:28 DEBUG "status": "success", 19:21:28 DEBUG "data": { 19:21:28 DEBUG "resultType": "vector", 19:21:28 DEBUG "result": [ 19:21:28 DEBUG { 19:21:28 DEBUG "metric": { 19:21:28 DEBUG "pod": "ds-idrepo-1" 19:21:28 DEBUG }, 19:21:28 DEBUG "value": [ 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG [...] 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:21:28 DEBUG }, 19:21:28 DEBUG "value": [ 19:21:28 DEBUG 1713550888, 19:21:28 DEBUG "0" 19:21:28 DEBUG ] 19:21:28 DEBUG } 19:21:28 DEBUG ] 19:21:28 DEBUG } 19:21:28 DEBUG } 19:21:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713550888" 19:21:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550888" 19:21:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550888" 19:21:28 INFO [http_cmd]: http status code OK 19:21:28 INFO [http_cmd]: http status code OK 19:21:28 DEBUG --- status code --- 19:21:28 DEBUG --- status code --- 19:21:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:28 DEBUG --- http response --- 19:21:28 DEBUG --- http response --- 19:21:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:21:28 DEBUG ----- output ----- 19:21:28 DEBUG { 19:21:28 DEBUG "status": "success", 19:21:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:21:28 DEBUG "data": { 19:21:28 DEBUG ----- output ----- 19:21:28 DEBUG "resultType": "vector", 19:21:28 DEBUG { 19:21:28 DEBUG "result": [ 19:21:28 DEBUG "status": "success", 19:21:28 DEBUG { 19:21:28 DEBUG "data": { 19:21:28 DEBUG "metric": { 19:21:28 DEBUG "resultType": "vector", 19:21:28 DEBUG "pod": "ds-idrepo-0" 19:21:28 DEBUG "result": [ 19:21:28 DEBUG }, 19:21:28 DEBUG { 19:21:28 DEBUG "value": [ 19:21:28 DEBUG "metric": { 19:21:28 DEBUG 19:21:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:21:28 DEBUG 19:21:28 DEBUG }, 19:21:28 DEBUG 19:21:28 DEBUG "value": [ 19:21:28 DEBUG [...] 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG [...] 19:21:28 DEBUG "pod": "ds-idrepo-2" 19:21:28 DEBUG 19:21:28 DEBUG }, 19:21:28 DEBUG "value": [ 19:21:28 DEBUG 19:21:28 DEBUG 1713550888, 19:21:28 DEBUG 19:21:28 DEBUG "0" 19:21:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:21:28 DEBUG ] 19:21:28 DEBUG }, 19:21:28 DEBUG } 19:21:28 DEBUG "value": [ 19:21:28 DEBUG ] 19:21:28 DEBUG 1713550888, 19:21:28 DEBUG } 19:21:28 DEBUG "0.003" 19:21:28 DEBUG } 19:21:28 DEBUG ] 19:21:28 DEBUG } 19:21:28 DEBUG ] 19:21:28 DEBUG } 19:21:28 DEBUG } 19:21:28 INFO [http_cmd]: http status code OK 19:21:28 DEBUG --- status code --- 19:21:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:28 DEBUG --- http response --- 19:21:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:21:28 DEBUG ----- output ----- 19:21:28 DEBUG { 19:21:28 DEBUG "status": "success", 19:21:28 DEBUG "data": { 19:21:28 DEBUG "resultType": "vector", 19:21:28 DEBUG "result": [ 19:21:28 DEBUG { 19:21:28 DEBUG "metric": { 19:21:28 DEBUG "instance": "10.142.0.107:9100" 19:21:28 DEBUG }, 19:21:28 DEBUG "value": [ 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG [...] 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG "instance": "10.142.0.43:9100" 19:21:28 DEBUG }, 19:21:28 DEBUG "value": [ 19:21:28 DEBUG 1713550888, 19:21:28 DEBUG "0.18556666666671712" 19:21:28 DEBUG ] 19:21:28 DEBUG } 19:21:28 DEBUG ] 19:21:28 DEBUG } 19:21:28 DEBUG } 19:21:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550888" 19:21:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550888" 19:21:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550888" 19:21:28 INFO [http_cmd]: http status code OK 19:21:28 DEBUG --- status code --- 19:21:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:28 DEBUG --- http response --- 19:21:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:21:28 DEBUG ----- output ----- 19:21:28 DEBUG { 19:21:28 DEBUG "status": "success", 19:21:28 DEBUG "data": { 19:21:28 DEBUG "resultType": "vector", 19:21:28 DEBUG "result": [ 19:21:28 DEBUG { 19:21:28 DEBUG "metric": { 19:21:28 DEBUG "pod": "ds-cts-0" 19:21:28 DEBUG }, 19:21:28 DEBUG "value": [ 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG [...] 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG "pod": "ds-cts-2" 19:21:28 DEBUG }, 19:21:28 DEBUG "value": [ 19:21:28 DEBUG 1713550888, 19:21:28 DEBUG "0" 19:21:28 DEBUG ] 19:21:28 DEBUG } 19:21:28 DEBUG ] 19:21:28 DEBUG } 19:21:28 DEBUG } 19:21:28 INFO [http_cmd]: http status code OK 19:21:28 DEBUG --- status code --- 19:21:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:28 DEBUG --- http response --- 19:21:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:21:28 DEBUG ----- output ----- 19:21:28 DEBUG { 19:21:28 DEBUG "status": "success", 19:21:28 DEBUG "data": { 19:21:28 DEBUG "resultType": "vector", 19:21:28 DEBUG "result": [ 19:21:28 DEBUG { 19:21:28 DEBUG "metric": { 19:21:28 DEBUG "instance": "10.142.0.107:9100" 19:21:28 DEBUG }, 19:21:28 DEBUG "value": [ 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG [...] 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG "instance": "10.142.0.43:9100" 19:21:28 DEBUG }, 19:21:28 DEBUG "value": [ 19:21:28 DEBUG 1713550888, 19:21:28 DEBUG "70770186.23333333" 19:21:28 DEBUG ] 19:21:28 DEBUG } 19:21:28 DEBUG ] 19:21:28 DEBUG } 19:21:28 DEBUG } 19:21:28 INFO [http_cmd]: http status code OK 19:21:28 DEBUG --- status code --- 19:21:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:28 DEBUG --- http response --- 19:21:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:21:28 DEBUG ----- output ----- 19:21:28 DEBUG { 19:21:28 DEBUG "status": "success", 19:21:28 DEBUG "data": { 19:21:28 DEBUG "resultType": "vector", 19:21:28 DEBUG "result": [ 19:21:28 DEBUG { 19:21:28 DEBUG "metric": { 19:21:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:21:28 DEBUG }, 19:21:28 DEBUG "value": [ 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG [...] 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:21:28 DEBUG }, 19:21:28 DEBUG "value": [ 19:21:28 DEBUG 1713550888, 19:21:28 DEBUG "2324.8" 19:21:28 DEBUG ] 19:21:28 DEBUG } 19:21:28 DEBUG ] 19:21:28 DEBUG } 19:21:28 DEBUG } 19:21:28 INFO 19:21:28 INFO [loop_until]: kubectl --namespace=xlou top pods 19:21:28 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:21:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550888" 19:21:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550888" 19:21:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550888" 19:21:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550888" 19:21:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713550888" 19:21:28 INFO [http_cmd]: http status code OK 19:21:28 INFO [http_cmd]: http status code OK 19:21:28 DEBUG --- status code --- 19:21:28 DEBUG --- status code --- 19:21:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:28 DEBUG --- http response --- 19:21:28 DEBUG --- http response --- 19:21:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:21:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:21:28 DEBUG ----- output ----- 19:21:28 DEBUG ----- output ----- 19:21:28 DEBUG { 19:21:28 DEBUG { 19:21:28 DEBUG "status": "success", 19:21:28 DEBUG "status": "success", 19:21:28 DEBUG "data": { 19:21:28 DEBUG "data": { 19:21:28 DEBUG "resultType": "vector", 19:21:28 DEBUG "resultType": "vector", 19:21:28 DEBUG "result": [ 19:21:28 DEBUG "result": [ 19:21:28 DEBUG { 19:21:28 DEBUG { 19:21:28 DEBUG "metric": { 19:21:28 DEBUG "metric": { 19:21:28 DEBUG "pod": "ds-idrepo-1" 19:21:28 DEBUG "instance": "10.142.0.107:9100" 19:21:28 DEBUG }, 19:21:28 DEBUG }, 19:21:28 DEBUG "value": [ 19:21:28 DEBUG "value": [ 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG [...] 19:21:28 INFO [http_cmd]: http status code OK 19:21:28 DEBUG [...] 19:21:28 INFO [http_cmd]: http status code OK 19:21:28 DEBUG 19:21:28 DEBUG --- status code --- 19:21:28 DEBUG 19:21:28 DEBUG --- status code --- 19:21:28 DEBUG 19:21:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:28 DEBUG 19:21:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:28 DEBUG 19:21:28 DEBUG --- http response --- 19:21:28 DEBUG 19:21:28 DEBUG --- http response --- 19:21:28 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:21:28 DEBUG "instance": "10.142.0.43:9100" 19:21:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:21:28 DEBUG }, 19:21:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:21:28 DEBUG }, 19:21:28 INFO [http_cmd]: http status code OK 19:21:28 DEBUG ----- output ----- 19:21:28 DEBUG "value": [ 19:21:28 DEBUG ----- output ----- 19:21:28 DEBUG "value": [ 19:21:28 DEBUG --- status code --- 19:21:28 DEBUG { 19:21:28 DEBUG 1713550888, 19:21:28 DEBUG { 19:21:28 DEBUG 1713550888, 19:21:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:28 DEBUG "status": "success", 19:21:28 DEBUG "0.000019698951477950743" 19:21:28 DEBUG "status": "success", 19:21:28 DEBUG "70767924.96666667" 19:21:28 DEBUG --- http response --- 19:21:28 DEBUG "data": { 19:21:28 DEBUG ] 19:21:28 DEBUG "data": { 19:21:28 DEBUG ] 19:21:28 DEBUG "resultType": "vector", 19:21:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:21:28 DEBUG } 19:21:28 DEBUG "resultType": "vector", 19:21:28 DEBUG } 19:21:28 DEBUG "result": [ 19:21:28 DEBUG ----- output ----- 19:21:28 DEBUG ] 19:21:28 DEBUG "result": [ 19:21:28 DEBUG ] 19:21:28 DEBUG { 19:21:28 DEBUG { 19:21:28 DEBUG } 19:21:28 DEBUG { 19:21:28 DEBUG } 19:21:28 DEBUG "metric": { 19:21:28 DEBUG "status": "success", 19:21:28 DEBUG } 19:21:28 DEBUG "metric": { 19:21:28 DEBUG } 19:21:28 DEBUG "pod": "ds-cts-0" 19:21:28 DEBUG "data": { 19:21:28 DEBUG "pod": "ds-idrepo-1" 19:21:28 DEBUG }, 19:21:28 DEBUG "resultType": "vector", 19:21:28 DEBUG }, 19:21:28 DEBUG "value": [ 19:21:28 DEBUG "result": [ 19:21:28 DEBUG "value": [ 19:21:28 DEBUG 19:21:28 DEBUG { 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG "metric": { 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:21:28 DEBUG 19:21:28 DEBUG [...] 19:21:28 DEBUG }, 19:21:28 DEBUG [...] 19:21:28 DEBUG 19:21:28 DEBUG "value": [ 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG 19:21:28 DEBUG "pod": "ds-cts-2" 19:21:28 DEBUG 19:21:28 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:21:28 DEBUG }, 19:21:28 DEBUG [...] 19:21:28 DEBUG }, 19:21:28 DEBUG "value": [ 19:21:28 DEBUG 19:21:28 DEBUG "value": [ 19:21:28 DEBUG 1713550888, 19:21:28 DEBUG 19:21:28 DEBUG 1713550888, 19:21:28 DEBUG "0.00017198666666666667" 19:21:28 DEBUG 19:21:28 DEBUG "0" 19:21:28 DEBUG ] 19:21:28 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:21:28 DEBUG ] 19:21:28 DEBUG } 19:21:28 DEBUG }, 19:21:28 DEBUG } 19:21:28 DEBUG ] 19:21:28 DEBUG "value": [ 19:21:28 DEBUG ] 19:21:28 DEBUG } 19:21:28 DEBUG 1713550888, 19:21:28 DEBUG } 19:21:28 DEBUG } 19:21:28 DEBUG "3379343360" 19:21:28 DEBUG } 19:21:28 DEBUG ] 19:21:28 DEBUG } 19:21:28 DEBUG ] 19:21:28 DEBUG } 19:21:28 DEBUG } 19:21:28 INFO [loop_until]: OK (rc = 0) 19:21:28 DEBUG --- stdout --- 19:21:28 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 14470m 8845Mi am-767f6b67b5-qcqz2 14137m 8852Mi am-767f6b67b5-vbltz 15456m 8860Mi ds-cts-0 15472m 10829Mi ds-cts-1 14030m 11087Mi ds-cts-2 15911m 10831Mi ds-idrepo-0 6351m 3590Mi ds-idrepo-1 6305m 3477Mi ds-idrepo-2 6332m 3492Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 4m 1217Mi lodemon-f8459c597-6tj6f 2m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7634m 3129Mi 19:21:28 DEBUG --- stderr --- 19:21:28 DEBUG 19:21:29 INFO 19:21:29 INFO [loop_until]: kubectl --namespace=xlou top node 19:21:29 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:21:29 INFO [loop_until]: OK (rc = 0) 19:21:29 DEBUG --- stdout --- 19:21:29 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 69m 0% 6236Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 6272m 21% 7166Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 14088m 47% 11554Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 87m 0% 3345Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 15296m 51% 13884Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 154m 0% 5193Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 14350m 48% 11732Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12251m 41% 3011Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 16182m 54% 11065Mi 9% gke-xlou-cdm-ds-0f601751-j488 6856m 22% 6731Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 13882m 46% 15920Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 6834m 22% 7368Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 15547m 52% 13352Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 7840m 49% 6938Mi 11% 19:21:29 DEBUG --- stderr --- 19:21:29 DEBUG 19:22:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550948" 19:22:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713550948" 19:22:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550948" 19:22:28 INFO [http_cmd]: http status code OK 19:22:28 DEBUG --- status code --- 19:22:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:28 DEBUG --- http response --- 19:22:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550948" 19:22:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:22:28 DEBUG ----- output ----- 19:22:28 DEBUG { 19:22:28 DEBUG "status": "success", 19:22:28 DEBUG "data": { 19:22:28 DEBUG "resultType": "vector", 19:22:28 DEBUG "result": [ 19:22:28 DEBUG { 19:22:28 DEBUG "metric": { 19:22:28 DEBUG "pod": "ds-idrepo-1" 19:22:28 DEBUG }, 19:22:28 DEBUG "value": [ 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG [...] 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:22:28 DEBUG }, 19:22:28 DEBUG "value": [ 19:22:28 DEBUG 1713550948, 19:22:28 DEBUG "0" 19:22:28 DEBUG ] 19:22:28 DEBUG } 19:22:28 DEBUG ] 19:22:28 DEBUG } 19:22:28 DEBUG } 19:22:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550948" 19:22:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713550948" 19:22:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550948" 19:22:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550948" 19:22:28 INFO [http_cmd]: http status code OK 19:22:28 DEBUG --- status code --- 19:22:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:28 DEBUG --- http response --- 19:22:28 INFO [http_cmd]: http status code OK 19:22:28 INFO [http_cmd]: http status code OK 19:22:28 DEBUG --- status code --- 19:22:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:22:28 DEBUG --- status code --- 19:22:28 INFO [http_cmd]: http status code OK 19:22:28 INFO [http_cmd]: http status code OK 19:22:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:28 DEBUG ----- output ----- 19:22:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:28 DEBUG --- status code --- 19:22:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713550948" 19:22:28 DEBUG --- status code --- 19:22:28 DEBUG --- http response --- 19:22:28 DEBUG { 19:22:28 DEBUG --- http response --- 19:22:28 INFO [http_cmd]: http status code OK 19:22:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550948" 19:22:28 DEBUG "status": "success", 19:22:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:22:28 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:22:28 DEBUG --- status code --- 19:22:28 DEBUG --- http response --- 19:22:28 DEBUG --- http response --- 19:22:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550948" 19:22:28 ERROR [http_cmd]: ERROR 19:22:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550948" 19:22:28 DEBUG "data": { 19:22:28 DEBUG ----- output ----- 19:22:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:22:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:22:28 ERROR --- status code --- 19:22:28 DEBUG "resultType": "vector", 19:22:28 DEBUG { 19:22:28 DEBUG --- http response --- 19:22:28 DEBUG ----- output ----- 19:22:28 DEBUG ----- output ----- 19:22:28 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:22:28 DEBUG "result": [ 19:22:28 DEBUG "status": "success", 19:22:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:22:28 DEBUG { 19:22:28 DEBUG { 19:22:28 ERROR --- http response --- 19:22:28 DEBUG { 19:22:28 DEBUG "data": { 19:22:28 DEBUG ----- output ----- 19:22:28 DEBUG "status": "success", 19:22:28 DEBUG "status": "success", 19:22:28 DEBUG "metric": { 19:22:28 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:22:28 DEBUG "resultType": "vector", 19:22:28 DEBUG { 19:22:28 DEBUG "data": { 19:22:28 DEBUG "data": { 19:22:28 INFO [http_cmd]: http status code OK 19:22:28 INFO [http_cmd]: http status code OK 19:22:28 DEBUG "pod": "ds-idrepo-0" 19:22:28 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:22:28 DEBUG "result": [ 19:22:28 DEBUG "status": "success", 19:22:28 DEBUG "resultType": "vector", 19:22:28 DEBUG "resultType": "vector", 19:22:28 DEBUG --- status code --- 19:22:28 DEBUG --- status code --- 19:22:28 DEBUG }, 19:22:28 DEBUG { 19:22:28 DEBUG "data": { 19:22:28 DEBUG "result": [ 19:22:28 DEBUG "result": [ 19:22:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:28 DEBUG "value": [ 19:22:28 DEBUG "metric": { 19:22:28 DEBUG "resultType": "vector", 19:22:28 DEBUG { 19:22:28 DEBUG { 19:22:28 DEBUG --- http response --- 19:22:28 DEBUG --- http response --- 19:22:28 DEBUG 19:22:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:22:28 DEBUG "result": [ 19:22:28 DEBUG "metric": { 19:22:28 DEBUG "metric": { 19:22:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:22:28 DEBUG 19:22:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:22:28 DEBUG }, 19:22:28 DEBUG { 19:22:28 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:22:28 DEBUG "pod": "ds-cts-0" 19:22:28 DEBUG ----- output ----- 19:22:28 DEBUG 19:22:28 DEBUG ----- output ----- 19:22:28 DEBUG "value": [ 19:22:28 DEBUG "metric": { 19:22:28 DEBUG }, 19:22:28 DEBUG }, 19:22:28 DEBUG { 19:22:28 DEBUG [...] 19:22:28 DEBUG { 19:22:28 DEBUG 19:22:28 DEBUG "pod": "ds-idrepo-0" 19:22:28 DEBUG "value": [ 19:22:28 DEBUG "value": [ 19:22:28 DEBUG "status": "success", 19:22:28 DEBUG 19:22:28 DEBUG "status": "success", 19:22:28 DEBUG 19:22:28 DEBUG }, 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG "data": { 19:22:28 DEBUG 19:22:28 DEBUG "data": { 19:22:28 DEBUG 19:22:28 DEBUG "value": [ 19:22:28 DEBUG 19:22:28 INFO [http_cmd]: http status code OK 19:22:28 DEBUG 19:22:28 INFO [http_cmd]: http status code OK 19:22:28 DEBUG "resultType": "vector", 19:22:28 DEBUG 19:22:28 DEBUG "resultType": "vector", 19:22:28 DEBUG [...] 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG --- status code --- 19:22:28 DEBUG 19:22:28 DEBUG --- status code --- 19:22:28 DEBUG "result": [ 19:22:28 DEBUG "pod": "ds-idrepo-2" 19:22:28 DEBUG "result": [ 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG [...] 19:22:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:28 DEBUG [...] 19:22:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:28 DEBUG { 19:22:28 DEBUG }, 19:22:28 DEBUG { 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG --- http response --- 19:22:28 DEBUG 19:22:28 DEBUG --- http response --- 19:22:28 DEBUG "metric": { 19:22:28 DEBUG "pod": "ds-idrepo-0" 19:22:28 DEBUG "metric": { 19:22:28 DEBUG 19:22:28 DEBUG [...] 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:22:28 DEBUG }, 19:22:28 DEBUG "value": [ 19:22:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:22:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:22:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG ----- output ----- 19:22:28 DEBUG "value": [ 19:22:28 DEBUG 1713550948, 19:22:28 DEBUG ----- output ----- 19:22:28 DEBUG }, 19:22:28 DEBUG }, 19:22:28 DEBUG 19:22:28 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:22:28 DEBUG "pod": "ds-cts-2" 19:22:28 DEBUG { 19:22:28 DEBUG 19:22:28 DEBUG "0" 19:22:28 DEBUG { 19:22:28 DEBUG "value": [ 19:22:28 DEBUG "value": [ 19:22:28 DEBUG 19:22:28 DEBUG }, 19:22:28 DEBUG }, 19:22:28 DEBUG "status": "success", 19:22:28 DEBUG 19:22:28 DEBUG ] 19:22:28 DEBUG "status": "success", 19:22:28 DEBUG 19:22:28 DEBUG 1713550948, 19:22:28 DEBUG "pod": "ds-idrepo-2" 19:22:28 DEBUG "value": [ 19:22:28 DEBUG "value": [ 19:22:28 DEBUG "data": { 19:22:28 DEBUG 19:22:28 DEBUG } 19:22:28 DEBUG "data": { 19:22:28 DEBUG 19:22:28 DEBUG "0" 19:22:28 DEBUG }, 19:22:28 DEBUG 1713550948, 19:22:28 DEBUG 1713550948, 19:22:28 DEBUG "resultType": "vector", 19:22:28 DEBUG [...] 19:22:28 DEBUG ] 19:22:28 DEBUG "resultType": "vector", 19:22:28 DEBUG 19:22:28 DEBUG ] 19:22:28 DEBUG "value": [ 19:22:28 DEBUG "7.69675304764289" 19:22:28 DEBUG "0" 19:22:28 DEBUG "result": [ 19:22:28 DEBUG 19:22:28 DEBUG } 19:22:28 DEBUG "result": [ 19:22:28 DEBUG [...] 19:22:28 DEBUG } 19:22:28 DEBUG 1713550948, 19:22:28 DEBUG ] 19:22:28 DEBUG ] 19:22:28 DEBUG { 19:22:28 DEBUG 19:22:28 DEBUG } 19:22:28 DEBUG { 19:22:28 DEBUG 19:22:28 DEBUG ] 19:22:28 DEBUG "0" 19:22:28 DEBUG } 19:22:28 DEBUG } 19:22:28 DEBUG "metric": { 19:22:28 DEBUG 19:22:28 DEBUG "metric": { 19:22:28 DEBUG 19:22:28 DEBUG } 19:22:28 DEBUG ] 19:22:28 DEBUG ] 19:22:28 DEBUG ] 19:22:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:22:28 DEBUG "pod": "ds-cts-2" 19:22:28 DEBUG "instance": "10.142.0.107:9100" 19:22:28 DEBUG 19:22:28 DEBUG } 19:22:28 DEBUG } 19:22:28 DEBUG } 19:22:28 DEBUG } 19:22:28 DEBUG }, 19:22:28 DEBUG }, 19:22:28 DEBUG }, 19:22:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:22:28 DEBUG ] 19:22:28 DEBUG } 19:22:28 DEBUG } 19:22:28 DEBUG "value": [ 19:22:28 DEBUG "value": [ 19:22:28 DEBUG "value": [ 19:22:28 DEBUG }, 19:22:28 DEBUG } 19:22:28 DEBUG 19:22:28 DEBUG 1713550948, 19:22:28 DEBUG 19:22:28 DEBUG "value": [ 19:22:28 DEBUG } 19:22:28 DEBUG 19:22:28 DEBUG "0" 19:22:28 DEBUG 19:22:28 DEBUG 1713550948, 19:22:28 DEBUG 19:22:28 DEBUG ] 19:22:28 DEBUG 19:22:28 DEBUG "2331.8333333333335" 19:22:28 DEBUG [...] 19:22:28 DEBUG } 19:22:28 DEBUG [...] 19:22:28 DEBUG ] 19:22:28 DEBUG } 19:22:28 DEBUG ] 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG ] 19:22:28 DEBUG } 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG } 19:22:28 DEBUG } 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG } 19:22:28 DEBUG "instance": "10.142.0.43:9100" 19:22:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:22:28 DEBUG }, 19:22:28 DEBUG }, 19:22:28 DEBUG "value": [ 19:22:28 DEBUG "value": [ 19:22:28 DEBUG 1713550948, 19:22:28 DEBUG 1713550948, 19:22:28 DEBUG "1870761630720" 19:22:28 DEBUG "2325.0333333333333" 19:22:28 DEBUG ] 19:22:28 DEBUG ] 19:22:28 DEBUG } 19:22:28 DEBUG } 19:22:28 DEBUG ] 19:22:28 DEBUG ] 19:22:28 DEBUG } 19:22:28 DEBUG } 19:22:28 DEBUG } 19:22:28 DEBUG } 19:22:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550948" 19:22:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550948" 19:22:28 INFO [http_cmd]: http status code OK 19:22:28 DEBUG --- status code --- 19:22:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:28 DEBUG --- http response --- 19:22:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:22:28 DEBUG ----- output ----- 19:22:28 DEBUG { 19:22:28 DEBUG "status": "success", 19:22:28 DEBUG "data": { 19:22:28 DEBUG "resultType": "vector", 19:22:28 DEBUG "result": [ 19:22:28 DEBUG { 19:22:28 DEBUG "metric": { 19:22:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:22:28 DEBUG }, 19:22:28 DEBUG "value": [ 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG [...] 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:22:28 DEBUG }, 19:22:28 DEBUG "value": [ 19:22:28 DEBUG 1713550948, 19:22:28 DEBUG "419923" 19:22:28 DEBUG ] 19:22:28 DEBUG } 19:22:28 DEBUG ] 19:22:28 DEBUG } 19:22:28 DEBUG } 19:22:28 INFO [http_cmd]: http status code OK 19:22:28 DEBUG --- status code --- 19:22:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:28 DEBUG --- http response --- 19:22:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:22:28 DEBUG ----- output ----- 19:22:28 DEBUG { 19:22:28 DEBUG "status": "success", 19:22:28 DEBUG "data": { 19:22:28 DEBUG "resultType": "vector", 19:22:28 DEBUG "result": [ 19:22:28 DEBUG { 19:22:28 DEBUG "metric": { 19:22:28 DEBUG "instance": "10.142.0.107:9100" 19:22:28 DEBUG }, 19:22:28 DEBUG "value": [ 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG [...] 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG "instance": "10.142.0.43:9100" 19:22:28 DEBUG }, 19:22:28 DEBUG "value": [ 19:22:28 DEBUG 1713550948, 19:22:28 DEBUG "0.08366666666666633" 19:22:28 DEBUG ] 19:22:28 DEBUG } 19:22:28 DEBUG ] 19:22:28 DEBUG } 19:22:28 DEBUG } 19:22:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550948" 19:22:28 INFO [http_cmd]: http status code OK 19:22:28 DEBUG --- status code --- 19:22:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:28 DEBUG --- http response --- 19:22:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:22:28 DEBUG ----- output ----- 19:22:28 DEBUG { 19:22:28 DEBUG "status": "success", 19:22:28 DEBUG "data": { 19:22:28 DEBUG "resultType": "vector", 19:22:28 DEBUG "result": [ 19:22:28 DEBUG { 19:22:28 DEBUG "metric": { 19:22:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:22:28 DEBUG }, 19:22:28 DEBUG "value": [ 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG [...] 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:22:28 DEBUG }, 19:22:28 DEBUG "value": [ 19:22:28 DEBUG 1713550948, 19:22:28 DEBUG "2324.366666666667" 19:22:28 DEBUG ] 19:22:28 DEBUG } 19:22:28 DEBUG ] 19:22:28 DEBUG } 19:22:28 DEBUG } 19:22:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550948" 19:22:28 INFO [http_cmd]: http status code OK 19:22:28 DEBUG --- status code --- 19:22:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:28 DEBUG --- http response --- 19:22:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:22:28 DEBUG ----- output ----- 19:22:28 DEBUG { 19:22:28 DEBUG "status": "success", 19:22:28 DEBUG "data": { 19:22:28 DEBUG "resultType": "vector", 19:22:28 DEBUG "result": [ 19:22:28 DEBUG { 19:22:28 DEBUG "metric": { 19:22:28 DEBUG "pod": "ds-idrepo-1" 19:22:28 DEBUG }, 19:22:28 DEBUG "value": [ 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG [...] 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:22:28 DEBUG }, 19:22:28 DEBUG "value": [ 19:22:28 DEBUG 1713550948, 19:22:28 DEBUG "0" 19:22:28 DEBUG ] 19:22:28 DEBUG } 19:22:28 DEBUG ] 19:22:28 DEBUG } 19:22:28 DEBUG } 19:22:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550948" 19:22:28 INFO [http_cmd]: http status code OK 19:22:28 DEBUG --- status code --- 19:22:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:28 DEBUG --- http response --- 19:22:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:22:28 DEBUG ----- output ----- 19:22:28 DEBUG { 19:22:28 DEBUG "status": "success", 19:22:28 DEBUG "data": { 19:22:28 DEBUG "resultType": "vector", 19:22:28 DEBUG "result": [ 19:22:28 DEBUG { 19:22:28 DEBUG "metric": { 19:22:28 DEBUG "pod": "ds-idrepo-0" 19:22:28 DEBUG }, 19:22:28 DEBUG "value": [ 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG [...] 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG "pod": "ds-idrepo-2" 19:22:28 DEBUG }, 19:22:28 DEBUG "value": [ 19:22:28 DEBUG 1713550948, 19:22:28 DEBUG "0" 19:22:28 DEBUG ] 19:22:28 DEBUG } 19:22:28 DEBUG ] 19:22:28 DEBUG } 19:22:28 DEBUG } 19:22:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713550948" 19:22:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550948" 19:22:28 INFO [http_cmd]: http status code OK 19:22:28 DEBUG --- status code --- 19:22:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:28 DEBUG --- http response --- 19:22:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:22:28 DEBUG ----- output ----- 19:22:28 DEBUG { 19:22:28 DEBUG "status": "success", 19:22:28 DEBUG "data": { 19:22:28 DEBUG "resultType": "vector", 19:22:28 DEBUG "result": [ 19:22:28 DEBUG { 19:22:28 DEBUG "metric": { 19:22:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:22:28 DEBUG }, 19:22:28 DEBUG "value": [ 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG [...] 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:22:28 DEBUG }, 19:22:28 DEBUG "value": [ 19:22:28 DEBUG 1713550948, 19:22:28 DEBUG "0.004" 19:22:28 DEBUG ] 19:22:28 DEBUG } 19:22:28 DEBUG ] 19:22:28 DEBUG } 19:22:28 DEBUG } 19:22:28 INFO [http_cmd]: http status code OK 19:22:28 DEBUG --- status code --- 19:22:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:28 DEBUG --- http response --- 19:22:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:22:28 DEBUG ----- output ----- 19:22:28 DEBUG { 19:22:28 DEBUG "status": "success", 19:22:28 DEBUG "data": { 19:22:28 DEBUG "resultType": "vector", 19:22:28 DEBUG "result": [ 19:22:28 DEBUG { 19:22:28 DEBUG "metric": { 19:22:28 DEBUG "instance": "10.142.0.107:9100" 19:22:28 DEBUG }, 19:22:28 DEBUG "value": [ 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG [...] 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG "instance": "10.142.0.43:9100" 19:22:28 DEBUG }, 19:22:28 DEBUG "value": [ 19:22:28 DEBUG 1713550948, 19:22:28 DEBUG "0.17733333333337667" 19:22:28 DEBUG ] 19:22:28 DEBUG } 19:22:28 DEBUG ] 19:22:28 DEBUG } 19:22:28 DEBUG } 19:22:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550948" 19:22:28 INFO [http_cmd]: http status code OK 19:22:28 DEBUG --- status code --- 19:22:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:28 DEBUG --- http response --- 19:22:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:22:28 DEBUG ----- output ----- 19:22:28 DEBUG { 19:22:28 DEBUG "status": "success", 19:22:28 DEBUG "data": { 19:22:28 DEBUG "resultType": "vector", 19:22:28 DEBUG "result": [ 19:22:28 DEBUG { 19:22:28 DEBUG "metric": { 19:22:28 DEBUG "pod": "ds-cts-0" 19:22:28 DEBUG }, 19:22:28 DEBUG "value": [ 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG [...] 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG "pod": "ds-cts-2" 19:22:28 DEBUG }, 19:22:28 DEBUG "value": [ 19:22:28 DEBUG 1713550948, 19:22:28 DEBUG "0" 19:22:28 DEBUG ] 19:22:28 DEBUG } 19:22:28 DEBUG ] 19:22:28 DEBUG } 19:22:28 DEBUG } 19:22:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550948" 19:22:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550948" 19:22:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550948" 19:22:28 INFO [http_cmd]: http status code OK 19:22:28 DEBUG --- status code --- 19:22:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:28 DEBUG --- http response --- 19:22:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:22:28 DEBUG ----- output ----- 19:22:28 DEBUG { 19:22:28 DEBUG "status": "success", 19:22:28 DEBUG "data": { 19:22:28 DEBUG "resultType": "vector", 19:22:28 DEBUG "result": [ 19:22:28 DEBUG { 19:22:28 DEBUG "metric": { 19:22:28 DEBUG "pod": "ds-idrepo-1" 19:22:28 DEBUG }, 19:22:28 DEBUG "value": [ 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG [...] 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 INFO [http_cmd]: http status code OK 19:22:28 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:22:28 DEBUG --- status code --- 19:22:28 DEBUG }, 19:22:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:28 DEBUG "value": [ 19:22:28 DEBUG --- http response --- 19:22:28 DEBUG 1713550948, 19:22:28 DEBUG "0" 19:22:28 DEBUG ] 19:22:28 DEBUG } 19:22:28 DEBUG ] 19:22:28 DEBUG } 19:22:28 DEBUG } 19:22:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:22:28 DEBUG ----- output ----- 19:22:28 DEBUG { 19:22:28 DEBUG "status": "success", 19:22:28 DEBUG "data": { 19:22:28 DEBUG "resultType": "vector", 19:22:28 DEBUG "result": [ 19:22:28 DEBUG { 19:22:28 DEBUG "metric": { 19:22:28 DEBUG "instance": "10.142.0.107:9100" 19:22:28 DEBUG }, 19:22:28 DEBUG "value": [ 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG [...] 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG "instance": "10.142.0.43:9100" 19:22:28 DEBUG }, 19:22:28 DEBUG "value": [ 19:22:28 DEBUG 1713550948, 19:22:28 DEBUG "70471207.36666667" 19:22:28 DEBUG ] 19:22:28 DEBUG } 19:22:28 DEBUG ] 19:22:28 DEBUG } 19:22:28 DEBUG } 19:22:28 INFO [http_cmd]: http status code OK 19:22:28 DEBUG --- status code --- 19:22:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:28 DEBUG --- http response --- 19:22:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:22:28 DEBUG ----- output ----- 19:22:28 DEBUG { 19:22:28 DEBUG "status": "success", 19:22:28 DEBUG "data": { 19:22:28 DEBUG "resultType": "vector", 19:22:28 DEBUG "result": [ 19:22:28 DEBUG { 19:22:28 DEBUG "metric": { 19:22:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:22:28 DEBUG }, 19:22:28 DEBUG "value": [ 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG [...] 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG 19:22:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:22:28 DEBUG }, 19:22:28 DEBUG "value": [ 19:22:28 DEBUG 1713550948, 19:22:28 DEBUG "2324.1666666666665" 19:22:28 DEBUG ] 19:22:28 DEBUG } 19:22:28 DEBUG ] 19:22:28 DEBUG } 19:22:28 DEBUG } 19:22:28 INFO 19:22:28 INFO [loop_until]: kubectl --namespace=xlou top pods 19:22:28 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:22:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550948" 19:22:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713550948" 19:22:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713550948" 19:22:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713550948" 19:22:28 INFO [http_cmd]: http status code OK 19:22:28 DEBUG --- status code --- 19:22:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:28 DEBUG --- http response --- 19:22:28 INFO [http_cmd]: http status code OK 19:22:28 INFO [http_cmd]: http status code OK 19:22:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:22:28 DEBUG --- status code --- 19:22:28 DEBUG --- status code --- 19:22:28 DEBUG ----- output ----- 19:22:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:28 DEBUG { 19:22:28 DEBUG --- http response --- 19:22:28 DEBUG --- http response --- 19:22:28 DEBUG "status": "success", 19:22:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:22:28 DEBUG "data": { 19:22:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:22:28 DEBUG ----- output ----- 19:22:28 DEBUG "resultType": "vector", 19:22:28 DEBUG ----- output ----- 19:22:28 DEBUG { 19:22:28 DEBUG "result": [ 19:22:28 DEBUG { 19:22:28 DEBUG "status": "success", 19:22:28 DEBUG { 19:22:28 DEBUG "status": "success", 19:22:28 DEBUG "data": { 19:22:28 DEBUG "metric": { 19:22:28 DEBUG "data": { 19:22:28 DEBUG "resultType": "vector", 19:22:28 DEBUG "pod": "ds-idrepo-1" 19:22:28 DEBUG "resultType": "vector", 19:22:28 DEBUG "result": [ 19:22:28 DEBUG }, 19:22:28 DEBUG "result": [ 19:22:28 DEBUG { 19:22:28 DEBUG "value": [ 19:22:28 DEBUG { 19:22:28 DEBUG "metric": { 19:22:28 DEBUG 19:22:28 DEBUG "metric": { 19:22:28 DEBUG "pod": "ds-cts-0" 19:22:28 DEBUG 19:22:28 DEBUG "instance": "10.142.0.107:9100" 19:22:28 DEBUG }, 19:22:28 DEBUG 19:22:28 DEBUG }, 19:22:29 DEBUG "value": [ 19:22:29 DEBUG [...] 19:22:29 DEBUG "value": [ 19:22:29 DEBUG 19:22:29 DEBUG 19:22:29 DEBUG 19:22:29 DEBUG 19:22:29 DEBUG 19:22:29 DEBUG 19:22:29 DEBUG 19:22:29 DEBUG 19:22:29 DEBUG 19:22:29 DEBUG [...] 19:22:29 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:22:29 DEBUG [...] 19:22:29 DEBUG 19:22:29 DEBUG }, 19:22:29 DEBUG 19:22:29 DEBUG 19:22:29 DEBUG "value": [ 19:22:29 DEBUG 19:22:29 INFO [http_cmd]: http status code OK 19:22:29 DEBUG 19:22:29 DEBUG 1713550948, 19:22:29 DEBUG 19:22:29 DEBUG --- status code --- 19:22:29 DEBUG "pod": "ds-cts-2" 19:22:29 DEBUG "0.000008028827037765083" 19:22:29 DEBUG "instance": "10.142.0.43:9100" 19:22:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:29 DEBUG }, 19:22:29 DEBUG ] 19:22:29 DEBUG }, 19:22:29 DEBUG --- http response --- 19:22:29 DEBUG "value": [ 19:22:29 DEBUG } 19:22:29 DEBUG "value": [ 19:22:29 DEBUG 1713550948, 19:22:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:22:29 DEBUG ] 19:22:29 DEBUG 1713550948, 19:22:29 DEBUG "0" 19:22:29 DEBUG ----- output ----- 19:22:29 DEBUG } 19:22:29 DEBUG "70469256.3" 19:22:29 DEBUG ] 19:22:29 DEBUG { 19:22:29 DEBUG } 19:22:29 DEBUG ] 19:22:29 DEBUG } 19:22:29 DEBUG "status": "success", 19:22:29 DEBUG } 19:22:29 DEBUG ] 19:22:29 DEBUG "data": { 19:22:29 DEBUG ] 19:22:29 DEBUG } 19:22:29 DEBUG "resultType": "vector", 19:22:29 DEBUG } 19:22:29 DEBUG } 19:22:29 DEBUG "result": [ 19:22:29 DEBUG } 19:22:29 DEBUG { 19:22:29 DEBUG "metric": { 19:22:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:22:29 DEBUG }, 19:22:29 DEBUG "value": [ 19:22:29 DEBUG 19:22:29 DEBUG 19:22:29 DEBUG 19:22:29 DEBUG [...] 19:22:29 DEBUG 19:22:29 DEBUG 19:22:29 DEBUG 19:22:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:22:29 DEBUG }, 19:22:29 DEBUG "value": [ 19:22:29 DEBUG 1713550948, 19:22:29 DEBUG "3382079488" 19:22:29 DEBUG ] 19:22:29 DEBUG } 19:22:29 DEBUG ] 19:22:29 DEBUG } 19:22:29 DEBUG } 19:22:29 INFO [loop_until]: OK (rc = 0) 19:22:29 DEBUG --- stdout --- 19:22:29 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 14237m 8846Mi am-767f6b67b5-qcqz2 14375m 8853Mi am-767f6b67b5-vbltz 15635m 8966Mi ds-cts-0 15699m 10835Mi ds-cts-1 14159m 11095Mi ds-cts-2 16126m 10836Mi ds-idrepo-0 6428m 3618Mi ds-idrepo-1 6373m 3478Mi ds-idrepo-2 6324m 3492Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 2m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7632m 3131Mi 19:22:29 DEBUG --- stderr --- 19:22:29 DEBUG 19:22:29 INFO 19:22:29 INFO [loop_until]: kubectl --namespace=xlou top node 19:22:29 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:22:29 INFO [loop_until]: OK (rc = 0) 19:22:29 DEBUG --- stdout --- 19:22:29 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 73m 0% 6252Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 6499m 21% 7201Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 14115m 47% 11552Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 3353Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 15615m 52% 13883Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 151m 0% 5200Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 14450m 48% 11767Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12337m 41% 3014Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 16246m 54% 11094Mi 9% gke-xlou-cdm-ds-0f601751-j488 6869m 23% 6726Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 14132m 47% 15917Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 6880m 23% 7367Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 15896m 53% 13340Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 7852m 49% 6939Mi 11% 19:22:29 DEBUG --- stderr --- 19:22:29 DEBUG 19:23:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551008" 19:23:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551008" 19:23:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713551008" 19:23:28 INFO [http_cmd]: http status code OK 19:23:28 DEBUG --- status code --- 19:23:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:28 DEBUG --- http response --- 19:23:28 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:23:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713551008" 19:23:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713551008" 19:23:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551008" 19:23:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551008" 19:23:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551008" 19:23:28 INFO [http_cmd]: http status code OK 19:23:28 DEBUG --- status code --- 19:23:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:28 DEBUG --- http response --- 19:23:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:23:28 DEBUG ----- output ----- 19:23:28 DEBUG { 19:23:28 DEBUG "status": "success", 19:23:28 DEBUG "data": { 19:23:28 DEBUG "resultType": "vector", 19:23:28 DEBUG "result": [ 19:23:28 DEBUG { 19:23:28 DEBUG "metric": { 19:23:28 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:23:28 DEBUG }, 19:23:28 DEBUG "value": [ 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG [...] 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG "pod": "ds-idrepo-2" 19:23:28 DEBUG }, 19:23:28 DEBUG "value": [ 19:23:28 DEBUG 1713551008, 19:23:28 DEBUG "3867.653724127511" 19:23:28 DEBUG ] 19:23:28 DEBUG } 19:23:28 DEBUG ] 19:23:28 DEBUG } 19:23:28 DEBUG } 19:23:28 INFO [http_cmd]: http status code OK 19:23:28 ERROR [http_cmd]: ERROR 19:23:28 DEBUG --- status code --- 19:23:28 ERROR --- status code --- 19:23:28 INFO [http_cmd]: http status code OK 19:23:28 INFO [http_cmd]: http status code OK 19:23:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:28 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:23:28 DEBUG --- status code --- 19:23:28 DEBUG --- status code --- 19:23:28 DEBUG --- http response --- 19:23:28 ERROR --- http response --- 19:23:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:28 INFO [http_cmd]: http status code OK 19:23:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:23:28 DEBUG --- http response --- 19:23:28 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:23:28 DEBUG --- status code --- 19:23:28 DEBUG --- http response --- 19:23:28 DEBUG ----- output ----- 19:23:28 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:23:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:23:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:28 DEBUG { 19:23:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:23:28 DEBUG ----- output ----- 19:23:28 DEBUG --- http response --- 19:23:28 DEBUG "status": "success", 19:23:28 DEBUG ----- output ----- 19:23:28 DEBUG { 19:23:28 DEBUG "data": { 19:23:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:23:28 DEBUG { 19:23:28 DEBUG "status": "success", 19:23:28 DEBUG "resultType": "vector", 19:23:28 DEBUG ----- output ----- 19:23:28 DEBUG "status": "success", 19:23:28 DEBUG "data": { 19:23:28 DEBUG "result": [ 19:23:28 DEBUG { 19:23:28 DEBUG "data": { 19:23:28 DEBUG "resultType": "vector", 19:23:28 DEBUG { 19:23:28 DEBUG "status": "success", 19:23:28 DEBUG "resultType": "vector", 19:23:28 DEBUG "result": [ 19:23:28 DEBUG "metric": { 19:23:28 DEBUG "data": { 19:23:28 DEBUG "result": [ 19:23:28 DEBUG { 19:23:28 DEBUG "pod": "ds-idrepo-0" 19:23:28 DEBUG "resultType": "vector", 19:23:28 DEBUG { 19:23:28 DEBUG "metric": { 19:23:28 DEBUG }, 19:23:28 DEBUG "result": [ 19:23:28 DEBUG "metric": { 19:23:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:23:28 DEBUG "value": [ 19:23:28 DEBUG { 19:23:28 DEBUG "pod": "ds-cts-0" 19:23:28 DEBUG }, 19:23:28 DEBUG 19:23:28 DEBUG "metric": { 19:23:28 DEBUG }, 19:23:28 DEBUG "value": [ 19:23:28 DEBUG 19:23:28 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:23:28 DEBUG "value": [ 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG }, 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG [...] 19:23:28 DEBUG "value": [ 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG [...] 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG [...] 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG "pod": "ds-cts-2" 19:23:28 DEBUG [...] 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG }, 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:23:28 DEBUG "value": [ 19:23:28 DEBUG 19:23:28 DEBUG "pod": "ds-cts-2" 19:23:28 DEBUG }, 19:23:28 INFO [http_cmd]: http status code OK 19:23:28 DEBUG 1713551008, 19:23:28 DEBUG 19:23:28 DEBUG }, 19:23:28 DEBUG "value": [ 19:23:28 DEBUG --- status code --- 19:23:28 DEBUG "0" 19:23:28 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:23:28 DEBUG "value": [ 19:23:28 DEBUG 1713551008, 19:23:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:28 DEBUG ] 19:23:28 DEBUG }, 19:23:28 DEBUG 1713551008, 19:23:28 DEBUG "0" 19:23:28 DEBUG --- http response --- 19:23:28 DEBUG } 19:23:28 DEBUG "value": [ 19:23:28 DEBUG "0" 19:23:28 DEBUG ] 19:23:28 DEBUG ] 19:23:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:23:28 DEBUG 1713551008, 19:23:28 DEBUG ] 19:23:28 DEBUG } 19:23:28 DEBUG } 19:23:28 DEBUG ----- output ----- 19:23:28 DEBUG "7.69130661366364" 19:23:28 DEBUG } 19:23:28 DEBUG ] 19:23:28 DEBUG } 19:23:28 DEBUG { 19:23:28 DEBUG ] 19:23:28 DEBUG ] 19:23:28 DEBUG } 19:23:28 DEBUG "status": "success", 19:23:28 DEBUG } 19:23:28 DEBUG } 19:23:28 DEBUG } 19:23:28 DEBUG "data": { 19:23:28 DEBUG ] 19:23:28 DEBUG } 19:23:28 DEBUG "resultType": "vector", 19:23:28 DEBUG } 19:23:28 DEBUG "result": [ 19:23:28 DEBUG } 19:23:28 DEBUG { 19:23:28 DEBUG "metric": { 19:23:28 DEBUG "pod": "ds-idrepo-0" 19:23:28 DEBUG }, 19:23:28 DEBUG "value": [ 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG [...] 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG "pod": "ds-idrepo-2" 19:23:28 DEBUG }, 19:23:28 DEBUG "value": [ 19:23:28 DEBUG 1713551008, 19:23:28 DEBUG "0" 19:23:28 DEBUG ] 19:23:28 DEBUG } 19:23:28 DEBUG ] 19:23:28 DEBUG } 19:23:28 DEBUG } 19:23:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551008" 19:23:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551008" 19:23:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551008" 19:23:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551008" 19:23:28 INFO [http_cmd]: http status code OK 19:23:28 DEBUG --- status code --- 19:23:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:28 DEBUG --- http response --- 19:23:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:23:28 DEBUG ----- output ----- 19:23:28 DEBUG { 19:23:28 DEBUG "status": "success", 19:23:28 DEBUG "data": { 19:23:28 DEBUG "resultType": "vector", 19:23:28 DEBUG "result": [ 19:23:28 DEBUG { 19:23:28 DEBUG "metric": { 19:23:28 DEBUG "pod": "ds-idrepo-0" 19:23:28 DEBUG }, 19:23:28 DEBUG "value": [ 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG [...] 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG "pod": "ds-idrepo-2" 19:23:28 DEBUG }, 19:23:28 DEBUG "value": [ 19:23:28 DEBUG 1713551008, 19:23:28 DEBUG "0" 19:23:28 DEBUG ] 19:23:28 DEBUG } 19:23:28 DEBUG ] 19:23:28 DEBUG } 19:23:28 DEBUG } 19:23:28 INFO [http_cmd]: http status code OK 19:23:28 INFO [http_cmd]: http status code OK 19:23:28 INFO [http_cmd]: http status code OK 19:23:28 DEBUG --- status code --- 19:23:28 DEBUG --- status code --- 19:23:28 DEBUG --- status code --- 19:23:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:28 DEBUG --- http response --- 19:23:28 DEBUG --- http response --- 19:23:28 DEBUG --- http response --- 19:23:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:23:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:23:28 DEBUG ----- output ----- 19:23:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:23:28 DEBUG ----- output ----- 19:23:28 DEBUG { 19:23:28 DEBUG ----- output ----- 19:23:28 DEBUG { 19:23:28 DEBUG "status": "success", 19:23:28 DEBUG { 19:23:28 DEBUG "status": "success", 19:23:28 DEBUG "data": { 19:23:28 DEBUG "status": "success", 19:23:28 DEBUG "data": { 19:23:28 DEBUG "resultType": "vector", 19:23:28 DEBUG "data": { 19:23:28 DEBUG "resultType": "vector", 19:23:28 DEBUG "result": [ 19:23:28 DEBUG "resultType": "vector", 19:23:28 DEBUG "result": [ 19:23:28 DEBUG { 19:23:28 DEBUG "result": [ 19:23:28 DEBUG { 19:23:28 DEBUG "metric": { 19:23:28 DEBUG { 19:23:28 DEBUG "metric": { 19:23:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:23:28 DEBUG "metric": { 19:23:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:23:28 DEBUG }, 19:23:28 DEBUG "instance": "10.142.0.107:9100" 19:23:28 DEBUG }, 19:23:28 DEBUG "value": [ 19:23:28 DEBUG }, 19:23:28 DEBUG "value": [ 19:23:28 DEBUG 19:23:28 DEBUG "value": [ 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG [...] 19:23:28 DEBUG 19:23:28 DEBUG [...] 19:23:28 DEBUG 19:23:28 DEBUG [...] 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:23:28 DEBUG 19:23:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:23:28 DEBUG }, 19:23:28 DEBUG "instance": "10.142.0.43:9100" 19:23:28 DEBUG }, 19:23:28 DEBUG "value": [ 19:23:28 DEBUG }, 19:23:28 DEBUG "value": [ 19:23:28 DEBUG 1713551008, 19:23:28 DEBUG "value": [ 19:23:28 DEBUG 1713551008, 19:23:28 DEBUG "2327.4787535410765" 19:23:28 DEBUG 1713551008, 19:23:28 DEBUG "2337.6103982669556" 19:23:28 DEBUG ] 19:23:28 DEBUG "1874127649792" 19:23:28 DEBUG ] 19:23:28 DEBUG } 19:23:28 DEBUG ] 19:23:28 DEBUG } 19:23:28 DEBUG ] 19:23:28 DEBUG } 19:23:28 DEBUG ] 19:23:28 DEBUG } 19:23:28 DEBUG ] 19:23:28 DEBUG } 19:23:28 DEBUG } 19:23:28 DEBUG } 19:23:28 DEBUG } 19:23:28 DEBUG } 19:23:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551008" 19:23:28 INFO [http_cmd]: http status code OK 19:23:28 DEBUG --- status code --- 19:23:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:28 DEBUG --- http response --- 19:23:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:23:28 DEBUG ----- output ----- 19:23:28 DEBUG { 19:23:28 DEBUG "status": "success", 19:23:28 DEBUG "data": { 19:23:28 DEBUG "resultType": "vector", 19:23:28 DEBUG "result": [ 19:23:28 DEBUG { 19:23:28 DEBUG "metric": { 19:23:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:23:28 DEBUG }, 19:23:28 DEBUG "value": [ 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG [...] 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:23:28 DEBUG }, 19:23:28 DEBUG "value": [ 19:23:28 DEBUG 1713551008, 19:23:28 DEBUG "419740" 19:23:28 DEBUG ] 19:23:28 DEBUG } 19:23:28 DEBUG ] 19:23:28 DEBUG } 19:23:28 DEBUG } 19:23:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551008" 19:23:28 INFO [http_cmd]: http status code OK 19:23:28 DEBUG --- status code --- 19:23:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:28 DEBUG --- http response --- 19:23:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:23:28 DEBUG ----- output ----- 19:23:28 DEBUG { 19:23:28 DEBUG "status": "success", 19:23:28 DEBUG "data": { 19:23:28 DEBUG "resultType": "vector", 19:23:28 DEBUG "result": [ 19:23:28 DEBUG { 19:23:28 DEBUG "metric": { 19:23:28 DEBUG "instance": "10.142.0.107:9100" 19:23:28 DEBUG }, 19:23:28 DEBUG "value": [ 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG [...] 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG "instance": "10.142.0.43:9100" 19:23:28 DEBUG }, 19:23:28 DEBUG "value": [ 19:23:28 DEBUG 1713551008, 19:23:28 DEBUG "0.05466666666667001" 19:23:28 DEBUG ] 19:23:28 DEBUG } 19:23:28 DEBUG ] 19:23:28 DEBUG } 19:23:28 DEBUG } 19:23:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551008" 19:23:28 INFO [http_cmd]: http status code OK 19:23:28 DEBUG --- status code --- 19:23:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:28 DEBUG --- http response --- 19:23:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:23:28 DEBUG ----- output ----- 19:23:28 DEBUG { 19:23:28 DEBUG "status": "success", 19:23:28 DEBUG "data": { 19:23:28 DEBUG "resultType": "vector", 19:23:28 DEBUG "result": [ 19:23:28 DEBUG { 19:23:28 DEBUG "metric": { 19:23:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:23:28 DEBUG }, 19:23:28 DEBUG "value": [ 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG [...] 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:23:28 DEBUG }, 19:23:28 DEBUG "value": [ 19:23:28 DEBUG 1713551008, 19:23:28 DEBUG "2338.2102982836195" 19:23:28 DEBUG ] 19:23:28 DEBUG } 19:23:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551008" 19:23:28 DEBUG ] 19:23:28 DEBUG } 19:23:28 DEBUG } 19:23:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551008" 19:23:28 INFO [http_cmd]: http status code OK 19:23:28 DEBUG --- status code --- 19:23:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:28 DEBUG --- http response --- 19:23:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:23:28 DEBUG ----- output ----- 19:23:28 DEBUG { 19:23:28 DEBUG "status": "success", 19:23:28 DEBUG "data": { 19:23:28 DEBUG "resultType": "vector", 19:23:28 DEBUG "result": [ 19:23:28 DEBUG { 19:23:28 DEBUG "metric": { 19:23:28 DEBUG "instance": "10.142.0.107:9100" 19:23:28 DEBUG }, 19:23:28 DEBUG "value": [ 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG [...] 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG "instance": "10.142.0.43:9100" 19:23:28 DEBUG }, 19:23:28 DEBUG "value": [ 19:23:28 DEBUG 1713551008, 19:23:28 DEBUG "0.11830000000000117" 19:23:28 DEBUG ] 19:23:28 DEBUG } 19:23:28 DEBUG ] 19:23:28 DEBUG } 19:23:28 DEBUG } 19:23:28 INFO [http_cmd]: http status code OK 19:23:28 DEBUG --- status code --- 19:23:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:28 DEBUG --- http response --- 19:23:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:23:28 DEBUG ----- output ----- 19:23:28 DEBUG { 19:23:28 DEBUG "status": "success", 19:23:28 DEBUG "data": { 19:23:28 DEBUG "resultType": "vector", 19:23:28 DEBUG "result": [ 19:23:28 DEBUG { 19:23:28 DEBUG "metric": { 19:23:28 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:23:28 DEBUG }, 19:23:28 DEBUG "value": [ 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG [...] 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG "pod": "ds-idrepo-2" 19:23:28 DEBUG }, 19:23:28 DEBUG "value": [ 19:23:28 DEBUG 1713551008, 19:23:28 DEBUG "17481.794833056352" 19:23:28 DEBUG ] 19:23:28 DEBUG } 19:23:28 DEBUG ] 19:23:28 DEBUG } 19:23:28 DEBUG } 19:23:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551008" 19:23:28 INFO [http_cmd]: http status code OK 19:23:28 DEBUG --- status code --- 19:23:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:28 DEBUG --- http response --- 19:23:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:23:28 DEBUG ----- output ----- 19:23:28 DEBUG { 19:23:28 DEBUG "status": "success", 19:23:28 DEBUG "data": { 19:23:28 DEBUG "resultType": "vector", 19:23:28 DEBUG "result": [ 19:23:28 DEBUG { 19:23:28 DEBUG "metric": { 19:23:28 DEBUG "pod": "ds-idrepo-0" 19:23:28 DEBUG }, 19:23:28 DEBUG "value": [ 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG [...] 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG "pod": "ds-idrepo-2" 19:23:28 DEBUG }, 19:23:28 DEBUG "value": [ 19:23:28 DEBUG 1713551008, 19:23:28 DEBUG "0" 19:23:28 DEBUG ] 19:23:28 DEBUG } 19:23:28 DEBUG ] 19:23:28 DEBUG } 19:23:28 DEBUG } 19:23:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551008" 19:23:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713551008" 19:23:28 INFO [http_cmd]: http status code OK 19:23:28 DEBUG --- status code --- 19:23:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:28 DEBUG --- http response --- 19:23:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:23:28 DEBUG ----- output ----- 19:23:28 DEBUG { 19:23:28 DEBUG "status": "success", 19:23:28 DEBUG "data": { 19:23:28 DEBUG "resultType": "vector", 19:23:28 DEBUG "result": [ 19:23:28 DEBUG { 19:23:28 DEBUG "metric": { 19:23:28 DEBUG "pod": "ds-cts-0" 19:23:28 DEBUG }, 19:23:28 DEBUG "value": [ 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG [...] 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG "pod": "ds-cts-2" 19:23:28 DEBUG }, 19:23:28 DEBUG "value": [ 19:23:28 DEBUG 1713551008, 19:23:28 DEBUG "0" 19:23:28 DEBUG ] 19:23:28 DEBUG } 19:23:28 DEBUG ] 19:23:28 DEBUG } 19:23:28 DEBUG } 19:23:28 INFO [http_cmd]: http status code OK 19:23:28 DEBUG --- status code --- 19:23:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:28 DEBUG --- http response --- 19:23:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:23:28 DEBUG ----- output ----- 19:23:28 DEBUG { 19:23:28 DEBUG "status": "success", 19:23:28 DEBUG "data": { 19:23:28 DEBUG "resultType": "vector", 19:23:28 DEBUG "result": [ 19:23:28 DEBUG { 19:23:28 DEBUG "metric": { 19:23:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:23:28 DEBUG }, 19:23:28 DEBUG "value": [ 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG [...] 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG 19:23:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:23:28 DEBUG }, 19:23:28 DEBUG "value": [ 19:23:28 DEBUG 1713551008, 19:23:28 DEBUG "0.004" 19:23:28 DEBUG ] 19:23:28 DEBUG } 19:23:28 DEBUG ] 19:23:28 DEBUG } 19:23:28 DEBUG } 19:23:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551009" 19:23:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551009" 19:23:29 INFO [http_cmd]: http status code OK 19:23:29 DEBUG --- status code --- 19:23:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:29 DEBUG --- http response --- 19:23:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:23:29 DEBUG ----- output ----- 19:23:29 DEBUG { 19:23:29 DEBUG "status": "success", 19:23:29 DEBUG "data": { 19:23:29 DEBUG "resultType": "vector", 19:23:29 DEBUG "result": [ 19:23:29 DEBUG { 19:23:29 DEBUG "metric": { 19:23:29 DEBUG "pod": "ds-cts-0" 19:23:29 DEBUG }, 19:23:29 DEBUG "value": [ 19:23:29 DEBUG 19:23:29 DEBUG 19:23:29 DEBUG 19:23:29 DEBUG [...] 19:23:29 DEBUG 19:23:29 DEBUG 19:23:29 DEBUG 19:23:29 DEBUG "pod": "ds-cts-2" 19:23:29 DEBUG }, 19:23:29 DEBUG "value": [ 19:23:29 DEBUG 1713551009, 19:23:29 DEBUG "0" 19:23:29 DEBUG ] 19:23:29 DEBUG } 19:23:29 DEBUG ] 19:23:29 DEBUG } 19:23:29 DEBUG } 19:23:29 INFO [http_cmd]: http status code OK 19:23:29 DEBUG --- status code --- 19:23:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:29 DEBUG --- http response --- 19:23:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:23:29 DEBUG ----- output ----- 19:23:29 DEBUG { 19:23:29 DEBUG "status": "success", 19:23:29 DEBUG "data": { 19:23:29 DEBUG "resultType": "vector", 19:23:29 DEBUG "result": [ 19:23:29 DEBUG { 19:23:29 DEBUG "metric": { 19:23:29 DEBUG "instance": "10.142.0.107:9100" 19:23:29 DEBUG }, 19:23:29 DEBUG "value": [ 19:23:29 DEBUG 19:23:29 DEBUG 19:23:29 DEBUG 19:23:29 DEBUG [...] 19:23:29 DEBUG 19:23:29 DEBUG 19:23:29 DEBUG 19:23:29 DEBUG "instance": "10.142.0.43:9100" 19:23:29 DEBUG }, 19:23:29 DEBUG "value": [ 19:23:29 DEBUG 1713551009, 19:23:29 DEBUG "70687522.46666667" 19:23:29 DEBUG ] 19:23:29 DEBUG } 19:23:29 DEBUG ] 19:23:29 DEBUG } 19:23:29 DEBUG } 19:23:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551009" 19:23:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551009" 19:23:29 INFO [http_cmd]: http status code OK 19:23:29 DEBUG --- status code --- 19:23:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:29 DEBUG --- http response --- 19:23:29 INFO [http_cmd]: http status code OK 19:23:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:23:29 DEBUG --- status code --- 19:23:29 DEBUG ----- output ----- 19:23:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:29 DEBUG { 19:23:29 DEBUG --- http response --- 19:23:29 DEBUG "status": "success", 19:23:29 DEBUG "data": { 19:23:29 DEBUG "resultType": "vector", 19:23:29 DEBUG "result": [ 19:23:29 DEBUG { 19:23:29 DEBUG "metric": { 19:23:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:23:29 DEBUG }, 19:23:29 DEBUG "value": [ 19:23:29 DEBUG 19:23:29 DEBUG 19:23:29 DEBUG 19:23:29 DEBUG [...] 19:23:29 DEBUG 19:23:29 DEBUG 19:23:29 DEBUG 19:23:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:23:29 DEBUG }, 19:23:29 DEBUG "value": [ 19:23:29 DEBUG 1713551009, 19:23:29 DEBUG "2338.0769871688053" 19:23:29 DEBUG ] 19:23:29 DEBUG } 19:23:29 DEBUG ] 19:23:29 DEBUG } 19:23:29 DEBUG } 19:23:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 19:23:29 DEBUG ----- output ----- 19:23:29 DEBUG { 19:23:29 DEBUG "status": "success", 19:23:29 DEBUG "data": { 19:23:29 DEBUG "resultType": "vector", 19:23:29 DEBUG "result": [ 19:23:29 DEBUG { 19:23:29 DEBUG "metric": { 19:23:29 DEBUG "pod": "ds-idrepo-1" 19:23:29 DEBUG }, 19:23:29 DEBUG "value": [ 19:23:29 DEBUG 19:23:29 DEBUG 19:23:29 DEBUG 19:23:29 DEBUG [...] 19:23:29 DEBUG 19:23:29 DEBUG 19:23:29 DEBUG 19:23:29 DEBUG "pod": "ds-idrepo-2" 19:23:29 DEBUG }, 19:23:29 DEBUG "value": [ 19:23:29 DEBUG 1713551009, 19:23:29 DEBUG "0" 19:23:29 DEBUG ] 19:23:29 DEBUG } 19:23:29 DEBUG ] 19:23:29 DEBUG } 19:23:29 DEBUG } 19:23:29 INFO 19:23:29 INFO [loop_until]: kubectl --namespace=xlou top pods 19:23:29 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:23:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551009" 19:23:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551009" 19:23:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713551009" 19:23:29 INFO [http_cmd]: http status code OK 19:23:29 DEBUG --- status code --- 19:23:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:29 DEBUG --- http response --- 19:23:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:23:29 DEBUG ----- output ----- 19:23:29 DEBUG { 19:23:29 DEBUG "status": "success", 19:23:29 DEBUG "data": { 19:23:29 DEBUG "resultType": "vector", 19:23:29 DEBUG "result": [ 19:23:29 DEBUG { 19:23:29 DEBUG "metric": { 19:23:29 DEBUG "pod": "ds-idrepo-1" 19:23:29 DEBUG }, 19:23:29 DEBUG "value": [ 19:23:29 DEBUG 19:23:29 DEBUG 19:23:29 DEBUG 19:23:29 DEBUG [...] 19:23:29 DEBUG 19:23:29 DEBUG 19:23:29 DEBUG 19:23:29 DEBUG "pod": "ds-idrepo-2" 19:23:29 DEBUG }, 19:23:29 DEBUG "value": [ 19:23:29 DEBUG 1713551009, 19:23:29 DEBUG "6.306283577580561" 19:23:29 DEBUG ] 19:23:29 DEBUG } 19:23:29 DEBUG ] 19:23:29 DEBUG } 19:23:29 DEBUG } 19:23:29 INFO [http_cmd]: http status code OK 19:23:29 DEBUG --- status code --- 19:23:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:29 DEBUG --- http response --- 19:23:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:23:29 DEBUG ----- output ----- 19:23:29 DEBUG { 19:23:29 DEBUG "status": "success", 19:23:29 DEBUG "data": { 19:23:29 DEBUG "resultType": "vector", 19:23:29 DEBUG "result": [ 19:23:29 DEBUG { 19:23:29 DEBUG "metric": { 19:23:29 DEBUG "instance": "10.142.0.107:9100" 19:23:29 DEBUG }, 19:23:29 DEBUG "value": [ 19:23:29 DEBUG 19:23:29 DEBUG 19:23:29 DEBUG 19:23:29 DEBUG [...] 19:23:29 DEBUG 19:23:29 DEBUG 19:23:29 DEBUG 19:23:29 DEBUG "instance": "10.142.0.43:9100" 19:23:29 DEBUG }, 19:23:29 DEBUG "value": [ 19:23:29 DEBUG 1713551009, 19:23:29 DEBUG "70683487.69999999" 19:23:29 DEBUG ] 19:23:29 DEBUG } 19:23:29 DEBUG ] 19:23:29 DEBUG } 19:23:29 DEBUG } 19:23:29 INFO [http_cmd]: http status code OK 19:23:29 DEBUG --- status code --- 19:23:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:29 DEBUG --- http response --- 19:23:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:23:29 DEBUG ----- output ----- 19:23:29 DEBUG { 19:23:29 DEBUG "status": "success", 19:23:29 DEBUG "data": { 19:23:29 DEBUG "resultType": "vector", 19:23:29 DEBUG "result": [ 19:23:29 DEBUG { 19:23:29 DEBUG "metric": { 19:23:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:23:29 DEBUG }, 19:23:29 DEBUG "value": [ 19:23:29 DEBUG 19:23:29 DEBUG 19:23:29 DEBUG 19:23:29 DEBUG [...] 19:23:29 DEBUG 19:23:29 DEBUG 19:23:29 DEBUG 19:23:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:23:29 DEBUG }, 19:23:29 DEBUG "value": [ 19:23:29 DEBUG 1713551009, 19:23:29 DEBUG "3384168448" 19:23:29 DEBUG ] 19:23:29 DEBUG } 19:23:29 DEBUG ] 19:23:29 DEBUG } 19:23:29 DEBUG } 19:23:29 INFO [loop_until]: OK (rc = 0) 19:23:29 DEBUG --- stdout --- 19:23:29 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 14234m 8998Mi am-767f6b67b5-qcqz2 14179m 9003Mi am-767f6b67b5-vbltz 15701m 8988Mi ds-cts-0 15487m 10843Mi ds-cts-1 13798m 11102Mi ds-cts-2 15587m 10857Mi ds-idrepo-0 6406m 3639Mi ds-idrepo-1 6191m 3507Mi ds-idrepo-2 6259m 3492Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 4m 1217Mi lodemon-f8459c597-6tj6f 2m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7596m 3133Mi 19:23:29 DEBUG --- stderr --- 19:23:29 DEBUG 19:23:29 INFO 19:23:29 INFO [loop_until]: kubectl --namespace=xlou top node 19:23:29 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:23:29 INFO [loop_until]: OK (rc = 0) 19:23:29 DEBUG --- stdout --- 19:23:29 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 74m 0% 6261Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 6542m 21% 7231Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 14210m 47% 11710Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 3348Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 15295m 51% 13880Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 137m 0% 5208Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 13980m 46% 11887Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12373m 41% 3002Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 16369m 54% 11204Mi 10% gke-xlou-cdm-ds-0f601751-j488 6923m 23% 6721Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 13752m 46% 15918Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 6966m 23% 7392Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 15354m 51% 13347Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 7879m 49% 6941Mi 11% 19:23:29 DEBUG --- stderr --- 19:23:29 DEBUG 19:24:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551068" 19:24:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713551068" 19:24:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551068" 19:24:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713551068" 19:24:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551068" 19:24:28 INFO [http_cmd]: http status code OK 19:24:28 DEBUG --- status code --- 19:24:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:28 DEBUG --- http response --- 19:24:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551068" 19:24:28 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:24:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551068" 19:24:28 INFO [http_cmd]: http status code OK 19:24:28 DEBUG --- status code --- 19:24:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:28 DEBUG --- http response --- 19:24:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:24:28 DEBUG ----- output ----- 19:24:28 DEBUG { 19:24:28 DEBUG "status": "success", 19:24:28 DEBUG "data": { 19:24:28 DEBUG "resultType": "vector", 19:24:28 DEBUG "result": [ 19:24:28 DEBUG { 19:24:28 DEBUG "metric": { 19:24:28 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:24:28 DEBUG }, 19:24:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713551068" 19:24:28 DEBUG "value": [ 19:24:28 INFO [http_cmd]: http status code OK 19:24:28 DEBUG 19:24:28 DEBUG --- status code --- 19:24:28 DEBUG 19:24:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:28 DEBUG 19:24:28 DEBUG --- http response --- 19:24:28 DEBUG [...] 19:24:28 DEBUG 19:24:28 DEBUG 19:24:28 DEBUG 19:24:28 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:24:28 DEBUG }, 19:24:28 DEBUG "value": [ 19:24:28 DEBUG 1713551068, 19:24:28 DEBUG "7.810517914881276" 19:24:28 DEBUG ] 19:24:28 DEBUG } 19:24:28 DEBUG ] 19:24:28 DEBUG } 19:24:28 DEBUG } 19:24:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551068" 19:24:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:24:28 DEBUG ----- output ----- 19:24:28 DEBUG { 19:24:28 DEBUG "status": "success", 19:24:28 DEBUG "data": { 19:24:28 DEBUG "resultType": "vector", 19:24:28 DEBUG "result": [ 19:24:28 DEBUG { 19:24:28 DEBUG "metric": { 19:24:28 DEBUG "pod": "ds-cts-0" 19:24:28 DEBUG }, 19:24:28 DEBUG "value": [ 19:24:28 DEBUG 19:24:28 DEBUG 19:24:28 DEBUG 19:24:28 DEBUG [...] 19:24:28 DEBUG 19:24:28 DEBUG 19:24:28 DEBUG 19:24:28 DEBUG "pod": "ds-cts-2" 19:24:28 DEBUG }, 19:24:28 DEBUG "value": [ 19:24:28 DEBUG 1713551068, 19:24:28 DEBUG "0" 19:24:28 DEBUG ] 19:24:28 DEBUG } 19:24:28 DEBUG ] 19:24:28 DEBUG } 19:24:28 DEBUG } 19:24:28 ERROR [http_cmd]: ERROR 19:24:28 ERROR --- status code --- 19:24:28 INFO [http_cmd]: http status code OK 19:24:28 INFO [http_cmd]: http status code OK 19:24:28 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:24:28 DEBUG --- status code --- 19:24:28 DEBUG --- status code --- 19:24:28 ERROR --- http response --- 19:24:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:28 DEBUG --- http response --- 19:24:28 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:24:28 DEBUG --- http response --- 19:24:28 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:24:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:24:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:24:28 DEBUG ----- output ----- 19:24:28 DEBUG ----- output ----- 19:24:28 DEBUG { 19:24:28 DEBUG { 19:24:28 DEBUG "status": "success", 19:24:28 DEBUG "status": "success", 19:24:28 DEBUG "data": { 19:24:28 DEBUG "data": { 19:24:28 DEBUG "resultType": "vector", 19:24:28 DEBUG "resultType": "vector", 19:24:28 DEBUG "result": [ 19:24:28 DEBUG "result": [ 19:24:28 DEBUG { 19:24:28 DEBUG { 19:24:28 DEBUG "metric": { 19:24:28 DEBUG "metric": { 19:24:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:24:28 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:24:28 DEBUG }, 19:24:28 DEBUG }, 19:24:28 DEBUG "value": [ 19:24:28 DEBUG "value": [ 19:24:28 DEBUG 19:24:28 DEBUG 19:24:28 DEBUG 19:24:28 DEBUG 19:24:28 DEBUG 19:24:28 DEBUG 19:24:28 DEBUG [...] 19:24:28 DEBUG [...] 19:24:28 DEBUG 19:24:28 DEBUG 19:24:28 DEBUG 19:24:28 DEBUG 19:24:28 DEBUG 19:24:28 DEBUG 19:24:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:24:28 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 19:24:28 DEBUG }, 19:24:28 DEBUG }, 19:24:28 DEBUG "value": [ 19:24:28 DEBUG "value": [ 19:24:28 INFO [http_cmd]: http status code OK 19:24:28 DEBUG 1713551068, 19:24:28 INFO [http_cmd]: http status code OK 19:24:28 INFO [http_cmd]: http status code OK 19:24:28 DEBUG 1713551068, 19:24:28 DEBUG --- status code --- 19:24:28 DEBUG "0" 19:24:28 DEBUG --- status code --- 19:24:28 DEBUG --- status code --- 19:24:28 DEBUG "0" 19:24:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:28 DEBUG ] 19:24:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:28 DEBUG ] 19:24:28 DEBUG --- http response --- 19:24:28 DEBUG } 19:24:28 DEBUG --- http response --- 19:24:28 DEBUG --- http response --- 19:24:28 DEBUG } 19:24:28 DEBUG ] 19:24:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:24:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:24:28 DEBUG ] 19:24:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:24:28 DEBUG } 19:24:28 DEBUG ----- output ----- 19:24:28 DEBUG ----- output ----- 19:24:28 DEBUG } 19:24:28 DEBUG ----- output ----- 19:24:28 DEBUG } 19:24:28 DEBUG { 19:24:28 DEBUG { 19:24:28 DEBUG } 19:24:28 DEBUG { 19:24:28 DEBUG "status": "success", 19:24:28 DEBUG "status": "success", 19:24:28 DEBUG "status": "success", 19:24:28 DEBUG "data": { 19:24:28 DEBUG "data": { 19:24:28 DEBUG "data": { 19:24:28 DEBUG "resultType": "vector", 19:24:28 DEBUG "resultType": "vector", 19:24:28 DEBUG "resultType": "vector", 19:24:28 DEBUG "result": [ 19:24:28 DEBUG "result": [ 19:24:28 DEBUG "result": [ 19:24:28 DEBUG { 19:24:28 DEBUG { 19:24:28 DEBUG { 19:24:28 DEBUG "metric": { 19:24:28 DEBUG "metric": { 19:24:28 DEBUG "metric": { 19:24:28 DEBUG "pod": "ds-idrepo-0" 19:24:28 DEBUG "pod": "ds-idrepo-0" 19:24:28 DEBUG "pod": "ds-idrepo-0" 19:24:28 DEBUG }, 19:24:28 DEBUG }, 19:24:28 DEBUG }, 19:24:28 DEBUG "value": [ 19:24:28 DEBUG "value": [ 19:24:28 DEBUG "value": [ 19:24:28 DEBUG 19:24:28 DEBUG 19:24:28 DEBUG 19:24:28 DEBUG 19:24:28 DEBUG 19:24:28 DEBUG 19:24:28 DEBUG 19:24:28 DEBUG 19:24:28 DEBUG 19:24:28 DEBUG [...] 19:24:28 DEBUG [...] 19:24:28 DEBUG [...] 19:24:28 DEBUG 19:24:28 DEBUG 19:24:28 DEBUG 19:24:28 DEBUG 19:24:28 DEBUG 19:24:28 DEBUG 19:24:28 DEBUG 19:24:28 DEBUG 19:24:28 DEBUG 19:24:28 DEBUG "pod": "ds-cts-2" 19:24:28 DEBUG "pod": "ds-idrepo-2" 19:24:28 DEBUG "pod": "ds-idrepo-2" 19:24:28 DEBUG }, 19:24:28 DEBUG }, 19:24:28 DEBUG }, 19:24:28 DEBUG "value": [ 19:24:28 DEBUG "value": [ 19:24:28 DEBUG "value": [ 19:24:28 DEBUG 1713551068, 19:24:28 DEBUG 1713551068, 19:24:28 DEBUG 1713551068, 19:24:28 DEBUG "0" 19:24:28 DEBUG "0" 19:24:28 DEBUG "0" 19:24:28 DEBUG ] 19:24:28 DEBUG ] 19:24:28 DEBUG ] 19:24:28 DEBUG } 19:24:28 DEBUG } 19:24:28 DEBUG } 19:24:28 DEBUG ] 19:24:28 DEBUG ] 19:24:28 DEBUG ] 19:24:28 DEBUG } 19:24:28 DEBUG } 19:24:28 DEBUG } 19:24:28 DEBUG } 19:24:28 DEBUG } 19:24:28 DEBUG } 19:24:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551068" 19:24:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551068" 19:24:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551068" 19:24:28 INFO [http_cmd]: http status code OK 19:24:28 INFO [http_cmd]: http status code OK 19:24:28 DEBUG --- status code --- 19:24:28 DEBUG --- status code --- 19:24:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:28 DEBUG --- http response --- 19:24:28 DEBUG --- http response --- 19:24:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:24:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:24:28 DEBUG ----- output ----- 19:24:28 DEBUG ----- output ----- 19:24:28 DEBUG { 19:24:28 DEBUG { 19:24:28 DEBUG "status": "success", 19:24:28 DEBUG "status": "success", 19:24:28 DEBUG "data": { 19:24:28 DEBUG "data": { 19:24:28 DEBUG "resultType": "vector", 19:24:28 DEBUG "resultType": "vector", 19:24:28 DEBUG "result": [ 19:24:28 INFO [http_cmd]: http status code OK 19:24:28 DEBUG "result": [ 19:24:28 DEBUG { 19:24:28 DEBUG --- status code --- 19:24:28 DEBUG { 19:24:28 DEBUG "metric": { 19:24:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:28 DEBUG "metric": { 19:24:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:24:28 DEBUG --- http response --- 19:24:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:24:28 DEBUG }, 19:24:28 DEBUG }, 19:24:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:24:28 DEBUG "value": [ 19:24:28 DEBUG "value": [ 19:24:28 DEBUG ----- output ----- 19:24:28 DEBUG 19:24:28 DEBUG 19:24:28 DEBUG { 19:24:28 DEBUG 19:24:28 DEBUG 19:24:28 DEBUG "status": "success", 19:24:28 DEBUG 19:24:28 DEBUG 19:24:28 DEBUG "data": { 19:24:28 DEBUG [...] 19:24:28 DEBUG [...] 19:24:28 DEBUG "resultType": "vector", 19:24:28 DEBUG 19:24:28 DEBUG 19:24:28 DEBUG "result": [ 19:24:28 DEBUG 19:24:28 DEBUG 19:24:28 DEBUG { 19:24:28 DEBUG 19:24:28 DEBUG 19:24:28 DEBUG "metric": { 19:24:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:24:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:24:28 DEBUG "instance": "10.142.0.107:9100" 19:24:28 DEBUG }, 19:24:28 DEBUG }, 19:24:28 DEBUG }, 19:24:28 DEBUG "value": [ 19:24:28 DEBUG "value": [ 19:24:28 DEBUG "value": [ 19:24:28 DEBUG 1713551068, 19:24:28 DEBUG 1713551068, 19:24:28 DEBUG 19:24:28 DEBUG "2324.0666666666666" 19:24:28 DEBUG "2342.7666666666664" 19:24:28 DEBUG 19:24:28 DEBUG ] 19:24:28 DEBUG ] 19:24:28 DEBUG 19:24:28 DEBUG } 19:24:28 DEBUG } 19:24:28 DEBUG [...] 19:24:28 DEBUG ] 19:24:28 DEBUG ] 19:24:28 DEBUG 19:24:28 DEBUG } 19:24:28 DEBUG } 19:24:28 DEBUG 19:24:28 DEBUG } 19:24:28 DEBUG } 19:24:28 DEBUG 19:24:28 DEBUG "instance": "10.142.0.43:9100" 19:24:28 DEBUG }, 19:24:28 DEBUG "value": [ 19:24:28 DEBUG 1713551068, 19:24:28 DEBUG "1877218737152" 19:24:28 DEBUG ] 19:24:28 DEBUG } 19:24:28 DEBUG ] 19:24:28 DEBUG } 19:24:28 DEBUG } 19:24:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551068" 19:24:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551068" 19:24:28 INFO [http_cmd]: http status code OK 19:24:28 DEBUG --- status code --- 19:24:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:28 DEBUG --- http response --- 19:24:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:24:28 DEBUG ----- output ----- 19:24:28 DEBUG { 19:24:28 DEBUG "status": "success", 19:24:28 DEBUG "data": { 19:24:28 DEBUG "resultType": "vector", 19:24:28 DEBUG "result": [ 19:24:28 DEBUG { 19:24:28 DEBUG "metric": { 19:24:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:24:28 DEBUG }, 19:24:28 DEBUG "value": [ 19:24:28 DEBUG 19:24:28 DEBUG 19:24:28 DEBUG 19:24:28 DEBUG [...] 19:24:28 DEBUG 19:24:28 DEBUG 19:24:28 DEBUG 19:24:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:24:28 DEBUG }, 19:24:28 DEBUG "value": [ 19:24:28 DEBUG 1713551068, 19:24:28 DEBUG "420216" 19:24:28 DEBUG ] 19:24:28 DEBUG } 19:24:28 DEBUG ] 19:24:28 DEBUG } 19:24:28 DEBUG } 19:24:28 INFO [http_cmd]: http status code OK 19:24:28 DEBUG --- status code --- 19:24:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:28 DEBUG --- http response --- 19:24:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:24:28 DEBUG ----- output ----- 19:24:28 DEBUG { 19:24:28 DEBUG "status": "success", 19:24:28 DEBUG "data": { 19:24:28 DEBUG "resultType": "vector", 19:24:28 DEBUG "result": [ 19:24:28 DEBUG { 19:24:28 DEBUG "metric": { 19:24:28 DEBUG "instance": "10.142.0.107:9100" 19:24:28 DEBUG }, 19:24:28 DEBUG "value": [ 19:24:28 DEBUG 19:24:28 DEBUG 19:24:28 DEBUG 19:24:28 DEBUG [...] 19:24:28 DEBUG 19:24:28 DEBUG 19:24:28 DEBUG 19:24:28 DEBUG "instance": "10.142.0.43:9100" 19:24:28 DEBUG }, 19:24:28 DEBUG "value": [ 19:24:28 DEBUG 1713551068, 19:24:28 DEBUG "0.1103333333333372" 19:24:28 DEBUG ] 19:24:28 DEBUG } 19:24:28 DEBUG ] 19:24:28 DEBUG } 19:24:28 DEBUG } 19:24:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551068" 19:24:29 INFO [http_cmd]: http status code OK 19:24:29 DEBUG --- status code --- 19:24:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:29 DEBUG --- http response --- 19:24:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551069" 19:24:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:24:29 DEBUG ----- output ----- 19:24:29 DEBUG { 19:24:29 DEBUG "status": "success", 19:24:29 DEBUG "data": { 19:24:29 DEBUG "resultType": "vector", 19:24:29 DEBUG "result": [ 19:24:29 DEBUG { 19:24:29 DEBUG "metric": { 19:24:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:24:29 DEBUG }, 19:24:29 DEBUG "value": [ 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG [...] 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:24:29 DEBUG }, 19:24:29 DEBUG "value": [ 19:24:29 DEBUG 1713551068, 19:24:29 DEBUG "2342.4666666666667" 19:24:29 DEBUG ] 19:24:29 DEBUG } 19:24:29 DEBUG ] 19:24:29 DEBUG } 19:24:29 DEBUG } 19:24:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551069" 19:24:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551069" 19:24:29 INFO [http_cmd]: http status code OK 19:24:29 DEBUG --- status code --- 19:24:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:29 DEBUG --- http response --- 19:24:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:24:29 DEBUG ----- output ----- 19:24:29 DEBUG { 19:24:29 DEBUG "status": "success", 19:24:29 DEBUG "data": { 19:24:29 DEBUG "resultType": "vector", 19:24:29 DEBUG "result": [ 19:24:29 DEBUG { 19:24:29 DEBUG "metric": { 19:24:29 DEBUG "instance": "10.142.0.107:9100" 19:24:29 DEBUG }, 19:24:29 DEBUG "value": [ 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG [...] 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG "instance": "10.142.0.43:9100" 19:24:29 DEBUG }, 19:24:29 DEBUG "value": [ 19:24:29 DEBUG 1713551069, 19:24:29 DEBUG "0.2648000000000167" 19:24:29 DEBUG ] 19:24:29 DEBUG } 19:24:29 DEBUG ] 19:24:29 DEBUG } 19:24:29 DEBUG } 19:24:29 INFO [http_cmd]: http status code OK 19:24:29 DEBUG --- status code --- 19:24:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:29 DEBUG --- http response --- 19:24:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:24:29 DEBUG ----- output ----- 19:24:29 DEBUG { 19:24:29 DEBUG "status": "success", 19:24:29 DEBUG "data": { 19:24:29 DEBUG "resultType": "vector", 19:24:29 DEBUG "result": [ 19:24:29 DEBUG { 19:24:29 DEBUG "metric": { 19:24:29 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:24:29 DEBUG }, 19:24:29 DEBUG "value": [ 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG [...] 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:24:29 DEBUG }, 19:24:29 DEBUG "value": [ 19:24:29 DEBUG 1713551069, 19:24:29 DEBUG "0" 19:24:29 DEBUG ] 19:24:29 DEBUG } 19:24:29 DEBUG ] 19:24:29 DEBUG } 19:24:29 DEBUG } 19:24:29 INFO [http_cmd]: http status code OK 19:24:29 DEBUG --- status code --- 19:24:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:29 DEBUG --- http response --- 19:24:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:24:29 DEBUG ----- output ----- 19:24:29 DEBUG { 19:24:29 DEBUG "status": "success", 19:24:29 DEBUG "data": { 19:24:29 DEBUG "resultType": "vector", 19:24:29 DEBUG "result": [ 19:24:29 DEBUG { 19:24:29 DEBUG "metric": { 19:24:29 DEBUG "pod": "ds-cts-0" 19:24:29 DEBUG }, 19:24:29 DEBUG "value": [ 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG [...] 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551069" 19:24:29 DEBUG "pod": "ds-cts-2" 19:24:29 DEBUG }, 19:24:29 DEBUG "value": [ 19:24:29 DEBUG 1713551069, 19:24:29 DEBUG "0" 19:24:29 DEBUG ] 19:24:29 DEBUG } 19:24:29 DEBUG ] 19:24:29 DEBUG } 19:24:29 DEBUG } 19:24:29 INFO [http_cmd]: http status code OK 19:24:29 DEBUG --- status code --- 19:24:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:29 DEBUG --- http response --- 19:24:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:24:29 DEBUG ----- output ----- 19:24:29 DEBUG { 19:24:29 DEBUG "status": "success", 19:24:29 DEBUG "data": { 19:24:29 DEBUG "resultType": "vector", 19:24:29 DEBUG "result": [ 19:24:29 DEBUG { 19:24:29 DEBUG "metric": { 19:24:29 DEBUG "pod": "ds-idrepo-0" 19:24:29 DEBUG }, 19:24:29 DEBUG "value": [ 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG [...] 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG "pod": "ds-idrepo-2" 19:24:29 DEBUG }, 19:24:29 DEBUG "value": [ 19:24:29 DEBUG 1713551069, 19:24:29 DEBUG "0" 19:24:29 DEBUG ] 19:24:29 DEBUG } 19:24:29 DEBUG ] 19:24:29 DEBUG } 19:24:29 DEBUG } 19:24:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551069" 19:24:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713551069" 19:24:29 INFO [http_cmd]: http status code OK 19:24:29 DEBUG --- status code --- 19:24:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:29 DEBUG --- http response --- 19:24:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:24:29 DEBUG ----- output ----- 19:24:29 DEBUG { 19:24:29 DEBUG "status": "success", 19:24:29 DEBUG "data": { 19:24:29 DEBUG "resultType": "vector", 19:24:29 DEBUG "result": [ 19:24:29 DEBUG { 19:24:29 DEBUG "metric": { 19:24:29 DEBUG "instance": "10.142.0.107:9100" 19:24:29 DEBUG }, 19:24:29 DEBUG "value": [ 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG [...] 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG "instance": "10.142.0.43:9100" 19:24:29 DEBUG }, 19:24:29 DEBUG "value": [ 19:24:29 DEBUG 1713551069, 19:24:29 DEBUG "70944741.26666667" 19:24:29 DEBUG ] 19:24:29 DEBUG } 19:24:29 DEBUG ] 19:24:29 DEBUG } 19:24:29 DEBUG } 19:24:29 INFO [http_cmd]: http status code OK 19:24:29 DEBUG --- status code --- 19:24:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:29 DEBUG --- http response --- 19:24:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:24:29 DEBUG ----- output ----- 19:24:29 DEBUG { 19:24:29 DEBUG "status": "success", 19:24:29 DEBUG "data": { 19:24:29 DEBUG "resultType": "vector", 19:24:29 DEBUG "result": [ 19:24:29 DEBUG { 19:24:29 DEBUG "metric": { 19:24:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:24:29 DEBUG }, 19:24:29 DEBUG "value": [ 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG [...] 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:24:29 DEBUG }, 19:24:29 DEBUG "value": [ 19:24:29 DEBUG 1713551069, 19:24:29 DEBUG "0.004" 19:24:29 DEBUG ] 19:24:29 DEBUG } 19:24:29 DEBUG ] 19:24:29 DEBUG } 19:24:29 DEBUG } 19:24:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551069" 19:24:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551069" 19:24:29 INFO [http_cmd]: http status code OK 19:24:29 DEBUG --- status code --- 19:24:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:29 DEBUG --- http response --- 19:24:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:24:29 DEBUG ----- output ----- 19:24:29 DEBUG { 19:24:29 DEBUG "status": "success", 19:24:29 DEBUG "data": { 19:24:29 DEBUG "resultType": "vector", 19:24:29 DEBUG "result": [ 19:24:29 DEBUG { 19:24:29 DEBUG "metric": { 19:24:29 DEBUG "pod": "ds-cts-0" 19:24:29 DEBUG }, 19:24:29 DEBUG "value": [ 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG [...] 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG "pod": "ds-cts-2" 19:24:29 DEBUG }, 19:24:29 DEBUG "value": [ 19:24:29 DEBUG 1713551069, 19:24:29 DEBUG "0" 19:24:29 DEBUG ] 19:24:29 DEBUG } 19:24:29 DEBUG ] 19:24:29 DEBUG } 19:24:29 DEBUG } 19:24:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551069" 19:24:29 INFO [http_cmd]: http status code OK 19:24:29 DEBUG --- status code --- 19:24:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:29 DEBUG --- http response --- 19:24:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:24:29 DEBUG ----- output ----- 19:24:29 DEBUG { 19:24:29 DEBUG "status": "success", 19:24:29 DEBUG "data": { 19:24:29 DEBUG "resultType": "vector", 19:24:29 DEBUG "result": [ 19:24:29 DEBUG { 19:24:29 DEBUG "metric": { 19:24:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:24:29 DEBUG }, 19:24:29 DEBUG "value": [ 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG [...] 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:24:29 DEBUG }, 19:24:29 DEBUG "value": [ 19:24:29 DEBUG 1713551069, 19:24:29 DEBUG "2344.8" 19:24:29 DEBUG ] 19:24:29 DEBUG } 19:24:29 DEBUG ] 19:24:29 DEBUG } 19:24:29 DEBUG } 19:24:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713551069" 19:24:29 INFO [http_cmd]: http status code OK 19:24:29 DEBUG --- status code --- 19:24:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:29 DEBUG --- http response --- 19:24:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 19:24:29 DEBUG ----- output ----- 19:24:29 DEBUG { 19:24:29 DEBUG "status": "success", 19:24:29 DEBUG "data": { 19:24:29 DEBUG "resultType": "vector", 19:24:29 DEBUG "result": [ 19:24:29 DEBUG { 19:24:29 DEBUG "metric": { 19:24:29 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:24:29 DEBUG }, 19:24:29 DEBUG "value": [ 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG [...] 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG "pod": "idm-b85f9d978-wnhxb" 19:24:29 DEBUG }, 19:24:29 DEBUG "value": [ 19:24:29 DEBUG 1713551069, 19:24:29 DEBUG "0" 19:24:29 DEBUG ] 19:24:29 DEBUG } 19:24:29 DEBUG ] 19:24:29 DEBUG } 19:24:29 DEBUG } 19:24:29 INFO [http_cmd]: http status code OK 19:24:29 DEBUG --- status code --- 19:24:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:29 DEBUG --- http response --- 19:24:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:24:29 DEBUG ----- output ----- 19:24:29 DEBUG { 19:24:29 DEBUG "status": "success", 19:24:29 DEBUG "data": { 19:24:29 DEBUG "resultType": "vector", 19:24:29 DEBUG "result": [ 19:24:29 DEBUG { 19:24:29 DEBUG "metric": { 19:24:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:24:29 DEBUG }, 19:24:29 DEBUG "value": [ 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG [...] 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:24:29 DEBUG }, 19:24:29 DEBUG "value": [ 19:24:29 DEBUG 1713551069, 19:24:29 DEBUG "3384885248" 19:24:29 DEBUG ] 19:24:29 DEBUG } 19:24:29 DEBUG ] 19:24:29 DEBUG } 19:24:29 DEBUG } 19:24:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551069" 19:24:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551069" 19:24:29 INFO [http_cmd]: http status code OK 19:24:29 DEBUG --- status code --- 19:24:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:29 DEBUG --- http response --- 19:24:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:24:29 DEBUG ----- output ----- 19:24:29 DEBUG { 19:24:29 DEBUG "status": "success", 19:24:29 DEBUG "data": { 19:24:29 DEBUG "resultType": "vector", 19:24:29 DEBUG "result": [ 19:24:29 DEBUG { 19:24:29 DEBUG "metric": { 19:24:29 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:24:29 DEBUG }, 19:24:29 DEBUG "value": [ 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG [...] 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:24:29 DEBUG }, 19:24:29 DEBUG "value": [ 19:24:29 DEBUG 1713551069, 19:24:29 DEBUG "0" 19:24:29 DEBUG ] 19:24:29 DEBUG } 19:24:29 DEBUG ] 19:24:29 DEBUG } 19:24:29 DEBUG } 19:24:29 INFO [http_cmd]: http status code OK 19:24:29 DEBUG --- status code --- 19:24:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:29 DEBUG --- http response --- 19:24:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:24:29 DEBUG ----- output ----- 19:24:29 DEBUG { 19:24:29 DEBUG "status": "success", 19:24:29 DEBUG "data": { 19:24:29 DEBUG "resultType": "vector", 19:24:29 DEBUG "result": [ 19:24:29 DEBUG { 19:24:29 DEBUG "metric": { 19:24:29 DEBUG "instance": "10.142.0.107:9100" 19:24:29 DEBUG }, 19:24:29 DEBUG "value": [ 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG [...] 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG 19:24:29 DEBUG "instance": "10.142.0.43:9100" 19:24:29 DEBUG }, 19:24:29 DEBUG "value": [ 19:24:29 DEBUG 1713551069, 19:24:29 DEBUG "70943394.9" 19:24:29 DEBUG ] 19:24:29 DEBUG } 19:24:29 DEBUG ] 19:24:29 DEBUG } 19:24:29 DEBUG } 19:24:29 INFO 19:24:29 INFO [loop_until]: kubectl --namespace=xlou top pods 19:24:29 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:24:29 INFO [loop_until]: OK (rc = 0) 19:24:29 DEBUG --- stdout --- 19:24:29 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 14741m 8996Mi am-767f6b67b5-qcqz2 14739m 8998Mi am-767f6b67b5-vbltz 15512m 8989Mi ds-cts-0 15243m 10838Mi ds-cts-1 14045m 11102Mi ds-cts-2 15305m 10841Mi ds-idrepo-0 6379m 3641Mi ds-idrepo-1 6306m 3507Mi ds-idrepo-2 6317m 3492Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7857m 3135Mi 19:24:29 DEBUG --- stderr --- 19:24:29 DEBUG 19:24:29 INFO 19:24:29 INFO [loop_until]: kubectl --namespace=xlou top node 19:24:29 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:24:29 INFO [loop_until]: OK (rc = 0) 19:24:29 DEBUG --- stdout --- 19:24:29 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 70m 0% 6260Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 6552m 21% 7231Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 14363m 48% 11709Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 85m 0% 3344Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 15276m 51% 13871Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 154m 0% 5201Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 14577m 48% 11877Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12369m 41% 3007Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 16417m 54% 11203Mi 10% gke-xlou-cdm-ds-0f601751-j488 6947m 23% 6730Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 13820m 46% 15926Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 7042m 23% 7408Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 15505m 51% 13349Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 7914m 49% 6940Mi 11% 19:24:29 DEBUG --- stderr --- 19:24:29 DEBUG 19:25:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551128" 19:25:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551128" 19:25:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713551128" 19:25:28 INFO [http_cmd]: http status code OK 19:25:28 DEBUG --- status code --- 19:25:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:28 DEBUG --- http response --- 19:25:28 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:25:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551128" 19:25:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551128" 19:25:28 INFO [http_cmd]: http status code OK 19:25:28 DEBUG --- status code --- 19:25:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:28 DEBUG --- http response --- 19:25:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:25:28 DEBUG ----- output ----- 19:25:28 DEBUG { 19:25:28 DEBUG "status": "success", 19:25:28 DEBUG "data": { 19:25:28 DEBUG "resultType": "vector", 19:25:28 DEBUG "result": [ 19:25:28 DEBUG { 19:25:28 DEBUG "metric": { 19:25:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:25:28 DEBUG }, 19:25:28 DEBUG "value": [ 19:25:28 DEBUG 19:25:28 DEBUG 19:25:28 DEBUG 19:25:28 DEBUG [...] 19:25:28 DEBUG 19:25:28 DEBUG 19:25:28 DEBUG 19:25:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:25:28 DEBUG }, 19:25:28 DEBUG "value": [ 19:25:28 DEBUG 1713551128, 19:25:28 DEBUG "0" 19:25:28 DEBUG ] 19:25:28 DEBUG } 19:25:28 DEBUG ] 19:25:28 DEBUG } 19:25:28 DEBUG } 19:25:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551128" 19:25:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713551128" 19:25:28 INFO [http_cmd]: http status code OK 19:25:28 DEBUG --- status code --- 19:25:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:28 DEBUG --- http response --- 19:25:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:25:28 DEBUG ----- output ----- 19:25:28 DEBUG { 19:25:28 DEBUG "status": "success", 19:25:28 DEBUG "data": { 19:25:28 DEBUG "resultType": "vector", 19:25:28 DEBUG "result": [ 19:25:28 DEBUG { 19:25:28 DEBUG "metric": { 19:25:28 DEBUG "instance": "10.142.0.107:9100" 19:25:28 DEBUG }, 19:25:28 DEBUG "value": [ 19:25:28 DEBUG 19:25:28 DEBUG 19:25:28 DEBUG 19:25:28 DEBUG [...] 19:25:28 DEBUG 19:25:28 DEBUG 19:25:28 DEBUG 19:25:28 DEBUG "instance": "10.142.0.43:9100" 19:25:28 DEBUG }, 19:25:28 DEBUG "value": [ 19:25:28 DEBUG 1713551128, 19:25:28 DEBUG "1880532597760" 19:25:28 DEBUG ] 19:25:28 DEBUG } 19:25:28 DEBUG ] 19:25:28 DEBUG } 19:25:28 DEBUG } 19:25:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713551128" 19:25:28 INFO [http_cmd]: http status code OK 19:25:28 DEBUG --- status code --- 19:25:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:28 DEBUG --- http response --- 19:25:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:25:28 DEBUG ----- output ----- 19:25:28 DEBUG { 19:25:28 DEBUG "status": "success", 19:25:28 DEBUG "data": { 19:25:28 DEBUG "resultType": "vector", 19:25:28 DEBUG "result": [ 19:25:28 DEBUG { 19:25:28 DEBUG "metric": { 19:25:28 DEBUG "pod": "ds-idrepo-0" 19:25:28 DEBUG }, 19:25:28 DEBUG "value": [ 19:25:28 DEBUG 19:25:28 DEBUG 19:25:28 DEBUG 19:25:28 DEBUG [...] 19:25:28 DEBUG 19:25:28 DEBUG 19:25:28 DEBUG 19:25:28 DEBUG "pod": "ds-idrepo-2" 19:25:28 DEBUG }, 19:25:28 DEBUG "value": [ 19:25:28 DEBUG 1713551128, 19:25:28 DEBUG "0" 19:25:28 DEBUG ] 19:25:28 DEBUG } 19:25:28 DEBUG ] 19:25:28 DEBUG } 19:25:28 DEBUG } 19:25:28 INFO [http_cmd]: http status code OK 19:25:28 DEBUG --- status code --- 19:25:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:28 DEBUG --- http response --- 19:25:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:25:28 INFO [http_cmd]: http status code OK 19:25:28 DEBUG ----- output ----- 19:25:28 DEBUG --- status code --- 19:25:28 DEBUG { 19:25:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:28 DEBUG "status": "success", 19:25:28 DEBUG --- http response --- 19:25:28 DEBUG "data": { 19:25:28 DEBUG "resultType": "vector", 19:25:28 DEBUG "result": [ 19:25:28 DEBUG { 19:25:28 DEBUG "metric": { 19:25:28 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:25:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:25:28 DEBUG }, 19:25:28 DEBUG ----- output ----- 19:25:28 DEBUG "value": [ 19:25:28 INFO [http_cmd]: http status code OK 19:25:28 DEBUG { 19:25:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551128" 19:25:28 DEBUG 19:25:28 DEBUG --- status code --- 19:25:28 DEBUG "status": "success", 19:25:28 DEBUG 19:25:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:28 DEBUG "data": { 19:25:28 DEBUG 19:25:28 DEBUG --- http response --- 19:25:28 DEBUG "resultType": "vector", 19:25:28 DEBUG [...] 19:25:28 DEBUG "result": [ 19:25:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:25:28 DEBUG 19:25:28 DEBUG { 19:25:28 DEBUG ----- output ----- 19:25:28 DEBUG 19:25:28 DEBUG "metric": { 19:25:28 DEBUG { 19:25:28 DEBUG 19:25:28 DEBUG "pod": "ds-cts-0" 19:25:28 DEBUG "status": "success", 19:25:28 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:25:28 DEBUG }, 19:25:28 DEBUG "data": { 19:25:28 DEBUG }, 19:25:28 DEBUG "value": [ 19:25:28 DEBUG "resultType": "vector", 19:25:28 DEBUG "value": [ 19:25:28 DEBUG 19:25:28 DEBUG "result": [ 19:25:28 DEBUG 1713551128, 19:25:28 DEBUG 19:25:28 DEBUG { 19:25:28 DEBUG "7.720732344499927" 19:25:28 DEBUG 19:25:28 DEBUG "metric": { 19:25:28 DEBUG ] 19:25:28 DEBUG [...] 19:25:28 ERROR [http_cmd]: ERROR 19:25:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551128" 19:25:28 DEBUG "pod": "ds-idrepo-0" 19:25:28 DEBUG } 19:25:28 DEBUG 19:25:28 ERROR --- status code --- 19:25:28 DEBUG }, 19:25:28 DEBUG ] 19:25:28 DEBUG 19:25:28 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:25:28 DEBUG "value": [ 19:25:28 DEBUG } 19:25:28 DEBUG 19:25:28 ERROR --- http response --- 19:25:28 DEBUG 19:25:28 DEBUG } 19:25:28 DEBUG "pod": "ds-cts-2" 19:25:28 DEBUG 19:25:28 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:25:28 DEBUG }, 19:25:28 DEBUG 19:25:28 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:25:28 DEBUG "value": [ 19:25:28 DEBUG [...] 19:25:28 DEBUG 1713551128, 19:25:28 DEBUG 19:25:28 DEBUG "0" 19:25:28 DEBUG 19:25:28 DEBUG ] 19:25:28 DEBUG 19:25:28 DEBUG } 19:25:28 DEBUG "pod": "ds-cts-2" 19:25:28 DEBUG ] 19:25:28 DEBUG }, 19:25:28 DEBUG } 19:25:28 DEBUG "value": [ 19:25:28 DEBUG } 19:25:28 DEBUG 1713551128, 19:25:28 DEBUG "0" 19:25:28 DEBUG ] 19:25:28 DEBUG } 19:25:28 DEBUG ] 19:25:28 DEBUG } 19:25:28 DEBUG } 19:25:28 INFO [http_cmd]: http status code OK 19:25:28 DEBUG --- status code --- 19:25:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:28 DEBUG --- http response --- 19:25:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:25:28 DEBUG ----- output ----- 19:25:28 DEBUG { 19:25:28 DEBUG "status": "success", 19:25:28 DEBUG "data": { 19:25:28 DEBUG "resultType": "vector", 19:25:28 DEBUG "result": [ 19:25:28 DEBUG { 19:25:28 DEBUG "metric": { 19:25:28 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:25:28 DEBUG }, 19:25:28 DEBUG "value": [ 19:25:28 DEBUG 19:25:28 DEBUG 19:25:28 DEBUG 19:25:28 DEBUG [...] 19:25:28 DEBUG 19:25:28 DEBUG 19:25:28 DEBUG 19:25:28 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:25:28 DEBUG }, 19:25:28 DEBUG "value": [ 19:25:28 DEBUG 1713551128, 19:25:28 DEBUG "0" 19:25:28 DEBUG ] 19:25:28 DEBUG } 19:25:28 DEBUG ] 19:25:28 DEBUG } 19:25:28 DEBUG } 19:25:28 INFO [http_cmd]: http status code OK 19:25:28 DEBUG --- status code --- 19:25:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:28 DEBUG --- http response --- 19:25:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:25:28 DEBUG ----- output ----- 19:25:28 DEBUG { 19:25:28 DEBUG "status": "success", 19:25:28 DEBUG "data": { 19:25:28 DEBUG "resultType": "vector", 19:25:28 DEBUG "result": [ 19:25:28 DEBUG { 19:25:28 DEBUG "metric": { 19:25:28 DEBUG "pod": "ds-idrepo-0" 19:25:28 DEBUG }, 19:25:28 DEBUG "value": [ 19:25:28 DEBUG 19:25:28 DEBUG 19:25:28 DEBUG 19:25:28 DEBUG [...] 19:25:28 DEBUG 19:25:28 DEBUG 19:25:28 DEBUG 19:25:28 DEBUG "pod": "ds-idrepo-2" 19:25:28 DEBUG }, 19:25:28 DEBUG "value": [ 19:25:28 DEBUG 1713551128, 19:25:28 DEBUG "0" 19:25:28 DEBUG ] 19:25:28 DEBUG } 19:25:28 DEBUG ] 19:25:28 DEBUG } 19:25:28 DEBUG } 19:25:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551128" 19:25:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551128" 19:25:28 INFO [http_cmd]: http status code OK 19:25:28 INFO [http_cmd]: http status code OK 19:25:28 DEBUG --- status code --- 19:25:28 DEBUG --- status code --- 19:25:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:28 DEBUG --- http response --- 19:25:28 DEBUG --- http response --- 19:25:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:25:28 DEBUG ----- output ----- 19:25:28 DEBUG { 19:25:28 DEBUG "status": "success", 19:25:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:25:28 DEBUG "data": { 19:25:28 DEBUG ----- output ----- 19:25:28 DEBUG "resultType": "vector", 19:25:28 DEBUG { 19:25:28 DEBUG "result": [ 19:25:28 DEBUG "status": "success", 19:25:28 DEBUG { 19:25:28 DEBUG "data": { 19:25:28 DEBUG "metric": { 19:25:28 DEBUG "resultType": "vector", 19:25:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:25:28 DEBUG "result": [ 19:25:28 DEBUG }, 19:25:28 DEBUG { 19:25:28 DEBUG "value": [ 19:25:28 DEBUG "metric": { 19:25:28 DEBUG 19:25:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:25:28 DEBUG 19:25:28 DEBUG }, 19:25:28 DEBUG 19:25:28 DEBUG "value": [ 19:25:28 DEBUG [...] 19:25:28 DEBUG 19:25:28 DEBUG 19:25:28 DEBUG 19:25:28 DEBUG 19:25:28 DEBUG 19:25:28 DEBUG 19:25:28 DEBUG [...] 19:25:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:25:28 DEBUG 19:25:28 DEBUG }, 19:25:28 DEBUG 19:25:28 DEBUG "value": [ 19:25:28 DEBUG 19:25:28 DEBUG 1713551128, 19:25:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:25:28 DEBUG "2325.0666666666666" 19:25:28 DEBUG }, 19:25:28 DEBUG ] 19:25:28 DEBUG "value": [ 19:25:28 DEBUG } 19:25:28 DEBUG 1713551128, 19:25:28 DEBUG ] 19:25:28 DEBUG "2330.6" 19:25:28 DEBUG } 19:25:28 DEBUG ] 19:25:28 DEBUG } 19:25:28 DEBUG } 19:25:28 DEBUG ] 19:25:28 DEBUG } 19:25:28 DEBUG } 19:25:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551129" 19:25:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551129" 19:25:29 INFO [http_cmd]: http status code OK 19:25:29 DEBUG --- status code --- 19:25:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:29 DEBUG --- http response --- 19:25:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:25:29 DEBUG ----- output ----- 19:25:29 DEBUG { 19:25:29 DEBUG "status": "success", 19:25:29 DEBUG "data": { 19:25:29 DEBUG "resultType": "vector", 19:25:29 DEBUG "result": [ 19:25:29 DEBUG { 19:25:29 DEBUG "metric": { 19:25:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:25:29 DEBUG }, 19:25:29 DEBUG "value": [ 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG [...] 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:25:29 DEBUG }, 19:25:29 DEBUG "value": [ 19:25:29 DEBUG 1713551129, 19:25:29 DEBUG "420733" 19:25:29 DEBUG ] 19:25:29 DEBUG } 19:25:29 DEBUG ] 19:25:29 DEBUG } 19:25:29 DEBUG } 19:25:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551129" 19:25:29 INFO [http_cmd]: http status code OK 19:25:29 DEBUG --- status code --- 19:25:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:29 DEBUG --- http response --- 19:25:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:25:29 DEBUG ----- output ----- 19:25:29 DEBUG { 19:25:29 DEBUG "status": "success", 19:25:29 DEBUG "data": { 19:25:29 DEBUG "resultType": "vector", 19:25:29 DEBUG "result": [ 19:25:29 DEBUG { 19:25:29 DEBUG "metric": { 19:25:29 DEBUG "pod": "ds-idrepo-0" 19:25:29 DEBUG }, 19:25:29 DEBUG "value": [ 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG [...] 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG "pod": "ds-idrepo-2" 19:25:29 DEBUG }, 19:25:29 DEBUG "value": [ 19:25:29 DEBUG 1713551129, 19:25:29 DEBUG "0" 19:25:29 DEBUG ] 19:25:29 DEBUG } 19:25:29 DEBUG ] 19:25:29 DEBUG } 19:25:29 DEBUG } 19:25:29 INFO [http_cmd]: http status code OK 19:25:29 DEBUG --- status code --- 19:25:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:29 DEBUG --- http response --- 19:25:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:25:29 DEBUG ----- output ----- 19:25:29 DEBUG { 19:25:29 DEBUG "status": "success", 19:25:29 DEBUG "data": { 19:25:29 DEBUG "resultType": "vector", 19:25:29 DEBUG "result": [ 19:25:29 DEBUG { 19:25:29 DEBUG "metric": { 19:25:29 DEBUG "instance": "10.142.0.107:9100" 19:25:29 DEBUG }, 19:25:29 DEBUG "value": [ 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG [...] 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG "instance": "10.142.0.43:9100" 19:25:29 DEBUG }, 19:25:29 DEBUG "value": [ 19:25:29 DEBUG 1713551129, 19:25:29 DEBUG "0.11133333333333059" 19:25:29 DEBUG ] 19:25:29 DEBUG } 19:25:29 DEBUG ] 19:25:29 DEBUG } 19:25:29 DEBUG } 19:25:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551129" 19:25:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551129" 19:25:29 INFO [http_cmd]: http status code OK 19:25:29 DEBUG --- status code --- 19:25:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:29 DEBUG --- http response --- 19:25:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:25:29 DEBUG ----- output ----- 19:25:29 DEBUG { 19:25:29 DEBUG "status": "success", 19:25:29 DEBUG "data": { 19:25:29 DEBUG "resultType": "vector", 19:25:29 DEBUG "result": [ 19:25:29 DEBUG { 19:25:29 DEBUG "metric": { 19:25:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:25:29 DEBUG }, 19:25:29 DEBUG "value": [ 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG [...] 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:25:29 DEBUG }, 19:25:29 DEBUG "value": [ 19:25:29 DEBUG 1713551129, 19:25:29 DEBUG "2330.4" 19:25:29 DEBUG ] 19:25:29 DEBUG } 19:25:29 DEBUG ] 19:25:29 DEBUG } 19:25:29 DEBUG } 19:25:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551129" 19:25:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551129" 19:25:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551129" 19:25:29 INFO [http_cmd]: http status code OK 19:25:29 DEBUG --- status code --- 19:25:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:29 DEBUG --- http response --- 19:25:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:25:29 DEBUG ----- output ----- 19:25:29 DEBUG { 19:25:29 DEBUG "status": "success", 19:25:29 DEBUG "data": { 19:25:29 DEBUG "resultType": "vector", 19:25:29 DEBUG "result": [ 19:25:29 DEBUG { 19:25:29 DEBUG "metric": { 19:25:29 DEBUG "instance": "10.142.0.107:9100" 19:25:29 DEBUG }, 19:25:29 DEBUG "value": [ 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG [...] 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG "instance": "10.142.0.43:9100" 19:25:29 DEBUG }, 19:25:29 DEBUG "value": [ 19:25:29 DEBUG 1713551129, 19:25:29 DEBUG "0.27313333333337747" 19:25:29 DEBUG ] 19:25:29 DEBUG } 19:25:29 DEBUG ] 19:25:29 DEBUG } 19:25:29 DEBUG } 19:25:29 INFO [http_cmd]: http status code OK 19:25:29 DEBUG --- status code --- 19:25:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:29 DEBUG --- http response --- 19:25:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:25:29 INFO [http_cmd]: http status code OK 19:25:29 DEBUG ----- output ----- 19:25:29 DEBUG --- status code --- 19:25:29 DEBUG { 19:25:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:29 DEBUG "status": "success", 19:25:29 DEBUG --- http response --- 19:25:29 DEBUG "data": { 19:25:29 DEBUG "resultType": "vector", 19:25:29 DEBUG "result": [ 19:25:29 DEBUG { 19:25:29 DEBUG "metric": { 19:25:29 DEBUG "pod": "ds-cts-0" 19:25:29 DEBUG }, 19:25:29 DEBUG "value": [ 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG [...] 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG "pod": "ds-cts-2" 19:25:29 DEBUG }, 19:25:29 DEBUG "value": [ 19:25:29 DEBUG 1713551129, 19:25:29 DEBUG "0" 19:25:29 DEBUG ] 19:25:29 DEBUG } 19:25:29 DEBUG ] 19:25:29 DEBUG } 19:25:29 DEBUG } 19:25:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:25:29 DEBUG ----- output ----- 19:25:29 DEBUG { 19:25:29 DEBUG "status": "success", 19:25:29 DEBUG "data": { 19:25:29 DEBUG "resultType": "vector", 19:25:29 DEBUG "result": [ 19:25:29 DEBUG { 19:25:29 DEBUG "metric": { 19:25:29 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:25:29 DEBUG }, 19:25:29 DEBUG "value": [ 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG [...] 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:25:29 DEBUG }, 19:25:29 DEBUG "value": [ 19:25:29 DEBUG 1713551129, 19:25:29 DEBUG "35242.78495665919" 19:25:29 DEBUG ] 19:25:29 DEBUG } 19:25:29 DEBUG ] 19:25:29 DEBUG } 19:25:29 DEBUG } 19:25:29 INFO [http_cmd]: http status code OK 19:25:29 DEBUG --- status code --- 19:25:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:29 DEBUG --- http response --- 19:25:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:25:29 DEBUG ----- output ----- 19:25:29 DEBUG { 19:25:29 DEBUG "status": "success", 19:25:29 DEBUG "data": { 19:25:29 DEBUG "resultType": "vector", 19:25:29 DEBUG "result": [ 19:25:29 DEBUG { 19:25:29 DEBUG "metric": { 19:25:29 DEBUG "instance": "10.142.0.107:9100" 19:25:29 DEBUG }, 19:25:29 DEBUG "value": [ 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG [...] 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG "instance": "10.142.0.43:9100" 19:25:29 DEBUG }, 19:25:29 DEBUG "value": [ 19:25:29 DEBUG 1713551129, 19:25:29 DEBUG "70528075.23333333" 19:25:29 DEBUG ] 19:25:29 DEBUG } 19:25:29 DEBUG ] 19:25:29 DEBUG } 19:25:29 DEBUG } 19:25:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713551129" 19:25:29 INFO [http_cmd]: http status code OK 19:25:29 DEBUG --- status code --- 19:25:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:29 DEBUG --- http response --- 19:25:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:25:29 DEBUG ----- output ----- 19:25:29 DEBUG { 19:25:29 DEBUG "status": "success", 19:25:29 DEBUG "data": { 19:25:29 DEBUG "resultType": "vector", 19:25:29 DEBUG "result": [ 19:25:29 DEBUG { 19:25:29 DEBUG "metric": { 19:25:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:25:29 DEBUG }, 19:25:29 DEBUG "value": [ 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG [...] 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:25:29 DEBUG }, 19:25:29 DEBUG "value": [ 19:25:29 DEBUG 1713551129, 19:25:29 DEBUG "0.005" 19:25:29 DEBUG ] 19:25:29 DEBUG } 19:25:29 DEBUG ] 19:25:29 DEBUG } 19:25:29 DEBUG } 19:25:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551129" 19:25:29 INFO [http_cmd]: http status code OK 19:25:29 DEBUG --- status code --- 19:25:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:29 DEBUG --- http response --- 19:25:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:25:29 DEBUG ----- output ----- 19:25:29 DEBUG { 19:25:29 DEBUG "status": "success", 19:25:29 DEBUG "data": { 19:25:29 DEBUG "resultType": "vector", 19:25:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551129" 19:25:29 DEBUG "result": [ 19:25:29 DEBUG { 19:25:29 DEBUG "metric": { 19:25:29 DEBUG "pod": "ds-cts-0" 19:25:29 DEBUG }, 19:25:29 DEBUG "value": [ 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG [...] 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG "pod": "ds-cts-2" 19:25:29 DEBUG }, 19:25:29 DEBUG "value": [ 19:25:29 DEBUG 1713551129, 19:25:29 DEBUG "0" 19:25:29 DEBUG ] 19:25:29 DEBUG } 19:25:29 DEBUG ] 19:25:29 DEBUG } 19:25:29 DEBUG } 19:25:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551129" 19:25:29 INFO [http_cmd]: http status code OK 19:25:29 DEBUG --- status code --- 19:25:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:29 DEBUG --- http response --- 19:25:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:25:29 DEBUG ----- output ----- 19:25:29 DEBUG { 19:25:29 DEBUG "status": "success", 19:25:29 DEBUG "data": { 19:25:29 DEBUG "resultType": "vector", 19:25:29 DEBUG "result": [ 19:25:29 DEBUG { 19:25:29 DEBUG "metric": { 19:25:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:25:29 DEBUG }, 19:25:29 DEBUG "value": [ 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG [...] 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:25:29 DEBUG }, 19:25:29 DEBUG "value": [ 19:25:29 DEBUG 1713551129, 19:25:29 DEBUG "2330.9666666666667" 19:25:29 DEBUG ] 19:25:29 DEBUG } 19:25:29 DEBUG ] 19:25:29 DEBUG } 19:25:29 DEBUG } 19:25:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713551129" 19:25:29 INFO [http_cmd]: http status code OK 19:25:29 DEBUG --- status code --- 19:25:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:29 DEBUG --- http response --- 19:25:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:25:29 DEBUG ----- output ----- 19:25:29 DEBUG { 19:25:29 DEBUG "status": "success", 19:25:29 DEBUG "data": { 19:25:29 DEBUG "resultType": "vector", 19:25:29 DEBUG "result": [ 19:25:29 DEBUG { 19:25:29 DEBUG "metric": { 19:25:29 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:25:29 DEBUG }, 19:25:29 DEBUG "value": [ 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG [...] 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:25:29 DEBUG }, 19:25:29 DEBUG "value": [ 19:25:29 DEBUG 1713551129, 19:25:29 DEBUG "0" 19:25:29 DEBUG ] 19:25:29 DEBUG } 19:25:29 DEBUG ] 19:25:29 DEBUG } 19:25:29 DEBUG } 19:25:29 INFO [http_cmd]: http status code OK 19:25:29 DEBUG --- status code --- 19:25:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:29 DEBUG --- http response --- 19:25:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:25:29 DEBUG ----- output ----- 19:25:29 DEBUG { 19:25:29 DEBUG "status": "success", 19:25:29 DEBUG "data": { 19:25:29 DEBUG "resultType": "vector", 19:25:29 DEBUG "result": [ 19:25:29 DEBUG { 19:25:29 DEBUG "metric": { 19:25:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:25:29 DEBUG }, 19:25:29 DEBUG "value": [ 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG [...] 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:25:29 DEBUG }, 19:25:29 DEBUG "value": [ 19:25:29 DEBUG 1713551129, 19:25:29 DEBUG "3386445824" 19:25:29 DEBUG ] 19:25:29 DEBUG } 19:25:29 DEBUG ] 19:25:29 DEBUG } 19:25:29 DEBUG } 19:25:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551129" 19:25:29 INFO [http_cmd]: http status code OK 19:25:29 DEBUG --- status code --- 19:25:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:29 DEBUG --- http response --- 19:25:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:25:29 DEBUG ----- output ----- 19:25:29 DEBUG { 19:25:29 DEBUG "status": "success", 19:25:29 DEBUG "data": { 19:25:29 DEBUG "resultType": "vector", 19:25:29 DEBUG "result": [ 19:25:29 DEBUG { 19:25:29 DEBUG "metric": { 19:25:29 DEBUG "instance": "10.142.0.107:9100" 19:25:29 DEBUG }, 19:25:29 DEBUG "value": [ 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG [...] 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG "instance": "10.142.0.43:9100" 19:25:29 DEBUG }, 19:25:29 DEBUG "value": [ 19:25:29 DEBUG 1713551129, 19:25:29 DEBUG "70525835.16666666" 19:25:29 DEBUG ] 19:25:29 DEBUG } 19:25:29 DEBUG ] 19:25:29 DEBUG } 19:25:29 DEBUG } 19:25:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551129" 19:25:29 INFO 19:25:29 INFO [loop_until]: kubectl --namespace=xlou top pods 19:25:29 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:25:29 INFO [http_cmd]: http status code OK 19:25:29 DEBUG --- status code --- 19:25:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:29 DEBUG --- http response --- 19:25:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:25:29 DEBUG ----- output ----- 19:25:29 DEBUG { 19:25:29 DEBUG "status": "success", 19:25:29 DEBUG "data": { 19:25:29 DEBUG "resultType": "vector", 19:25:29 DEBUG "result": [ 19:25:29 DEBUG { 19:25:29 DEBUG "metric": { 19:25:29 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:25:29 DEBUG }, 19:25:29 DEBUG "value": [ 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG [...] 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG 19:25:29 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:25:29 DEBUG }, 19:25:29 DEBUG "value": [ 19:25:29 DEBUG 1713551129, 19:25:29 DEBUG "3.6994665497147285" 19:25:29 DEBUG ] 19:25:29 DEBUG } 19:25:29 DEBUG ] 19:25:29 DEBUG } 19:25:29 DEBUG } 19:25:29 INFO [loop_until]: OK (rc = 0) 19:25:29 DEBUG --- stdout --- 19:25:29 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 14844m 8997Mi am-767f6b67b5-qcqz2 14533m 8999Mi am-767f6b67b5-vbltz 15560m 8983Mi ds-cts-0 15181m 10843Mi ds-cts-1 14014m 11102Mi ds-cts-2 16098m 10854Mi ds-idrepo-0 6504m 3647Mi ds-idrepo-1 6301m 3518Mi ds-idrepo-2 6189m 3491Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1175Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 1m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7574m 3136Mi 19:25:29 DEBUG --- stderr --- 19:25:29 DEBUG 19:25:29 INFO 19:25:29 INFO [loop_until]: kubectl --namespace=xlou top node 19:25:29 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:25:29 INFO [loop_until]: OK (rc = 0) 19:25:29 DEBUG --- stdout --- 19:25:29 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 71m 0% 6252Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 6495m 21% 7233Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 14270m 47% 11714Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 86m 0% 3353Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 15281m 51% 13890Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 153m 0% 5222Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 14544m 48% 11880Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12319m 41% 3004Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 16394m 54% 11204Mi 10% gke-xlou-cdm-ds-0f601751-j488 6911m 23% 6733Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 13395m 44% 15932Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 7015m 23% 7409Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 15973m 53% 13352Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 7871m 49% 6940Mi 11% 19:25:29 DEBUG --- stderr --- 19:25:29 DEBUG 19:26:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713551188" 19:26:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551188" 19:26:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551188" 19:26:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551188" 19:26:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551188" 19:26:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551188" 19:26:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551188" 19:26:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713551188" 19:26:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551188" 19:26:28 INFO [http_cmd]: http status code OK 19:26:28 DEBUG --- status code --- 19:26:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:28 DEBUG --- http response --- 19:26:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:26:28 DEBUG ----- output ----- 19:26:28 DEBUG { 19:26:28 DEBUG "status": "success", 19:26:28 DEBUG "data": { 19:26:28 DEBUG "resultType": "vector", 19:26:28 DEBUG "result": [ 19:26:28 DEBUG { 19:26:28 DEBUG "metric": { 19:26:28 DEBUG "pod": "ds-idrepo-0" 19:26:28 DEBUG }, 19:26:28 DEBUG "value": [ 19:26:28 DEBUG 19:26:28 DEBUG 19:26:28 DEBUG 19:26:28 DEBUG [...] 19:26:28 DEBUG 19:26:28 DEBUG 19:26:28 DEBUG 19:26:28 DEBUG "pod": "ds-cts-2" 19:26:28 DEBUG }, 19:26:28 DEBUG "value": [ 19:26:28 DEBUG 1713551188, 19:26:28 DEBUG "0" 19:26:28 DEBUG ] 19:26:28 DEBUG } 19:26:28 DEBUG ] 19:26:28 DEBUG } 19:26:28 DEBUG } 19:26:28 INFO [http_cmd]: http status code OK 19:26:28 DEBUG --- status code --- 19:26:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:28 DEBUG --- http response --- 19:26:28 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:26:28 INFO [http_cmd]: http status code OK 19:26:28 DEBUG --- status code --- 19:26:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:28 DEBUG --- http response --- 19:26:28 INFO [http_cmd]: http status code OK 19:26:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:26:28 DEBUG --- status code --- 19:26:28 DEBUG ----- output ----- 19:26:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713551188" 19:26:28 INFO [http_cmd]: http status code OK 19:26:28 INFO [http_cmd]: http status code OK 19:26:28 ERROR [http_cmd]: ERROR 19:26:28 INFO [http_cmd]: http status code OK 19:26:28 INFO [http_cmd]: http status code OK 19:26:28 DEBUG { 19:26:28 DEBUG --- http response --- 19:26:28 DEBUG --- status code --- 19:26:28 DEBUG --- status code --- 19:26:28 ERROR --- status code --- 19:26:28 DEBUG --- status code --- 19:26:28 DEBUG --- status code --- 19:26:28 DEBUG "status": "success", 19:26:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:26:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:28 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:26:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:28 DEBUG "data": { 19:26:28 DEBUG --- http response --- 19:26:28 DEBUG ----- output ----- 19:26:28 DEBUG --- http response --- 19:26:28 ERROR --- http response --- 19:26:28 DEBUG --- http response --- 19:26:28 DEBUG --- http response --- 19:26:28 DEBUG "resultType": "vector", 19:26:28 DEBUG { 19:26:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:26:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:26:28 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:26:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:26:28 DEBUG "result": [ 19:26:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:26:28 DEBUG "status": "success", 19:26:28 DEBUG ----- output ----- 19:26:28 DEBUG ----- output ----- 19:26:28 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:26:28 DEBUG ----- output ----- 19:26:28 DEBUG { 19:26:28 DEBUG ----- output ----- 19:26:28 DEBUG "data": { 19:26:28 DEBUG { 19:26:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551188" 19:26:28 DEBUG { 19:26:28 DEBUG { 19:26:28 DEBUG "metric": { 19:26:28 DEBUG { 19:26:28 DEBUG "resultType": "vector", 19:26:28 DEBUG "status": "success", 19:26:28 DEBUG "status": "success", 19:26:28 DEBUG "status": "success", 19:26:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:26:28 DEBUG "status": "success", 19:26:28 DEBUG "result": [ 19:26:28 DEBUG "data": { 19:26:28 DEBUG "data": { 19:26:28 DEBUG "data": { 19:26:28 DEBUG }, 19:26:28 DEBUG "data": { 19:26:28 DEBUG { 19:26:28 DEBUG "resultType": "vector", 19:26:28 DEBUG "resultType": "vector", 19:26:28 DEBUG "resultType": "vector", 19:26:28 DEBUG "value": [ 19:26:28 DEBUG "resultType": "vector", 19:26:28 DEBUG "metric": { 19:26:28 DEBUG "result": [ 19:26:28 DEBUG "result": [ 19:26:28 DEBUG "result": [ 19:26:28 DEBUG 19:26:28 DEBUG "result": [ 19:26:28 DEBUG "pod": "ds-idrepo-0" 19:26:28 DEBUG { 19:26:28 DEBUG { 19:26:28 DEBUG { 19:26:28 DEBUG 19:26:28 DEBUG { 19:26:28 DEBUG }, 19:26:28 DEBUG "metric": { 19:26:28 DEBUG "metric": { 19:26:28 DEBUG "metric": { 19:26:28 DEBUG 19:26:28 DEBUG "metric": { 19:26:28 DEBUG "value": [ 19:26:28 DEBUG "pod": "am-767f6b67b5-9zw22" 19:26:28 DEBUG "instance": "10.142.0.107:9100" 19:26:28 DEBUG "pod": "ds-idrepo-0" 19:26:28 DEBUG [...] 19:26:28 DEBUG "pod": "ds-cts-0" 19:26:28 DEBUG 19:26:28 DEBUG }, 19:26:28 DEBUG }, 19:26:28 DEBUG }, 19:26:28 DEBUG 19:26:28 DEBUG }, 19:26:28 DEBUG 19:26:28 DEBUG "value": [ 19:26:28 DEBUG "value": [ 19:26:28 DEBUG "value": [ 19:26:28 DEBUG 19:26:28 DEBUG "value": [ 19:26:28 DEBUG 19:26:28 DEBUG 19:26:28 DEBUG 19:26:28 DEBUG 19:26:28 DEBUG 19:26:28 DEBUG 19:26:28 DEBUG [...] 19:26:28 DEBUG 19:26:28 DEBUG 19:26:28 DEBUG 19:26:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:26:28 DEBUG 19:26:28 DEBUG 19:26:28 DEBUG 19:26:28 DEBUG 19:26:28 DEBUG 19:26:28 DEBUG }, 19:26:28 DEBUG 19:26:28 DEBUG 19:26:28 DEBUG [...] 19:26:28 DEBUG [...] 19:26:28 DEBUG [...] 19:26:28 DEBUG "value": [ 19:26:28 DEBUG [...] 19:26:28 DEBUG 19:26:28 DEBUG 19:26:28 DEBUG 19:26:28 DEBUG 19:26:28 DEBUG 1713551188, 19:26:28 DEBUG 19:26:28 DEBUG "pod": "ds-idrepo-2" 19:26:28 DEBUG 19:26:28 DEBUG 19:26:28 DEBUG 19:26:28 DEBUG "0" 19:26:28 DEBUG 19:26:28 DEBUG }, 19:26:28 DEBUG 19:26:28 DEBUG 19:26:28 DEBUG 19:26:28 DEBUG ] 19:26:28 DEBUG 19:26:28 DEBUG "value": [ 19:26:28 DEBUG "pod": "am-767f6b67b5-vbltz" 19:26:28 DEBUG "instance": "10.142.0.43:9100" 19:26:28 DEBUG "pod": "ds-idrepo-2" 19:26:28 DEBUG } 19:26:28 DEBUG "pod": "ds-cts-2" 19:26:28 DEBUG 1713551188, 19:26:28 DEBUG }, 19:26:28 DEBUG "value": [ 19:26:28 DEBUG }, 19:26:28 DEBUG ] 19:26:28 DEBUG }, 19:26:28 DEBUG "0" 19:26:28 DEBUG }, 19:26:28 DEBUG 1713551188, 19:26:28 DEBUG "value": [ 19:26:28 DEBUG } 19:26:28 DEBUG "value": [ 19:26:28 DEBUG ] 19:26:28 DEBUG "value": [ 19:26:28 DEBUG "2333.366666666667" 19:26:28 DEBUG 1713551188, 19:26:28 DEBUG } 19:26:28 DEBUG 1713551188, 19:26:28 DEBUG } 19:26:28 DEBUG 1713551188, 19:26:28 DEBUG ] 19:26:28 DEBUG "1883987667968" 19:26:28 DEBUG "0" 19:26:28 DEBUG ] 19:26:28 DEBUG "0" 19:26:28 DEBUG } 19:26:28 DEBUG ] 19:26:28 DEBUG ] 19:26:28 DEBUG } 19:26:28 DEBUG ] 19:26:28 DEBUG ] 19:26:28 DEBUG } 19:26:28 DEBUG } 19:26:28 DEBUG } 19:26:28 DEBUG } 19:26:28 DEBUG } 19:26:28 DEBUG ] 19:26:28 DEBUG ] 19:26:28 DEBUG ] 19:26:28 DEBUG } 19:26:28 DEBUG } 19:26:28 DEBUG } 19:26:28 DEBUG } 19:26:28 DEBUG } 19:26:28 DEBUG } 19:26:28 DEBUG } 19:26:28 INFO [http_cmd]: http status code OK 19:26:28 DEBUG --- status code --- 19:26:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:28 DEBUG --- http response --- 19:26:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:26:28 DEBUG ----- output ----- 19:26:28 DEBUG { 19:26:28 DEBUG "status": "success", 19:26:28 DEBUG "data": { 19:26:28 DEBUG "resultType": "vector", 19:26:28 DEBUG "result": [ 19:26:28 DEBUG { 19:26:28 DEBUG "metric": { 19:26:28 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:26:28 DEBUG }, 19:26:28 DEBUG "value": [ 19:26:28 DEBUG 19:26:28 DEBUG 19:26:28 DEBUG 19:26:28 DEBUG [...] 19:26:28 DEBUG 19:26:28 DEBUG 19:26:28 DEBUG 19:26:28 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:26:28 DEBUG }, 19:26:28 DEBUG "value": [ 19:26:28 DEBUG 1713551188, 19:26:28 DEBUG "7.748777096613116" 19:26:28 DEBUG ] 19:26:28 DEBUG } 19:26:28 DEBUG ] 19:26:28 DEBUG } 19:26:28 DEBUG } 19:26:28 INFO [http_cmd]: http status code OK 19:26:28 DEBUG --- status code --- 19:26:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:28 DEBUG --- http response --- 19:26:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:26:28 DEBUG ----- output ----- 19:26:28 DEBUG { 19:26:28 DEBUG "status": "success", 19:26:28 DEBUG "data": { 19:26:28 DEBUG "resultType": "vector", 19:26:28 DEBUG "result": [ 19:26:28 DEBUG { 19:26:28 DEBUG "metric": { 19:26:28 DEBUG "pod": "ds-idrepo-0" 19:26:28 DEBUG }, 19:26:28 DEBUG "value": [ 19:26:28 DEBUG 19:26:28 DEBUG 19:26:28 DEBUG 19:26:28 DEBUG [...] 19:26:28 DEBUG 19:26:28 DEBUG 19:26:28 DEBUG 19:26:28 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:26:28 DEBUG }, 19:26:28 DEBUG "value": [ 19:26:28 DEBUG 1713551188, 19:26:28 DEBUG "0" 19:26:28 DEBUG ] 19:26:28 DEBUG } 19:26:28 DEBUG ] 19:26:28 DEBUG } 19:26:28 DEBUG } 19:26:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551189" 19:26:29 INFO [http_cmd]: http status code OK 19:26:29 DEBUG --- status code --- 19:26:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:29 DEBUG --- http response --- 19:26:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:26:29 DEBUG ----- output ----- 19:26:29 DEBUG { 19:26:29 DEBUG "status": "success", 19:26:29 DEBUG "data": { 19:26:29 DEBUG "resultType": "vector", 19:26:29 DEBUG "result": [ 19:26:29 DEBUG { 19:26:29 DEBUG "metric": { 19:26:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:26:29 DEBUG }, 19:26:29 DEBUG "value": [ 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG [...] 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:26:29 DEBUG }, 19:26:29 DEBUG "value": [ 19:26:29 DEBUG 1713551189, 19:26:29 DEBUG "2333.4666666666667" 19:26:29 DEBUG ] 19:26:29 DEBUG } 19:26:29 DEBUG ] 19:26:29 DEBUG } 19:26:29 DEBUG } 19:26:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551189" 19:26:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551189" 19:26:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551189" 19:26:29 INFO [http_cmd]: http status code OK 19:26:29 DEBUG --- status code --- 19:26:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:29 DEBUG --- http response --- 19:26:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:26:29 DEBUG ----- output ----- 19:26:29 DEBUG { 19:26:29 DEBUG "status": "success", 19:26:29 DEBUG "data": { 19:26:29 DEBUG "resultType": "vector", 19:26:29 DEBUG "result": [ 19:26:29 DEBUG { 19:26:29 DEBUG "metric": { 19:26:29 DEBUG "pod": "ds-idrepo-0" 19:26:29 DEBUG }, 19:26:29 DEBUG "value": [ 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG [...] 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG "pod": "ds-cts-0" 19:26:29 DEBUG }, 19:26:29 DEBUG "value": [ 19:26:29 DEBUG 1713551189, 19:26:29 DEBUG "964481.0664227138" 19:26:29 DEBUG ] 19:26:29 DEBUG } 19:26:29 DEBUG ] 19:26:29 DEBUG } 19:26:29 DEBUG } 19:26:29 INFO [http_cmd]: http status code OK 19:26:29 DEBUG --- status code --- 19:26:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:29 DEBUG --- http response --- 19:26:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:26:29 DEBUG ----- output ----- 19:26:29 DEBUG { 19:26:29 DEBUG "status": "success", 19:26:29 DEBUG "data": { 19:26:29 DEBUG "resultType": "vector", 19:26:29 DEBUG "result": [ 19:26:29 DEBUG { 19:26:29 DEBUG "metric": { 19:26:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:26:29 DEBUG }, 19:26:29 DEBUG "value": [ 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG [...] 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:26:29 DEBUG }, 19:26:29 DEBUG "value": [ 19:26:29 DEBUG 1713551189, 19:26:29 DEBUG "2334.8333333333335" 19:26:29 DEBUG ] 19:26:29 DEBUG } 19:26:29 DEBUG ] 19:26:29 DEBUG } 19:26:29 DEBUG } 19:26:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551189" 19:26:29 INFO [http_cmd]: http status code OK 19:26:29 DEBUG --- status code --- 19:26:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:29 DEBUG --- http response --- 19:26:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:26:29 DEBUG ----- output ----- 19:26:29 DEBUG { 19:26:29 DEBUG "status": "success", 19:26:29 DEBUG "data": { 19:26:29 DEBUG "resultType": "vector", 19:26:29 DEBUG "result": [ 19:26:29 DEBUG { 19:26:29 DEBUG "metric": { 19:26:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:26:29 DEBUG }, 19:26:29 DEBUG "value": [ 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG [...] 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:26:29 DEBUG }, 19:26:29 DEBUG "value": [ 19:26:29 DEBUG 1713551189, 19:26:29 DEBUG "420910.5" 19:26:29 DEBUG ] 19:26:29 DEBUG } 19:26:29 DEBUG ] 19:26:29 DEBUG } 19:26:29 DEBUG } 19:26:29 INFO [http_cmd]: http status code OK 19:26:29 DEBUG --- status code --- 19:26:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:29 DEBUG --- http response --- 19:26:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:26:29 DEBUG ----- output ----- 19:26:29 DEBUG { 19:26:29 DEBUG "status": "success", 19:26:29 DEBUG "data": { 19:26:29 DEBUG "resultType": "vector", 19:26:29 DEBUG "result": [ 19:26:29 DEBUG { 19:26:29 DEBUG "metric": { 19:26:29 DEBUG "pod": "ds-idrepo-0" 19:26:29 DEBUG }, 19:26:29 DEBUG "value": [ 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG [...] 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG "pod": "ds-idrepo-2" 19:26:29 DEBUG }, 19:26:29 DEBUG "value": [ 19:26:29 DEBUG 1713551189, 19:26:29 DEBUG "0" 19:26:29 DEBUG ] 19:26:29 DEBUG } 19:26:29 DEBUG ] 19:26:29 DEBUG } 19:26:29 DEBUG } 19:26:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551189" 19:26:29 INFO [http_cmd]: http status code OK 19:26:29 DEBUG --- status code --- 19:26:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:29 DEBUG --- http response --- 19:26:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:26:29 DEBUG ----- output ----- 19:26:29 DEBUG { 19:26:29 DEBUG "status": "success", 19:26:29 DEBUG "data": { 19:26:29 DEBUG "resultType": "vector", 19:26:29 DEBUG "result": [ 19:26:29 DEBUG { 19:26:29 DEBUG "metric": { 19:26:29 DEBUG "instance": "10.142.0.107:9100" 19:26:29 DEBUG }, 19:26:29 DEBUG "value": [ 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG [...] 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG "instance": "10.142.0.43:9100" 19:26:29 DEBUG }, 19:26:29 DEBUG "value": [ 19:26:29 DEBUG 1713551189, 19:26:29 DEBUG "0.07433333333333206" 19:26:29 DEBUG ] 19:26:29 DEBUG } 19:26:29 DEBUG ] 19:26:29 DEBUG } 19:26:29 DEBUG } 19:26:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551189" 19:26:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551189" 19:26:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551189" 19:26:29 INFO [http_cmd]: http status code OK 19:26:29 DEBUG --- status code --- 19:26:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:29 DEBUG --- http response --- 19:26:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:26:29 DEBUG ----- output ----- 19:26:29 DEBUG { 19:26:29 DEBUG "status": "success", 19:26:29 DEBUG "data": { 19:26:29 DEBUG "resultType": "vector", 19:26:29 DEBUG "result": [ 19:26:29 DEBUG { 19:26:29 DEBUG "metric": { 19:26:29 DEBUG "instance": "10.142.0.107:9100" 19:26:29 DEBUG }, 19:26:29 DEBUG "value": [ 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG [...] 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG "instance": "10.142.0.43:9100" 19:26:29 DEBUG }, 19:26:29 DEBUG "value": [ 19:26:29 DEBUG 1713551189, 19:26:29 DEBUG "0.1660333333333786" 19:26:29 DEBUG ] 19:26:29 DEBUG } 19:26:29 DEBUG ] 19:26:29 DEBUG } 19:26:29 DEBUG } 19:26:29 INFO [http_cmd]: http status code OK 19:26:29 DEBUG --- status code --- 19:26:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:29 DEBUG --- http response --- 19:26:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:26:29 DEBUG ----- output ----- 19:26:29 DEBUG { 19:26:29 DEBUG "status": "success", 19:26:29 DEBUG "data": { 19:26:29 INFO [http_cmd]: http status code OK 19:26:29 DEBUG "resultType": "vector", 19:26:29 DEBUG --- status code --- 19:26:29 DEBUG "result": [ 19:26:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:29 DEBUG { 19:26:29 DEBUG --- http response --- 19:26:29 DEBUG "metric": { 19:26:29 DEBUG "pod": "ds-cts-0" 19:26:29 DEBUG }, 19:26:29 DEBUG "value": [ 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:26:29 DEBUG [...] 19:26:29 DEBUG ----- output ----- 19:26:29 DEBUG 19:26:29 DEBUG { 19:26:29 DEBUG 19:26:29 DEBUG "status": "success", 19:26:29 DEBUG 19:26:29 DEBUG "data": { 19:26:29 DEBUG "pod": "ds-cts-2" 19:26:29 DEBUG "resultType": "vector", 19:26:29 DEBUG }, 19:26:29 DEBUG "result": [ 19:26:29 DEBUG "value": [ 19:26:29 DEBUG { 19:26:29 DEBUG 1713551189, 19:26:29 DEBUG "metric": { 19:26:29 DEBUG "0" 19:26:29 DEBUG "instance": "10.142.0.107:9100" 19:26:29 DEBUG ] 19:26:29 DEBUG }, 19:26:29 DEBUG } 19:26:29 DEBUG "value": [ 19:26:29 DEBUG ] 19:26:29 DEBUG 19:26:29 DEBUG } 19:26:29 DEBUG 19:26:29 DEBUG } 19:26:29 DEBUG 19:26:29 DEBUG [...] 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG "instance": "10.142.0.43:9100" 19:26:29 DEBUG }, 19:26:29 DEBUG "value": [ 19:26:29 DEBUG 1713551189, 19:26:29 DEBUG "70719960.9" 19:26:29 DEBUG ] 19:26:29 DEBUG } 19:26:29 DEBUG ] 19:26:29 DEBUG } 19:26:29 DEBUG } 19:26:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713551189" 19:26:29 INFO [http_cmd]: http status code OK 19:26:29 DEBUG --- status code --- 19:26:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:29 DEBUG --- http response --- 19:26:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:26:29 DEBUG ----- output ----- 19:26:29 DEBUG { 19:26:29 DEBUG "status": "success", 19:26:29 DEBUG "data": { 19:26:29 DEBUG "resultType": "vector", 19:26:29 DEBUG "result": [ 19:26:29 DEBUG { 19:26:29 DEBUG "metric": { 19:26:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:26:29 DEBUG }, 19:26:29 DEBUG "value": [ 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG [...] 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:26:29 DEBUG }, 19:26:29 DEBUG "value": [ 19:26:29 DEBUG 1713551189, 19:26:29 DEBUG "0.005" 19:26:29 DEBUG ] 19:26:29 DEBUG } 19:26:29 DEBUG ] 19:26:29 DEBUG } 19:26:29 DEBUG } 19:26:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551189" 19:26:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551189" 19:26:29 INFO [http_cmd]: http status code OK 19:26:29 DEBUG --- status code --- 19:26:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:29 DEBUG --- http response --- 19:26:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:26:29 DEBUG ----- output ----- 19:26:29 DEBUG { 19:26:29 DEBUG "status": "success", 19:26:29 DEBUG "data": { 19:26:29 DEBUG "resultType": "vector", 19:26:29 DEBUG "result": [ 19:26:29 DEBUG { 19:26:29 DEBUG "metric": { 19:26:29 DEBUG "pod": "ds-cts-0" 19:26:29 DEBUG }, 19:26:29 DEBUG "value": [ 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG [...] 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG "pod": "ds-cts-2" 19:26:29 DEBUG }, 19:26:29 DEBUG "value": [ 19:26:29 DEBUG 1713551189, 19:26:29 DEBUG "0.000056431111111111105" 19:26:29 DEBUG ] 19:26:29 DEBUG } 19:26:29 DEBUG ] 19:26:29 DEBUG } 19:26:29 DEBUG } 19:26:29 INFO [http_cmd]: http status code OK 19:26:29 DEBUG --- status code --- 19:26:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:29 DEBUG --- http response --- 19:26:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:26:29 DEBUG ----- output ----- 19:26:29 DEBUG { 19:26:29 DEBUG "status": "success", 19:26:29 DEBUG "data": { 19:26:29 DEBUG "resultType": "vector", 19:26:29 DEBUG "result": [ 19:26:29 DEBUG { 19:26:29 DEBUG "metric": { 19:26:29 DEBUG "pod": "ds-idrepo-0" 19:26:29 DEBUG }, 19:26:29 DEBUG "value": [ 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG [...] 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG "pod": "ds-cts-0" 19:26:29 DEBUG }, 19:26:29 DEBUG "value": [ 19:26:29 DEBUG 1713551189, 19:26:29 DEBUG "5464500.539092365" 19:26:29 DEBUG ] 19:26:29 DEBUG } 19:26:29 DEBUG ] 19:26:29 DEBUG } 19:26:29 DEBUG } 19:26:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551189" 19:26:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551189" 19:26:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713551189" 19:26:29 INFO [http_cmd]: http status code OK 19:26:29 DEBUG --- status code --- 19:26:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:29 DEBUG --- http response --- 19:26:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:26:29 DEBUG ----- output ----- 19:26:29 DEBUG { 19:26:29 DEBUG "status": "success", 19:26:29 DEBUG "data": { 19:26:29 INFO [http_cmd]: http status code OK 19:26:29 DEBUG "resultType": "vector", 19:26:29 DEBUG --- status code --- 19:26:29 DEBUG "result": [ 19:26:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:29 DEBUG { 19:26:29 DEBUG --- http response --- 19:26:29 DEBUG "metric": { 19:26:29 INFO [http_cmd]: http status code OK 19:26:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:26:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:26:29 DEBUG --- status code --- 19:26:29 DEBUG }, 19:26:29 DEBUG ----- output ----- 19:26:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:29 DEBUG "value": [ 19:26:29 DEBUG { 19:26:29 DEBUG --- http response --- 19:26:29 DEBUG 19:26:29 DEBUG "status": "success", 19:26:29 DEBUG 19:26:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:26:29 DEBUG "data": { 19:26:29 DEBUG 19:26:29 DEBUG ----- output ----- 19:26:29 DEBUG "resultType": "vector", 19:26:29 DEBUG [...] 19:26:29 DEBUG { 19:26:29 DEBUG "result": [ 19:26:29 DEBUG 19:26:29 DEBUG "status": "success", 19:26:29 DEBUG { 19:26:29 DEBUG 19:26:29 DEBUG "data": { 19:26:29 DEBUG "metric": { 19:26:29 DEBUG 19:26:29 DEBUG "resultType": "vector", 19:26:29 DEBUG "instance": "10.142.0.107:9100" 19:26:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:26:29 DEBUG "result": [ 19:26:29 DEBUG }, 19:26:29 DEBUG }, 19:26:29 DEBUG { 19:26:29 DEBUG "value": [ 19:26:29 DEBUG "value": [ 19:26:29 DEBUG "metric": { 19:26:29 DEBUG 19:26:29 DEBUG 1713551189, 19:26:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:26:29 DEBUG 19:26:29 DEBUG "2337.233333333333" 19:26:29 DEBUG }, 19:26:29 DEBUG 19:26:29 DEBUG ] 19:26:29 DEBUG "value": [ 19:26:29 DEBUG [...] 19:26:29 DEBUG } 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG ] 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG } 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG } 19:26:29 DEBUG [...] 19:26:29 DEBUG "instance": "10.142.0.43:9100" 19:26:29 DEBUG 19:26:29 DEBUG }, 19:26:29 DEBUG 19:26:29 DEBUG "value": [ 19:26:29 DEBUG 19:26:29 DEBUG 1713551189, 19:26:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:26:29 DEBUG "70717626.46666667" 19:26:29 DEBUG }, 19:26:29 DEBUG ] 19:26:29 DEBUG "value": [ 19:26:29 DEBUG } 19:26:29 DEBUG 1713551189, 19:26:29 DEBUG ] 19:26:29 DEBUG "3385704448" 19:26:29 DEBUG } 19:26:29 DEBUG ] 19:26:29 DEBUG } 19:26:29 DEBUG } 19:26:29 DEBUG ] 19:26:29 DEBUG } 19:26:29 DEBUG } 19:26:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551189" 19:26:29 INFO [http_cmd]: http status code OK 19:26:29 DEBUG --- status code --- 19:26:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:29 DEBUG --- http response --- 19:26:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:26:29 DEBUG ----- output ----- 19:26:29 DEBUG { 19:26:29 DEBUG "status": "success", 19:26:29 DEBUG "data": { 19:26:29 DEBUG "resultType": "vector", 19:26:29 DEBUG "result": [ 19:26:29 DEBUG { 19:26:29 DEBUG "metric": { 19:26:29 DEBUG "pod": "ds-idrepo-0" 19:26:29 DEBUG }, 19:26:29 DEBUG "value": [ 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG [...] 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG 19:26:29 DEBUG "pod": "ds-cts-0" 19:26:29 DEBUG }, 19:26:29 DEBUG "value": [ 19:26:29 DEBUG 1713551189, 19:26:29 DEBUG "15.178328715655956" 19:26:29 DEBUG ] 19:26:29 DEBUG } 19:26:29 DEBUG ] 19:26:29 DEBUG } 19:26:29 DEBUG } 19:26:29 INFO 19:26:29 INFO [loop_until]: kubectl --namespace=xlou top pods 19:26:29 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:26:29 INFO [loop_until]: OK (rc = 0) 19:26:29 DEBUG --- stdout --- 19:26:29 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 14395m 8991Mi am-767f6b67b5-qcqz2 14311m 9002Mi am-767f6b67b5-vbltz 15212m 8984Mi ds-cts-0 15456m 10871Mi ds-cts-1 13826m 11112Mi ds-cts-2 15668m 10876Mi ds-idrepo-0 6433m 3647Mi ds-idrepo-1 6348m 3517Mi ds-idrepo-2 6501m 3492Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 4m 1217Mi lodemon-f8459c597-6tj6f 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7648m 3139Mi 19:26:29 DEBUG --- stderr --- 19:26:29 DEBUG 19:26:29 INFO 19:26:29 INFO [loop_until]: kubectl --namespace=xlou top node 19:26:29 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:26:29 INFO [loop_until]: OK (rc = 0) 19:26:29 DEBUG --- stdout --- 19:26:29 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 70m 0% 6254Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 6445m 21% 7235Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 14249m 47% 11697Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 82m 0% 3344Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 15722m 52% 13892Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 152m 0% 5192Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 14535m 48% 11879Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12351m 41% 2994Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 15912m 53% 11199Mi 10% gke-xlou-cdm-ds-0f601751-j488 6894m 23% 6733Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 14066m 47% 15924Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 7062m 23% 7401Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 16002m 53% 13360Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 7618m 47% 6937Mi 11% 19:26:29 DEBUG --- stderr --- 19:26:29 DEBUG 19:27:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551249" 19:27:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551249" 19:27:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551249" 19:27:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713551249" 19:27:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551249" 19:27:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551249" 19:27:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713551249" 19:27:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713551249" 19:27:29 INFO [http_cmd]: http status code OK 19:27:29 INFO [http_cmd]: http status code OK 19:27:29 DEBUG --- status code --- 19:27:29 DEBUG --- status code --- 19:27:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:29 DEBUG --- http response --- 19:27:29 DEBUG --- http response --- 19:27:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:27:29 DEBUG ----- output ----- 19:27:29 DEBUG { 19:27:29 DEBUG "status": "success", 19:27:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:27:29 DEBUG "data": { 19:27:29 DEBUG ----- output ----- 19:27:29 DEBUG "resultType": "vector", 19:27:29 INFO [http_cmd]: http status code OK 19:27:29 DEBUG { 19:27:29 DEBUG "result": [ 19:27:29 DEBUG --- status code --- 19:27:29 DEBUG "status": "success", 19:27:29 DEBUG { 19:27:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:29 DEBUG "data": { 19:27:29 DEBUG "metric": { 19:27:29 DEBUG --- http response --- 19:27:29 DEBUG "resultType": "vector", 19:27:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:27:29 DEBUG "result": [ 19:27:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:27:29 INFO [http_cmd]: http status code OK 19:27:29 INFO [http_cmd]: http status code OK 19:27:29 INFO [http_cmd]: http status code OK 19:27:29 DEBUG }, 19:27:29 DEBUG { 19:27:29 DEBUG ----- output ----- 19:27:29 DEBUG --- status code --- 19:27:29 DEBUG --- status code --- 19:27:29 DEBUG --- status code --- 19:27:29 DEBUG "value": [ 19:27:29 DEBUG "metric": { 19:27:29 DEBUG { 19:27:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:29 DEBUG 19:27:29 DEBUG "pod": "ds-cts-0" 19:27:29 DEBUG "status": "success", 19:27:29 DEBUG --- http response --- 19:27:29 DEBUG --- http response --- 19:27:29 DEBUG --- http response --- 19:27:29 DEBUG 19:27:29 DEBUG }, 19:27:29 DEBUG "data": { 19:27:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:27:29 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:27:29 DEBUG 19:27:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:27:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551249" 19:27:29 DEBUG "value": [ 19:27:29 DEBUG "resultType": "vector", 19:27:29 INFO [http_cmd]: http status code OK 19:27:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551249" 19:27:29 DEBUG ----- output ----- 19:27:29 DEBUG [...] 19:27:29 DEBUG ----- output ----- 19:27:29 DEBUG 19:27:29 DEBUG "result": [ 19:27:29 DEBUG --- status code --- 19:27:29 DEBUG { 19:27:29 DEBUG 19:27:29 DEBUG { 19:27:29 DEBUG 19:27:29 DEBUG { 19:27:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:29 DEBUG "status": "success", 19:27:29 DEBUG 19:27:29 DEBUG "status": "success", 19:27:29 DEBUG 19:27:29 DEBUG "metric": { 19:27:29 DEBUG --- http response --- 19:27:29 DEBUG "data": { 19:27:29 DEBUG 19:27:29 DEBUG "data": { 19:27:29 DEBUG [...] 19:27:29 DEBUG "instance": "10.142.0.107:9100" 19:27:29 ERROR [http_cmd]: ERROR 19:27:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:27:29 DEBUG "resultType": "vector", 19:27:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:27:29 DEBUG "resultType": "vector", 19:27:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551249" 19:27:29 DEBUG 19:27:29 DEBUG }, 19:27:29 ERROR --- status code --- 19:27:29 DEBUG ----- output ----- 19:27:29 DEBUG "result": [ 19:27:29 DEBUG }, 19:27:29 DEBUG "result": [ 19:27:29 DEBUG 19:27:29 DEBUG "value": [ 19:27:29 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:27:29 DEBUG { 19:27:29 DEBUG { 19:27:29 DEBUG "value": [ 19:27:29 DEBUG { 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 ERROR --- http response --- 19:27:29 DEBUG "status": "success", 19:27:29 DEBUG "metric": { 19:27:29 DEBUG 1713551249, 19:27:29 DEBUG "metric": { 19:27:29 DEBUG "pod": "ds-cts-2" 19:27:29 DEBUG 19:27:29 DEBUG "data": { 19:27:29 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:27:29 DEBUG "pod": "ds-idrepo-0" 19:27:29 DEBUG "2342.7666666666664" 19:27:29 DEBUG "pod": "ds-idrepo-0" 19:27:29 DEBUG }, 19:27:29 DEBUG 19:27:29 DEBUG "resultType": "vector", 19:27:29 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:27:29 DEBUG }, 19:27:29 DEBUG ] 19:27:29 DEBUG }, 19:27:29 DEBUG "value": [ 19:27:29 DEBUG [...] 19:27:29 DEBUG "result": [ 19:27:29 DEBUG "value": [ 19:27:29 DEBUG } 19:27:29 DEBUG "value": [ 19:27:29 DEBUG 1713551249, 19:27:29 DEBUG 19:27:29 DEBUG { 19:27:29 DEBUG 19:27:29 DEBUG ] 19:27:29 DEBUG 19:27:29 DEBUG "0" 19:27:29 DEBUG 19:27:29 DEBUG "metric": { 19:27:29 DEBUG 19:27:29 DEBUG } 19:27:29 DEBUG 19:27:29 DEBUG ] 19:27:29 DEBUG 19:27:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:27:29 DEBUG 19:27:29 DEBUG } 19:27:29 DEBUG 19:27:29 DEBUG } 19:27:29 DEBUG "instance": "10.142.0.43:9100" 19:27:29 DEBUG }, 19:27:29 DEBUG [...] 19:27:29 DEBUG [...] 19:27:29 DEBUG ] 19:27:29 DEBUG }, 19:27:29 DEBUG "value": [ 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG } 19:27:29 DEBUG "value": [ 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG } 19:27:29 DEBUG 1713551249, 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG "1887044770816" 19:27:29 DEBUG 19:27:29 DEBUG "pod": "ds-idrepo-2" 19:27:29 DEBUG "pod": "ds-cts-2" 19:27:29 DEBUG ] 19:27:29 DEBUG [...] 19:27:29 DEBUG }, 19:27:29 DEBUG }, 19:27:29 DEBUG } 19:27:29 DEBUG 19:27:29 INFO [http_cmd]: http status code OK 19:27:29 DEBUG "value": [ 19:27:29 DEBUG "value": [ 19:27:29 DEBUG ] 19:27:29 DEBUG 19:27:29 DEBUG --- status code --- 19:27:29 DEBUG 1713551249, 19:27:29 DEBUG 1713551249, 19:27:29 DEBUG } 19:27:29 DEBUG 19:27:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:29 DEBUG "0" 19:27:29 DEBUG "0" 19:27:29 DEBUG } 19:27:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:27:29 DEBUG --- http response --- 19:27:29 DEBUG ] 19:27:29 DEBUG ] 19:27:29 DEBUG }, 19:27:29 DEBUG } 19:27:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:27:29 DEBUG } 19:27:29 DEBUG "value": [ 19:27:29 DEBUG ] 19:27:29 DEBUG ----- output ----- 19:27:29 DEBUG ] 19:27:29 DEBUG 1713551249, 19:27:29 DEBUG } 19:27:29 DEBUG { 19:27:29 DEBUG } 19:27:29 DEBUG "7.703384245521958" 19:27:29 DEBUG } 19:27:29 DEBUG "status": "success", 19:27:29 DEBUG } 19:27:29 DEBUG ] 19:27:29 DEBUG "data": { 19:27:29 DEBUG } 19:27:29 DEBUG "resultType": "vector", 19:27:29 DEBUG ] 19:27:29 DEBUG "result": [ 19:27:29 DEBUG } 19:27:29 DEBUG { 19:27:29 DEBUG } 19:27:29 DEBUG "metric": { 19:27:29 DEBUG "pod": "ds-idrepo-0" 19:27:29 DEBUG }, 19:27:29 DEBUG "value": [ 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG [...] 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG "pod": "ds-idrepo-2" 19:27:29 DEBUG }, 19:27:29 DEBUG "value": [ 19:27:29 DEBUG 1713551249, 19:27:29 DEBUG "0" 19:27:29 DEBUG ] 19:27:29 DEBUG } 19:27:29 DEBUG ] 19:27:29 DEBUG } 19:27:29 DEBUG } 19:27:29 INFO [http_cmd]: http status code OK 19:27:29 DEBUG --- status code --- 19:27:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:29 DEBUG --- http response --- 19:27:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:27:29 DEBUG ----- output ----- 19:27:29 DEBUG { 19:27:29 DEBUG "status": "success", 19:27:29 DEBUG "data": { 19:27:29 DEBUG "resultType": "vector", 19:27:29 DEBUG "result": [ 19:27:29 DEBUG { 19:27:29 DEBUG "metric": { 19:27:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:27:29 DEBUG }, 19:27:29 DEBUG "value": [ 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG [...] 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:27:29 DEBUG }, 19:27:29 DEBUG "value": [ 19:27:29 DEBUG 1713551249, 19:27:29 DEBUG "0" 19:27:29 DEBUG ] 19:27:29 DEBUG } 19:27:29 DEBUG ] 19:27:29 DEBUG } 19:27:29 DEBUG } 19:27:29 INFO [http_cmd]: http status code OK 19:27:29 DEBUG --- status code --- 19:27:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:29 DEBUG --- http response --- 19:27:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:27:29 DEBUG ----- output ----- 19:27:29 DEBUG { 19:27:29 DEBUG "status": "success", 19:27:29 DEBUG "data": { 19:27:29 DEBUG "resultType": "vector", 19:27:29 DEBUG "result": [ 19:27:29 DEBUG { 19:27:29 DEBUG "metric": { 19:27:29 DEBUG "pod": "ds-idrepo-1" 19:27:29 DEBUG }, 19:27:29 DEBUG "value": [ 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG [...] 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG "pod": "idm-b85f9d978-wnhxb" 19:27:29 DEBUG }, 19:27:29 DEBUG "value": [ 19:27:29 DEBUG 1713551249, 19:27:29 DEBUG "1798.082131295253" 19:27:29 DEBUG ] 19:27:29 DEBUG } 19:27:29 DEBUG ] 19:27:29 DEBUG } 19:27:29 DEBUG } 19:27:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551249" 19:27:29 INFO [http_cmd]: http status code OK 19:27:29 DEBUG --- status code --- 19:27:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:29 DEBUG --- http response --- 19:27:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:27:29 DEBUG ----- output ----- 19:27:29 DEBUG { 19:27:29 DEBUG "status": "success", 19:27:29 DEBUG "data": { 19:27:29 DEBUG "resultType": "vector", 19:27:29 DEBUG "result": [ 19:27:29 DEBUG { 19:27:29 DEBUG "metric": { 19:27:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:27:29 DEBUG }, 19:27:29 DEBUG "value": [ 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG [...] 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:27:29 DEBUG }, 19:27:29 DEBUG "value": [ 19:27:29 DEBUG 1713551249, 19:27:29 DEBUG "2328.8333333333335" 19:27:29 DEBUG ] 19:27:29 DEBUG } 19:27:29 DEBUG ] 19:27:29 DEBUG } 19:27:29 DEBUG } 19:27:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551249" 19:27:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551249" 19:27:29 INFO [http_cmd]: http status code OK 19:27:29 DEBUG --- status code --- 19:27:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:29 DEBUG --- http response --- 19:27:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:27:29 DEBUG ----- output ----- 19:27:29 DEBUG { 19:27:29 DEBUG "status": "success", 19:27:29 DEBUG "data": { 19:27:29 DEBUG "resultType": "vector", 19:27:29 DEBUG "result": [ 19:27:29 DEBUG { 19:27:29 DEBUG "metric": { 19:27:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:27:29 DEBUG }, 19:27:29 DEBUG "value": [ 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG [...] 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:27:29 DEBUG }, 19:27:29 DEBUG "value": [ 19:27:29 DEBUG 1713551249, 19:27:29 DEBUG "420658.5" 19:27:29 DEBUG ] 19:27:29 DEBUG } 19:27:29 DEBUG ] 19:27:29 DEBUG } 19:27:29 DEBUG } 19:27:29 INFO [http_cmd]: http status code OK 19:27:29 DEBUG --- status code --- 19:27:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:29 DEBUG --- http response --- 19:27:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:27:29 DEBUG ----- output ----- 19:27:29 DEBUG { 19:27:29 DEBUG "status": "success", 19:27:29 DEBUG "data": { 19:27:29 DEBUG "resultType": "vector", 19:27:29 DEBUG "result": [ 19:27:29 DEBUG { 19:27:29 DEBUG "metric": { 19:27:29 DEBUG "pod": "ds-idrepo-1" 19:27:29 DEBUG }, 19:27:29 DEBUG "value": [ 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG [...] 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:27:29 DEBUG }, 19:27:29 DEBUG "value": [ 19:27:29 DEBUG 1713551249, 19:27:29 DEBUG "0" 19:27:29 DEBUG ] 19:27:29 DEBUG } 19:27:29 DEBUG ] 19:27:29 DEBUG } 19:27:29 DEBUG } 19:27:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551249" 19:27:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551249" 19:27:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551249" 19:27:29 INFO [http_cmd]: http status code OK 19:27:29 DEBUG --- status code --- 19:27:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:29 DEBUG --- http response --- 19:27:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:27:29 DEBUG ----- output ----- 19:27:29 DEBUG { 19:27:29 DEBUG "status": "success", 19:27:29 DEBUG "data": { 19:27:29 DEBUG "resultType": "vector", 19:27:29 DEBUG "result": [ 19:27:29 DEBUG { 19:27:29 DEBUG "metric": { 19:27:29 DEBUG "instance": "10.142.0.107:9100" 19:27:29 DEBUG }, 19:27:29 DEBUG "value": [ 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG [...] 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG "instance": "10.142.0.43:9100" 19:27:29 DEBUG }, 19:27:29 DEBUG "value": [ 19:27:29 DEBUG 1713551249, 19:27:29 DEBUG "70738250.26666667" 19:27:29 DEBUG ] 19:27:29 DEBUG } 19:27:29 DEBUG ] 19:27:29 DEBUG } 19:27:29 DEBUG } 19:27:29 INFO [http_cmd]: http status code OK 19:27:29 DEBUG --- status code --- 19:27:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:29 DEBUG --- http response --- 19:27:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:27:29 DEBUG ----- output ----- 19:27:29 DEBUG { 19:27:29 DEBUG "status": "success", 19:27:29 DEBUG "data": { 19:27:29 DEBUG "resultType": "vector", 19:27:29 DEBUG "result": [ 19:27:29 DEBUG { 19:27:29 DEBUG "metric": { 19:27:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:27:29 DEBUG }, 19:27:29 DEBUG "value": [ 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG [...] 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:27:29 DEBUG }, 19:27:29 DEBUG "value": [ 19:27:29 DEBUG 1713551249, 19:27:29 DEBUG "2328.9333333333334" 19:27:29 DEBUG ] 19:27:29 DEBUG } 19:27:29 DEBUG ] 19:27:29 DEBUG } 19:27:29 DEBUG } 19:27:29 INFO [http_cmd]: http status code OK 19:27:29 DEBUG --- status code --- 19:27:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:29 DEBUG --- http response --- 19:27:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:27:29 DEBUG ----- output ----- 19:27:29 DEBUG { 19:27:29 DEBUG "status": "success", 19:27:29 DEBUG "data": { 19:27:29 DEBUG "resultType": "vector", 19:27:29 DEBUG "result": [ 19:27:29 DEBUG { 19:27:29 DEBUG "metric": { 19:27:29 DEBUG "instance": "10.142.0.107:9100" 19:27:29 DEBUG }, 19:27:29 DEBUG "value": [ 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG [...] 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG "instance": "10.142.0.43:9100" 19:27:29 DEBUG }, 19:27:29 DEBUG "value": [ 19:27:29 DEBUG 1713551249, 19:27:29 DEBUG "0.156266666666743" 19:27:29 DEBUG ] 19:27:29 DEBUG } 19:27:29 DEBUG ] 19:27:29 DEBUG } 19:27:29 DEBUG } 19:27:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551249" 19:27:29 INFO [http_cmd]: http status code OK 19:27:29 DEBUG --- status code --- 19:27:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:29 DEBUG --- http response --- 19:27:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:27:29 DEBUG ----- output ----- 19:27:29 DEBUG { 19:27:29 DEBUG "status": "success", 19:27:29 DEBUG "data": { 19:27:29 DEBUG "resultType": "vector", 19:27:29 DEBUG "result": [ 19:27:29 DEBUG { 19:27:29 DEBUG "metric": { 19:27:29 DEBUG "pod": "ds-idrepo-0" 19:27:29 DEBUG }, 19:27:29 DEBUG "value": [ 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG [...] 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG "pod": "ds-idrepo-2" 19:27:29 DEBUG }, 19:27:29 DEBUG "value": [ 19:27:29 DEBUG 1713551249, 19:27:29 DEBUG "0" 19:27:29 DEBUG ] 19:27:29 DEBUG } 19:27:29 DEBUG ] 19:27:29 DEBUG } 19:27:29 DEBUG } 19:27:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551249" 19:27:29 INFO [http_cmd]: http status code OK 19:27:29 DEBUG --- status code --- 19:27:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:29 DEBUG --- http response --- 19:27:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:27:29 DEBUG ----- output ----- 19:27:29 DEBUG { 19:27:29 DEBUG "status": "success", 19:27:29 DEBUG "data": { 19:27:29 DEBUG "resultType": "vector", 19:27:29 DEBUG "result": [ 19:27:29 DEBUG { 19:27:29 DEBUG "metric": { 19:27:29 DEBUG "instance": "10.142.0.107:9100" 19:27:29 DEBUG }, 19:27:29 DEBUG "value": [ 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG [...] 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG "instance": "10.142.0.43:9100" 19:27:29 DEBUG }, 19:27:29 DEBUG "value": [ 19:27:29 DEBUG 1713551249, 19:27:29 DEBUG "0.06866666666665537" 19:27:29 DEBUG ] 19:27:29 DEBUG } 19:27:29 DEBUG ] 19:27:29 DEBUG } 19:27:29 DEBUG } 19:27:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551249" 19:27:29 INFO [http_cmd]: http status code OK 19:27:29 DEBUG --- status code --- 19:27:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:29 DEBUG --- http response --- 19:27:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:27:29 DEBUG ----- output ----- 19:27:29 DEBUG { 19:27:29 DEBUG "status": "success", 19:27:29 DEBUG "data": { 19:27:29 DEBUG "resultType": "vector", 19:27:29 DEBUG "result": [ 19:27:29 DEBUG { 19:27:29 DEBUG "metric": { 19:27:29 DEBUG "pod": "ds-cts-0" 19:27:29 DEBUG }, 19:27:29 DEBUG "value": [ 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG [...] 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG "pod": "ds-cts-2" 19:27:29 DEBUG }, 19:27:29 DEBUG "value": [ 19:27:29 DEBUG 1713551249, 19:27:29 DEBUG "21.733333333333334" 19:27:29 DEBUG ] 19:27:29 DEBUG } 19:27:29 DEBUG ] 19:27:29 DEBUG } 19:27:29 DEBUG } 19:27:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551249" 19:27:29 INFO [http_cmd]: http status code OK 19:27:29 DEBUG --- status code --- 19:27:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:29 DEBUG --- http response --- 19:27:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:27:29 DEBUG ----- output ----- 19:27:29 DEBUG { 19:27:29 DEBUG "status": "success", 19:27:29 DEBUG "data": { 19:27:29 DEBUG "resultType": "vector", 19:27:29 DEBUG "result": [ 19:27:29 DEBUG { 19:27:29 DEBUG "metric": { 19:27:29 DEBUG "pod": "ds-cts-0" 19:27:29 DEBUG }, 19:27:29 DEBUG "value": [ 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG [...] 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG "pod": "ds-cts-2" 19:27:29 DEBUG }, 19:27:29 DEBUG "value": [ 19:27:29 DEBUG 1713551249, 19:27:29 DEBUG "0.00006666666666666667" 19:27:29 DEBUG ] 19:27:29 DEBUG } 19:27:29 DEBUG ] 19:27:29 DEBUG } 19:27:29 DEBUG } 19:27:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713551249" 19:27:29 INFO [http_cmd]: http status code OK 19:27:29 DEBUG --- status code --- 19:27:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:29 DEBUG --- http response --- 19:27:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:27:29 DEBUG ----- output ----- 19:27:29 DEBUG { 19:27:29 DEBUG "status": "success", 19:27:29 DEBUG "data": { 19:27:29 DEBUG "resultType": "vector", 19:27:29 DEBUG "result": [ 19:27:29 DEBUG { 19:27:29 DEBUG "metric": { 19:27:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:27:29 DEBUG }, 19:27:29 DEBUG "value": [ 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG [...] 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:27:29 DEBUG }, 19:27:29 DEBUG "value": [ 19:27:29 DEBUG 1713551249, 19:27:29 DEBUG "0.005" 19:27:29 DEBUG ] 19:27:29 DEBUG } 19:27:29 DEBUG ] 19:27:29 DEBUG } 19:27:29 DEBUG } 19:27:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551249" 19:27:29 INFO [http_cmd]: http status code OK 19:27:29 DEBUG --- status code --- 19:27:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:29 DEBUG --- http response --- 19:27:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:27:29 DEBUG ----- output ----- 19:27:29 DEBUG { 19:27:29 DEBUG "status": "success", 19:27:29 DEBUG "data": { 19:27:29 DEBUG "resultType": "vector", 19:27:29 DEBUG "result": [ 19:27:29 DEBUG { 19:27:29 DEBUG "metric": { 19:27:29 DEBUG "pod": "ds-idrepo-1" 19:27:29 DEBUG }, 19:27:29 DEBUG "value": [ 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG [...] 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG "pod": "idm-b85f9d978-wnhxb" 19:27:29 DEBUG }, 19:27:29 DEBUG "value": [ 19:27:29 DEBUG 1713551249, 19:27:29 DEBUG "0" 19:27:29 DEBUG ] 19:27:29 DEBUG } 19:27:29 DEBUG ] 19:27:29 DEBUG } 19:27:29 DEBUG } 19:27:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551249" 19:27:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551249" 19:27:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713551249" 19:27:29 INFO [http_cmd]: http status code OK 19:27:29 DEBUG --- status code --- 19:27:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:29 DEBUG --- http response --- 19:27:29 INFO [http_cmd]: http status code OK 19:27:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:27:29 DEBUG --- status code --- 19:27:29 DEBUG ----- output ----- 19:27:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:29 DEBUG { 19:27:29 DEBUG --- http response --- 19:27:29 DEBUG "status": "success", 19:27:29 DEBUG "data": { 19:27:29 DEBUG "resultType": "vector", 19:27:29 DEBUG "result": [ 19:27:29 DEBUG { 19:27:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:27:29 DEBUG "metric": { 19:27:29 DEBUG ----- output ----- 19:27:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:27:29 DEBUG { 19:27:29 DEBUG }, 19:27:29 DEBUG "status": "success", 19:27:29 DEBUG "value": [ 19:27:29 DEBUG "data": { 19:27:29 DEBUG 19:27:29 DEBUG "resultType": "vector", 19:27:29 DEBUG 19:27:29 DEBUG "result": [ 19:27:29 DEBUG 19:27:29 DEBUG { 19:27:29 DEBUG [...] 19:27:29 DEBUG "metric": { 19:27:29 DEBUG 19:27:29 DEBUG "instance": "10.142.0.107:9100" 19:27:29 DEBUG 19:27:29 DEBUG }, 19:27:29 DEBUG 19:27:29 DEBUG "value": [ 19:27:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:27:29 DEBUG 19:27:29 DEBUG }, 19:27:29 DEBUG 19:27:29 DEBUG "value": [ 19:27:29 DEBUG 19:27:29 DEBUG 1713551249, 19:27:29 DEBUG [...] 19:27:29 DEBUG "2328.733333333333" 19:27:29 DEBUG 19:27:29 DEBUG ] 19:27:29 DEBUG 19:27:29 DEBUG } 19:27:29 DEBUG 19:27:29 DEBUG ] 19:27:29 DEBUG "instance": "10.142.0.43:9100" 19:27:29 DEBUG } 19:27:29 DEBUG }, 19:27:29 DEBUG } 19:27:29 DEBUG "value": [ 19:27:29 DEBUG 1713551249, 19:27:29 DEBUG "70736026.5" 19:27:29 DEBUG ] 19:27:29 DEBUG } 19:27:29 DEBUG ] 19:27:29 DEBUG } 19:27:29 DEBUG } 19:27:29 INFO [http_cmd]: http status code OK 19:27:29 DEBUG --- status code --- 19:27:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:29 DEBUG --- http response --- 19:27:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:27:29 DEBUG ----- output ----- 19:27:29 DEBUG { 19:27:29 DEBUG "status": "success", 19:27:29 DEBUG "data": { 19:27:29 DEBUG "resultType": "vector", 19:27:29 DEBUG "result": [ 19:27:29 DEBUG { 19:27:29 DEBUG "metric": { 19:27:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:27:29 DEBUG }, 19:27:29 DEBUG "value": [ 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG [...] 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:27:29 DEBUG }, 19:27:29 DEBUG "value": [ 19:27:29 DEBUG 1713551249, 19:27:29 DEBUG "3387895808" 19:27:29 DEBUG ] 19:27:29 DEBUG } 19:27:29 DEBUG ] 19:27:29 DEBUG } 19:27:29 DEBUG } 19:27:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551249" 19:27:29 INFO [http_cmd]: http status code OK 19:27:29 DEBUG --- status code --- 19:27:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:29 DEBUG --- http response --- 19:27:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:27:29 DEBUG ----- output ----- 19:27:29 DEBUG { 19:27:29 DEBUG "status": "success", 19:27:29 DEBUG "data": { 19:27:29 DEBUG "resultType": "vector", 19:27:29 DEBUG "result": [ 19:27:29 DEBUG { 19:27:29 DEBUG "metric": { 19:27:29 DEBUG "pod": "ds-idrepo-1" 19:27:29 DEBUG }, 19:27:29 DEBUG "value": [ 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG [...] 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG 19:27:29 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:27:29 DEBUG }, 19:27:29 DEBUG "value": [ 19:27:29 DEBUG 1713551249, 19:27:29 DEBUG "0" 19:27:29 DEBUG ] 19:27:29 DEBUG } 19:27:29 DEBUG ] 19:27:29 DEBUG } 19:27:29 DEBUG } 19:27:29 INFO 19:27:29 INFO [loop_until]: kubectl --namespace=xlou top pods 19:27:29 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:27:29 INFO [loop_until]: OK (rc = 0) 19:27:29 DEBUG --- stdout --- 19:27:29 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 14186m 8992Mi am-767f6b67b5-qcqz2 14375m 9004Mi am-767f6b67b5-vbltz 15767m 8984Mi ds-cts-0 15585m 10855Mi ds-cts-1 13782m 11109Mi ds-cts-2 16099m 10868Mi ds-idrepo-0 6399m 3649Mi ds-idrepo-1 6407m 3529Mi ds-idrepo-2 6267m 3493Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7758m 3141Mi 19:27:29 DEBUG --- stderr --- 19:27:29 DEBUG 19:27:29 INFO 19:27:29 INFO [loop_until]: kubectl --namespace=xlou top node 19:27:29 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:27:29 INFO [loop_until]: OK (rc = 0) 19:27:29 DEBUG --- stdout --- 19:27:29 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 69m 0% 6247Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 6458m 21% 7249Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 14155m 47% 11702Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 88m 0% 3351Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 15430m 51% 13891Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 152m 0% 5184Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 14467m 48% 11887Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12265m 41% 3008Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 16370m 54% 11190Mi 10% gke-xlou-cdm-ds-0f601751-j488 6690m 22% 6734Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 13931m 46% 15937Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 7041m 23% 7412Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 15745m 52% 13354Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 7844m 49% 6938Mi 11% 19:27:29 DEBUG --- stderr --- 19:27:29 DEBUG 19:28:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713551309" 19:28:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551309" 19:28:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551309" 19:28:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551309" 19:28:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713551309" 19:28:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551309" 19:28:29 INFO [http_cmd]: http status code OK 19:28:29 ERROR [http_cmd]: ERROR 19:28:29 INFO [http_cmd]: http status code OK 19:28:29 DEBUG --- status code --- 19:28:29 ERROR --- status code --- 19:28:29 DEBUG --- status code --- 19:28:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:29 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:28:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:29 DEBUG --- http response --- 19:28:29 ERROR --- http response --- 19:28:29 DEBUG --- http response --- 19:28:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 19:28:29 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:28:29 DEBUG ----- output ----- 19:28:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:28:29 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:28:29 DEBUG { 19:28:29 DEBUG ----- output ----- 19:28:29 DEBUG "status": "success", 19:28:29 DEBUG { 19:28:29 DEBUG "data": { 19:28:29 DEBUG "status": "success", 19:28:29 DEBUG "resultType": "vector", 19:28:29 DEBUG "data": { 19:28:29 DEBUG "result": [ 19:28:29 DEBUG "resultType": "vector", 19:28:29 DEBUG { 19:28:29 DEBUG "result": [ 19:28:29 DEBUG "metric": { 19:28:29 DEBUG { 19:28:29 DEBUG "pod": "ds-idrepo-1" 19:28:29 DEBUG "metric": { 19:28:29 DEBUG }, 19:28:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:28:29 DEBUG "value": [ 19:28:29 DEBUG }, 19:28:29 DEBUG 19:28:29 DEBUG "value": [ 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG [...] 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG [...] 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:28:29 DEBUG 19:28:29 DEBUG }, 19:28:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:28:29 DEBUG "value": [ 19:28:29 DEBUG }, 19:28:29 DEBUG 1713551309, 19:28:29 DEBUG "value": [ 19:28:29 DEBUG "0" 19:28:29 DEBUG 1713551309, 19:28:29 DEBUG ] 19:28:29 DEBUG "2331.4666666666667" 19:28:29 DEBUG } 19:28:29 DEBUG ] 19:28:29 DEBUG ] 19:28:29 DEBUG } 19:28:29 DEBUG } 19:28:29 DEBUG ] 19:28:29 DEBUG } 19:28:29 DEBUG } 19:28:29 INFO [http_cmd]: http status code OK 19:28:29 DEBUG } 19:28:29 DEBUG --- status code --- 19:28:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:29 DEBUG --- http response --- 19:28:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:28:29 DEBUG ----- output ----- 19:28:29 DEBUG { 19:28:29 DEBUG "status": "success", 19:28:29 DEBUG "data": { 19:28:29 DEBUG "resultType": "vector", 19:28:29 DEBUG "result": [ 19:28:29 DEBUG { 19:28:29 DEBUG "metric": { 19:28:29 DEBUG "pod": "ds-idrepo-0" 19:28:29 DEBUG }, 19:28:29 DEBUG "value": [ 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG [...] 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG "pod": "ds-idrepo-2" 19:28:29 DEBUG }, 19:28:29 DEBUG "value": [ 19:28:29 DEBUG 1713551309, 19:28:29 DEBUG "0" 19:28:29 DEBUG ] 19:28:29 DEBUG } 19:28:29 DEBUG ] 19:28:29 DEBUG } 19:28:29 DEBUG } 19:28:29 INFO [http_cmd]: http status code OK 19:28:29 DEBUG --- status code --- 19:28:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:29 DEBUG --- http response --- 19:28:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:28:29 DEBUG ----- output ----- 19:28:29 DEBUG { 19:28:29 DEBUG "status": "success", 19:28:29 DEBUG "data": { 19:28:29 DEBUG "resultType": "vector", 19:28:29 DEBUG "result": [ 19:28:29 DEBUG { 19:28:29 DEBUG "metric": { 19:28:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:28:29 DEBUG }, 19:28:29 DEBUG "value": [ 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG [...] 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:28:29 DEBUG }, 19:28:29 DEBUG "value": [ 19:28:29 DEBUG 1713551309, 19:28:29 DEBUG "7.744905098499154" 19:28:29 DEBUG ] 19:28:29 DEBUG } 19:28:29 DEBUG ] 19:28:29 DEBUG } 19:28:29 DEBUG } 19:28:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551309" 19:28:29 INFO [http_cmd]: http status code OK 19:28:29 DEBUG --- status code --- 19:28:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:29 DEBUG --- http response --- 19:28:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:28:29 DEBUG ----- output ----- 19:28:29 DEBUG { 19:28:29 DEBUG "status": "success", 19:28:29 DEBUG "data": { 19:28:29 DEBUG "resultType": "vector", 19:28:29 DEBUG "result": [ 19:28:29 DEBUG { 19:28:29 DEBUG "metric": { 19:28:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:28:29 DEBUG }, 19:28:29 DEBUG "value": [ 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG [...] 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:28:29 DEBUG }, 19:28:29 DEBUG "value": [ 19:28:29 DEBUG 1713551309, 19:28:29 DEBUG "0" 19:28:29 DEBUG ] 19:28:29 DEBUG } 19:28:29 DEBUG ] 19:28:29 DEBUG } 19:28:29 DEBUG } 19:28:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551309" 19:28:29 INFO [http_cmd]: http status code OK 19:28:29 DEBUG --- status code --- 19:28:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:29 DEBUG --- http response --- 19:28:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:28:29 DEBUG ----- output ----- 19:28:29 DEBUG { 19:28:29 DEBUG "status": "success", 19:28:29 DEBUG "data": { 19:28:29 DEBUG "resultType": "vector", 19:28:29 DEBUG "result": [ 19:28:29 DEBUG { 19:28:29 DEBUG "metric": { 19:28:29 DEBUG "instance": "10.142.0.107:9100" 19:28:29 DEBUG }, 19:28:29 DEBUG "value": [ 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG [...] 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG "instance": "10.142.0.43:9100" 19:28:29 DEBUG }, 19:28:29 DEBUG "value": [ 19:28:29 DEBUG 1713551309, 19:28:29 DEBUG "1889916445696" 19:28:29 DEBUG ] 19:28:29 DEBUG } 19:28:29 DEBUG ] 19:28:29 DEBUG } 19:28:29 DEBUG } 19:28:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551309" 19:28:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551309" 19:28:29 INFO [http_cmd]: http status code OK 19:28:29 DEBUG --- status code --- 19:28:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:29 DEBUG --- http response --- 19:28:29 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:28:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713551309" 19:28:29 INFO [http_cmd]: http status code OK 19:28:29 DEBUG --- status code --- 19:28:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:29 DEBUG --- http response --- 19:28:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:28:29 DEBUG ----- output ----- 19:28:29 DEBUG { 19:28:29 DEBUG "status": "success", 19:28:29 DEBUG "data": { 19:28:29 DEBUG "resultType": "vector", 19:28:29 DEBUG "result": [ 19:28:29 DEBUG { 19:28:29 DEBUG "metric": { 19:28:29 DEBUG "pod": "ds-idrepo-0" 19:28:29 DEBUG }, 19:28:29 DEBUG "value": [ 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG [...] 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG "pod": "ds-idrepo-2" 19:28:29 DEBUG }, 19:28:29 DEBUG "value": [ 19:28:29 DEBUG 1713551309, 19:28:29 DEBUG "0" 19:28:29 DEBUG ] 19:28:29 DEBUG } 19:28:29 DEBUG ] 19:28:29 DEBUG } 19:28:29 DEBUG } 19:28:29 INFO [http_cmd]: http status code OK 19:28:29 INFO [http_cmd]: http status code OK 19:28:29 DEBUG --- status code --- 19:28:29 DEBUG --- status code --- 19:28:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:29 DEBUG --- http response --- 19:28:29 DEBUG --- http response --- 19:28:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:28:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:28:29 DEBUG ----- output ----- 19:28:29 DEBUG ----- output ----- 19:28:29 DEBUG { 19:28:29 DEBUG { 19:28:29 DEBUG "status": "success", 19:28:29 DEBUG "status": "success", 19:28:29 DEBUG "data": { 19:28:29 DEBUG "data": { 19:28:29 DEBUG "resultType": "vector", 19:28:29 DEBUG "resultType": "vector", 19:28:29 DEBUG "result": [ 19:28:29 DEBUG "result": [ 19:28:29 DEBUG { 19:28:29 DEBUG { 19:28:29 DEBUG "metric": { 19:28:29 DEBUG "metric": { 19:28:29 DEBUG "pod": "ds-cts-0" 19:28:29 DEBUG "pod": "ds-idrepo-0" 19:28:29 DEBUG }, 19:28:29 DEBUG }, 19:28:29 DEBUG "value": [ 19:28:29 DEBUG "value": [ 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG [...] 19:28:29 DEBUG [...] 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG "pod": "ds-cts-2" 19:28:29 DEBUG "pod": "ds-cts-2" 19:28:29 DEBUG }, 19:28:29 DEBUG }, 19:28:29 DEBUG "value": [ 19:28:29 DEBUG "value": [ 19:28:29 DEBUG 1713551309, 19:28:29 DEBUG 1713551309, 19:28:29 DEBUG "0" 19:28:29 DEBUG "0" 19:28:29 DEBUG ] 19:28:29 DEBUG ] 19:28:29 DEBUG } 19:28:29 DEBUG } 19:28:29 DEBUG ] 19:28:29 DEBUG ] 19:28:29 DEBUG } 19:28:29 DEBUG } 19:28:29 DEBUG } 19:28:29 DEBUG } 19:28:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551309" 19:28:29 INFO [http_cmd]: http status code OK 19:28:29 DEBUG --- status code --- 19:28:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:29 DEBUG --- http response --- 19:28:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:28:29 DEBUG ----- output ----- 19:28:29 DEBUG { 19:28:29 DEBUG "status": "success", 19:28:29 DEBUG "data": { 19:28:29 DEBUG "resultType": "vector", 19:28:29 DEBUG "result": [ 19:28:29 DEBUG { 19:28:29 DEBUG "metric": { 19:28:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:28:29 DEBUG }, 19:28:29 DEBUG "value": [ 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG [...] 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:28:29 DEBUG }, 19:28:29 DEBUG "value": [ 19:28:29 DEBUG 1713551309, 19:28:29 DEBUG "2331.9666666666667" 19:28:29 DEBUG ] 19:28:29 DEBUG } 19:28:29 DEBUG ] 19:28:29 DEBUG } 19:28:29 DEBUG } 19:28:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551309" 19:28:29 INFO [http_cmd]: http status code OK 19:28:29 DEBUG --- status code --- 19:28:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:29 DEBUG --- http response --- 19:28:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:28:29 DEBUG ----- output ----- 19:28:29 DEBUG { 19:28:29 DEBUG "status": "success", 19:28:29 DEBUG "data": { 19:28:29 DEBUG "resultType": "vector", 19:28:29 DEBUG "result": [ 19:28:29 DEBUG { 19:28:29 DEBUG "metric": { 19:28:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:28:29 DEBUG }, 19:28:29 DEBUG "value": [ 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG [...] 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:28:29 DEBUG }, 19:28:29 DEBUG "value": [ 19:28:29 DEBUG 1713551309, 19:28:29 DEBUG "420402.5" 19:28:29 DEBUG ] 19:28:29 DEBUG } 19:28:29 DEBUG ] 19:28:29 DEBUG } 19:28:29 DEBUG } 19:28:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551309" 19:28:29 INFO [http_cmd]: http status code OK 19:28:29 DEBUG --- status code --- 19:28:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:29 DEBUG --- http response --- 19:28:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:28:29 DEBUG ----- output ----- 19:28:29 DEBUG { 19:28:29 DEBUG "status": "success", 19:28:29 DEBUG "data": { 19:28:29 DEBUG "resultType": "vector", 19:28:29 DEBUG "result": [ 19:28:29 DEBUG { 19:28:29 DEBUG "metric": { 19:28:29 DEBUG "instance": "10.142.0.107:9100" 19:28:29 DEBUG }, 19:28:29 DEBUG "value": [ 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG [...] 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG "instance": "10.142.0.43:9100" 19:28:29 DEBUG }, 19:28:29 DEBUG "value": [ 19:28:29 DEBUG 1713551309, 19:28:29 DEBUG "0.08620000000003074" 19:28:29 DEBUG ] 19:28:29 DEBUG } 19:28:29 DEBUG ] 19:28:29 DEBUG } 19:28:29 DEBUG } 19:28:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551309" 19:28:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551309" 19:28:29 INFO [http_cmd]: http status code OK 19:28:29 DEBUG --- status code --- 19:28:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:29 DEBUG --- http response --- 19:28:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:28:29 DEBUG ----- output ----- 19:28:29 DEBUG { 19:28:29 DEBUG "status": "success", 19:28:29 DEBUG "data": { 19:28:29 DEBUG "resultType": "vector", 19:28:29 DEBUG "result": [ 19:28:29 DEBUG { 19:28:29 DEBUG "metric": { 19:28:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:28:29 DEBUG }, 19:28:29 DEBUG "value": [ 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG [...] 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:28:29 DEBUG }, 19:28:29 DEBUG "value": [ 19:28:29 DEBUG 1713551309, 19:28:29 DEBUG "2331.9333333333334" 19:28:29 DEBUG ] 19:28:29 DEBUG } 19:28:29 DEBUG ] 19:28:29 DEBUG } 19:28:29 DEBUG } 19:28:29 INFO [http_cmd]: http status code OK 19:28:29 DEBUG --- status code --- 19:28:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:29 DEBUG --- http response --- 19:28:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:28:29 DEBUG ----- output ----- 19:28:29 DEBUG { 19:28:29 DEBUG "status": "success", 19:28:29 DEBUG "data": { 19:28:29 DEBUG "resultType": "vector", 19:28:29 DEBUG "result": [ 19:28:29 DEBUG { 19:28:29 DEBUG "metric": { 19:28:29 DEBUG "pod": "ds-idrepo-1" 19:28:29 DEBUG }, 19:28:29 DEBUG "value": [ 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG [...] 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:28:29 DEBUG }, 19:28:29 DEBUG "value": [ 19:28:29 DEBUG 1713551309, 19:28:29 DEBUG "0" 19:28:29 DEBUG ] 19:28:29 DEBUG } 19:28:29 DEBUG ] 19:28:29 DEBUG } 19:28:29 DEBUG } 19:28:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551309" 19:28:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551309" 19:28:29 INFO [http_cmd]: http status code OK 19:28:29 DEBUG --- status code --- 19:28:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:29 DEBUG --- http response --- 19:28:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:28:29 DEBUG ----- output ----- 19:28:29 DEBUG { 19:28:29 DEBUG "status": "success", 19:28:29 DEBUG "data": { 19:28:29 DEBUG "resultType": "vector", 19:28:29 DEBUG "result": [ 19:28:29 DEBUG { 19:28:29 DEBUG "metric": { 19:28:29 DEBUG "instance": "10.142.0.107:9100" 19:28:29 DEBUG }, 19:28:29 DEBUG "value": [ 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG [...] 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG "instance": "10.142.0.43:9100" 19:28:29 DEBUG }, 19:28:29 DEBUG "value": [ 19:28:29 DEBUG 1713551309, 19:28:29 DEBUG "70735913.03333333" 19:28:29 DEBUG ] 19:28:29 DEBUG } 19:28:29 DEBUG ] 19:28:29 DEBUG } 19:28:29 DEBUG } 19:28:29 INFO [http_cmd]: http status code OK 19:28:29 DEBUG --- status code --- 19:28:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:29 DEBUG --- http response --- 19:28:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:28:29 DEBUG ----- output ----- 19:28:29 DEBUG { 19:28:29 DEBUG "status": "success", 19:28:29 DEBUG "data": { 19:28:29 DEBUG "resultType": "vector", 19:28:29 DEBUG "result": [ 19:28:29 DEBUG { 19:28:29 DEBUG "metric": { 19:28:29 DEBUG "instance": "10.142.0.107:9100" 19:28:29 DEBUG }, 19:28:29 DEBUG "value": [ 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG [...] 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG "instance": "10.142.0.43:9100" 19:28:29 DEBUG }, 19:28:29 DEBUG "value": [ 19:28:29 DEBUG 1713551309, 19:28:29 DEBUG "0.0476666666666612" 19:28:29 DEBUG ] 19:28:29 DEBUG } 19:28:29 DEBUG ] 19:28:29 DEBUG } 19:28:29 DEBUG } 19:28:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551309" 19:28:29 INFO [http_cmd]: http status code OK 19:28:29 DEBUG --- status code --- 19:28:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:29 DEBUG --- http response --- 19:28:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:28:29 DEBUG ----- output ----- 19:28:29 DEBUG { 19:28:29 DEBUG "status": "success", 19:28:29 DEBUG "data": { 19:28:29 DEBUG "resultType": "vector", 19:28:29 DEBUG "result": [ 19:28:29 DEBUG { 19:28:29 DEBUG "metric": { 19:28:29 DEBUG "pod": "ds-idrepo-0" 19:28:29 DEBUG }, 19:28:29 DEBUG "value": [ 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG [...] 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG "pod": "ds-idrepo-2" 19:28:29 DEBUG }, 19:28:29 DEBUG "value": [ 19:28:29 DEBUG 1713551309, 19:28:29 DEBUG "0" 19:28:29 DEBUG ] 19:28:29 DEBUG } 19:28:29 DEBUG ] 19:28:29 DEBUG } 19:28:29 DEBUG } 19:28:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551309" 19:28:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551309" 19:28:29 INFO [http_cmd]: http status code OK 19:28:29 DEBUG --- status code --- 19:28:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:29 DEBUG --- http response --- 19:28:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:28:29 DEBUG ----- output ----- 19:28:29 DEBUG { 19:28:29 DEBUG "status": "success", 19:28:29 DEBUG "data": { 19:28:29 DEBUG "resultType": "vector", 19:28:29 DEBUG "result": [ 19:28:29 DEBUG { 19:28:29 DEBUG "metric": { 19:28:29 DEBUG "pod": "ds-cts-0" 19:28:29 DEBUG }, 19:28:29 DEBUG "value": [ 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG [...] 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG "pod": "ds-cts-2" 19:28:29 DEBUG }, 19:28:29 DEBUG "value": [ 19:28:29 DEBUG 1713551309, 19:28:29 DEBUG "30" 19:28:29 DEBUG ] 19:28:29 DEBUG } 19:28:29 DEBUG ] 19:28:29 DEBUG } 19:28:29 DEBUG } 19:28:29 INFO [http_cmd]: http status code OK 19:28:29 DEBUG --- status code --- 19:28:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:29 DEBUG --- http response --- 19:28:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:28:29 DEBUG ----- output ----- 19:28:29 DEBUG { 19:28:29 DEBUG "status": "success", 19:28:29 DEBUG "data": { 19:28:29 DEBUG "resultType": "vector", 19:28:29 DEBUG "result": [ 19:28:29 DEBUG { 19:28:29 DEBUG "metric": { 19:28:29 DEBUG "pod": "ds-cts-0" 19:28:29 DEBUG }, 19:28:29 DEBUG "value": [ 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG [...] 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG "pod": "ds-cts-2" 19:28:29 DEBUG }, 19:28:29 DEBUG "value": [ 19:28:29 DEBUG 1713551309, 19:28:29 DEBUG "0.000033333333333333335" 19:28:29 DEBUG ] 19:28:29 DEBUG } 19:28:29 DEBUG ] 19:28:29 DEBUG } 19:28:29 DEBUG } 19:28:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713551309" 19:28:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551309" 19:28:29 INFO [http_cmd]: http status code OK 19:28:29 DEBUG --- status code --- 19:28:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:29 DEBUG --- http response --- 19:28:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:28:29 DEBUG ----- output ----- 19:28:29 DEBUG { 19:28:29 DEBUG "status": "success", 19:28:29 DEBUG "data": { 19:28:29 DEBUG "resultType": "vector", 19:28:29 DEBUG "result": [ 19:28:29 DEBUG { 19:28:29 DEBUG "metric": { 19:28:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:28:29 DEBUG }, 19:28:29 DEBUG "value": [ 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG [...] 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:28:29 DEBUG }, 19:28:29 DEBUG "value": [ 19:28:29 DEBUG 1713551309, 19:28:29 DEBUG "0.006500000000000001" 19:28:29 DEBUG ] 19:28:29 DEBUG } 19:28:29 DEBUG ] 19:28:29 DEBUG } 19:28:29 DEBUG } 19:28:29 INFO [http_cmd]: http status code OK 19:28:29 DEBUG --- status code --- 19:28:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:29 DEBUG --- http response --- 19:28:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:28:29 DEBUG ----- output ----- 19:28:29 DEBUG { 19:28:29 DEBUG "status": "success", 19:28:29 DEBUG "data": { 19:28:29 DEBUG "resultType": "vector", 19:28:29 DEBUG "result": [ 19:28:29 DEBUG { 19:28:29 DEBUG "metric": { 19:28:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:28:29 DEBUG }, 19:28:29 DEBUG "value": [ 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG [...] 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:28:29 DEBUG }, 19:28:29 DEBUG "value": [ 19:28:29 DEBUG 1713551309, 19:28:29 DEBUG "2331.9" 19:28:29 DEBUG ] 19:28:29 DEBUG } 19:28:29 DEBUG ] 19:28:29 DEBUG } 19:28:29 DEBUG } 19:28:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551309" 19:28:29 INFO [http_cmd]: http status code OK 19:28:29 DEBUG --- status code --- 19:28:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:29 DEBUG --- http response --- 19:28:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 19:28:29 DEBUG ----- output ----- 19:28:29 DEBUG { 19:28:29 DEBUG "status": "success", 19:28:29 DEBUG "data": { 19:28:29 DEBUG "resultType": "vector", 19:28:29 DEBUG "result": [ 19:28:29 DEBUG { 19:28:29 DEBUG "metric": { 19:28:29 DEBUG "pod": "ds-idrepo-1" 19:28:29 DEBUG }, 19:28:29 DEBUG "value": [ 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG [...] 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:28:29 DEBUG }, 19:28:29 DEBUG "value": [ 19:28:29 DEBUG 1713551309, 19:28:29 DEBUG "0" 19:28:29 DEBUG ] 19:28:29 DEBUG } 19:28:29 DEBUG ] 19:28:29 DEBUG } 19:28:29 DEBUG } 19:28:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551309" 19:28:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713551309" 19:28:29 INFO [http_cmd]: http status code OK 19:28:29 DEBUG --- status code --- 19:28:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:29 DEBUG --- http response --- 19:28:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:28:29 DEBUG ----- output ----- 19:28:29 DEBUG { 19:28:29 DEBUG "status": "success", 19:28:29 DEBUG "data": { 19:28:29 DEBUG "resultType": "vector", 19:28:29 DEBUG "result": [ 19:28:29 DEBUG { 19:28:29 DEBUG "metric": { 19:28:29 DEBUG "instance": "10.142.0.107:9100" 19:28:29 DEBUG }, 19:28:29 DEBUG "value": [ 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG [...] 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG "instance": "10.142.0.43:9100" 19:28:29 DEBUG }, 19:28:29 DEBUG "value": [ 19:28:29 DEBUG 1713551309, 19:28:29 DEBUG "70732105.5" 19:28:29 DEBUG ] 19:28:29 DEBUG } 19:28:29 DEBUG ] 19:28:29 DEBUG } 19:28:29 DEBUG } 19:28:29 INFO [http_cmd]: http status code OK 19:28:29 DEBUG --- status code --- 19:28:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:29 DEBUG --- http response --- 19:28:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:28:29 DEBUG ----- output ----- 19:28:29 DEBUG { 19:28:29 DEBUG "status": "success", 19:28:29 DEBUG "data": { 19:28:29 DEBUG "resultType": "vector", 19:28:29 DEBUG "result": [ 19:28:29 DEBUG { 19:28:29 DEBUG "metric": { 19:28:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:28:29 DEBUG }, 19:28:29 DEBUG "value": [ 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG [...] 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:28:29 DEBUG }, 19:28:29 DEBUG "value": [ 19:28:29 DEBUG 1713551309, 19:28:29 DEBUG "3390332928" 19:28:29 DEBUG ] 19:28:29 DEBUG } 19:28:29 DEBUG ] 19:28:29 DEBUG } 19:28:29 DEBUG } 19:28:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551309" 19:28:29 INFO [http_cmd]: http status code OK 19:28:29 DEBUG --- status code --- 19:28:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:29 DEBUG --- http response --- 19:28:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:28:29 DEBUG ----- output ----- 19:28:29 DEBUG { 19:28:29 DEBUG "status": "success", 19:28:29 DEBUG "data": { 19:28:29 DEBUG "resultType": "vector", 19:28:29 DEBUG "result": [ 19:28:29 DEBUG { 19:28:29 DEBUG "metric": { 19:28:29 DEBUG "pod": "ds-idrepo-1" 19:28:29 DEBUG }, 19:28:29 DEBUG "value": [ 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG [...] 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG 19:28:29 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:28:29 DEBUG }, 19:28:29 DEBUG "value": [ 19:28:29 DEBUG 1713551309, 19:28:29 DEBUG "0.00002272404890919038" 19:28:29 DEBUG ] 19:28:29 DEBUG } 19:28:29 DEBUG ] 19:28:29 DEBUG } 19:28:29 DEBUG } 19:28:29 INFO 19:28:29 INFO [loop_until]: kubectl --namespace=xlou top pods 19:28:29 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:28:29 INFO [loop_until]: OK (rc = 0) 19:28:29 DEBUG --- stdout --- 19:28:29 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 14287m 8992Mi am-767f6b67b5-qcqz2 14364m 8998Mi am-767f6b67b5-vbltz 15504m 8982Mi ds-cts-0 15830m 10865Mi ds-cts-1 13413m 11124Mi ds-cts-2 15598m 10856Mi ds-idrepo-0 6525m 3654Mi ds-idrepo-1 6210m 3529Mi ds-idrepo-2 6263m 3493Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 4m 1217Mi lodemon-f8459c597-6tj6f 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7945m 3143Mi 19:28:29 DEBUG --- stderr --- 19:28:29 DEBUG 19:28:29 INFO 19:28:29 INFO [loop_until]: kubectl --namespace=xlou top node 19:28:29 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:28:29 INFO [loop_until]: OK (rc = 0) 19:28:29 DEBUG --- stdout --- 19:28:29 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 72m 0% 6252Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 6483m 21% 7251Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 14145m 47% 11695Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 80m 0% 3339Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 15721m 52% 13897Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 137m 0% 5201Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 14396m 48% 11873Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12334m 41% 3004Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 16341m 54% 11201Mi 10% gke-xlou-cdm-ds-0f601751-j488 6957m 23% 6734Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 13940m 46% 15926Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 6978m 23% 7434Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 15617m 52% 13350Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 7859m 49% 6936Mi 11% 19:28:29 DEBUG --- stderr --- 19:28:29 DEBUG 19:29:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551369" 19:29:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713551369" 19:29:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551369" 19:29:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551369" 19:29:29 ERROR [http_cmd]: ERROR 19:29:29 ERROR --- status code --- 19:29:29 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:29:29 ERROR --- http response --- 19:29:29 INFO [http_cmd]: http status code OK 19:29:29 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:29:29 DEBUG --- status code --- 19:29:29 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:29:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:29 DEBUG --- http response --- 19:29:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:29:29 DEBUG ----- output ----- 19:29:29 DEBUG { 19:29:29 DEBUG "status": "success", 19:29:29 DEBUG "data": { 19:29:29 DEBUG "resultType": "vector", 19:29:29 DEBUG "result": [ 19:29:29 DEBUG { 19:29:29 DEBUG "metric": { 19:29:29 DEBUG "pod": "ds-idrepo-0" 19:29:29 DEBUG }, 19:29:29 DEBUG "value": [ 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG [...] 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG "pod": "ds-idrepo-2" 19:29:29 DEBUG }, 19:29:29 DEBUG "value": [ 19:29:29 DEBUG 1713551369, 19:29:29 DEBUG "0" 19:29:29 DEBUG ] 19:29:29 DEBUG } 19:29:29 DEBUG ] 19:29:29 DEBUG } 19:29:29 DEBUG } 19:29:29 INFO [http_cmd]: http status code OK 19:29:29 DEBUG --- status code --- 19:29:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:29 DEBUG --- http response --- 19:29:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:29:29 DEBUG ----- output ----- 19:29:29 DEBUG { 19:29:29 DEBUG "status": "success", 19:29:29 DEBUG "data": { 19:29:29 DEBUG "resultType": "vector", 19:29:29 DEBUG "result": [ 19:29:29 DEBUG { 19:29:29 DEBUG "metric": { 19:29:29 DEBUG "instance": "10.142.0.107:9100" 19:29:29 DEBUG }, 19:29:29 DEBUG "value": [ 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG [...] 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG "instance": "10.142.0.43:9100" 19:29:29 DEBUG }, 19:29:29 DEBUG "value": [ 19:29:29 DEBUG 1713551369, 19:29:29 DEBUG "1893490230272" 19:29:29 DEBUG ] 19:29:29 DEBUG } 19:29:29 DEBUG ] 19:29:29 DEBUG } 19:29:29 DEBUG } 19:29:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551369" 19:29:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713551369" 19:29:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713551369" 19:29:29 INFO [http_cmd]: http status code OK 19:29:29 DEBUG --- status code --- 19:29:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:29 DEBUG --- http response --- 19:29:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:29:29 DEBUG ----- output ----- 19:29:29 DEBUG { 19:29:29 DEBUG "status": "success", 19:29:29 DEBUG "data": { 19:29:29 DEBUG "resultType": "vector", 19:29:29 DEBUG "result": [ 19:29:29 DEBUG { 19:29:29 DEBUG "metric": { 19:29:29 DEBUG "pod": "ds-idrepo-1" 19:29:29 DEBUG }, 19:29:29 DEBUG "value": [ 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG [...] 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 19:29:29 DEBUG }, 19:29:29 DEBUG "value": [ 19:29:29 DEBUG 1713551369, 19:29:29 DEBUG "0" 19:29:29 DEBUG ] 19:29:29 DEBUG } 19:29:29 DEBUG ] 19:29:29 DEBUG } 19:29:29 DEBUG } 19:29:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551369" 19:29:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551369" 19:29:29 INFO [http_cmd]: http status code OK 19:29:29 INFO [http_cmd]: http status code OK 19:29:29 DEBUG --- status code --- 19:29:29 DEBUG --- status code --- 19:29:29 INFO [http_cmd]: http status code OK 19:29:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:29 DEBUG --- status code --- 19:29:29 DEBUG --- http response --- 19:29:29 DEBUG --- http response --- 19:29:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:29:29 DEBUG --- http response --- 19:29:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:29:29 DEBUG ----- output ----- 19:29:29 DEBUG ----- output ----- 19:29:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:29:29 DEBUG { 19:29:29 DEBUG { 19:29:29 DEBUG ----- output ----- 19:29:29 DEBUG "status": "success", 19:29:29 DEBUG "status": "success", 19:29:29 DEBUG { 19:29:29 DEBUG "data": { 19:29:29 DEBUG "data": { 19:29:29 DEBUG "status": "success", 19:29:29 DEBUG "resultType": "vector", 19:29:29 DEBUG "resultType": "vector", 19:29:29 DEBUG "data": { 19:29:29 DEBUG "result": [ 19:29:29 DEBUG "result": [ 19:29:29 DEBUG "resultType": "vector", 19:29:29 DEBUG { 19:29:29 DEBUG { 19:29:29 DEBUG "result": [ 19:29:29 DEBUG "metric": { 19:29:29 DEBUG "metric": { 19:29:29 DEBUG { 19:29:29 DEBUG "pod": "ds-idrepo-0" 19:29:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:29:29 DEBUG "metric": { 19:29:29 DEBUG }, 19:29:29 DEBUG }, 19:29:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:29:29 DEBUG "value": [ 19:29:29 DEBUG "value": [ 19:29:29 DEBUG }, 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG "value": [ 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG [...] 19:29:29 DEBUG [...] 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG [...] 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG "pod": "ds-cts-2" 19:29:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:29:29 DEBUG 19:29:29 DEBUG }, 19:29:29 DEBUG }, 19:29:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:29:29 DEBUG "value": [ 19:29:29 DEBUG "value": [ 19:29:29 DEBUG }, 19:29:29 DEBUG 1713551369, 19:29:29 DEBUG 1713551369, 19:29:29 DEBUG "value": [ 19:29:29 DEBUG "0" 19:29:29 DEBUG "7.756853472953702" 19:29:29 DEBUG 1713551369, 19:29:29 DEBUG ] 19:29:29 DEBUG ] 19:29:29 DEBUG "2332.0666666666666" 19:29:29 DEBUG } 19:29:29 DEBUG } 19:29:29 DEBUG ] 19:29:29 DEBUG ] 19:29:29 DEBUG ] 19:29:29 DEBUG } 19:29:29 DEBUG } 19:29:29 DEBUG } 19:29:29 DEBUG ] 19:29:29 DEBUG } 19:29:29 DEBUG } 19:29:29 DEBUG } 19:29:29 DEBUG } 19:29:29 INFO [http_cmd]: http status code OK 19:29:29 DEBUG --- status code --- 19:29:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:29 DEBUG --- http response --- 19:29:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:29:29 INFO [http_cmd]: http status code OK 19:29:29 DEBUG ----- output ----- 19:29:29 DEBUG --- status code --- 19:29:29 DEBUG { 19:29:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:29 DEBUG "status": "success", 19:29:29 DEBUG --- http response --- 19:29:29 DEBUG "data": { 19:29:29 DEBUG "resultType": "vector", 19:29:29 DEBUG "result": [ 19:29:29 DEBUG { 19:29:29 DEBUG "metric": { 19:29:29 DEBUG "pod": "ds-cts-0" 19:29:29 DEBUG }, 19:29:29 DEBUG "value": [ 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG [...] 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG "pod": "ds-cts-2" 19:29:29 DEBUG }, 19:29:29 DEBUG "value": [ 19:29:29 DEBUG 1713551369, 19:29:29 DEBUG "0" 19:29:29 DEBUG ] 19:29:29 DEBUG } 19:29:29 DEBUG ] 19:29:29 DEBUG } 19:29:29 DEBUG } 19:29:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:29:29 DEBUG ----- output ----- 19:29:29 DEBUG { 19:29:29 DEBUG "status": "success", 19:29:29 DEBUG "data": { 19:29:29 DEBUG "resultType": "vector", 19:29:29 DEBUG "result": [ 19:29:29 DEBUG { 19:29:29 DEBUG "metric": { 19:29:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:29:29 DEBUG }, 19:29:29 DEBUG "value": [ 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG [...] 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:29:29 DEBUG }, 19:29:29 DEBUG "value": [ 19:29:29 DEBUG 1713551369, 19:29:29 DEBUG "0" 19:29:29 DEBUG ] 19:29:29 DEBUG } 19:29:29 DEBUG ] 19:29:29 DEBUG } 19:29:29 DEBUG } 19:29:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551369" 19:29:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551369" 19:29:29 INFO [http_cmd]: http status code OK 19:29:29 DEBUG --- status code --- 19:29:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:29 DEBUG --- http response --- 19:29:29 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:29:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551369" 19:29:29 INFO [http_cmd]: http status code OK 19:29:29 DEBUG --- status code --- 19:29:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:29 DEBUG --- http response --- 19:29:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:29:29 DEBUG ----- output ----- 19:29:29 DEBUG { 19:29:29 DEBUG "status": "success", 19:29:29 DEBUG "data": { 19:29:29 DEBUG "resultType": "vector", 19:29:29 DEBUG "result": [ 19:29:29 DEBUG { 19:29:29 DEBUG "metric": { 19:29:29 DEBUG "pod": "ds-idrepo-0" 19:29:29 DEBUG }, 19:29:29 DEBUG "value": [ 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG [...] 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG "pod": "ds-idrepo-2" 19:29:29 DEBUG }, 19:29:29 DEBUG "value": [ 19:29:29 DEBUG 1713551369, 19:29:29 DEBUG "0" 19:29:29 DEBUG ] 19:29:29 DEBUG } 19:29:29 DEBUG ] 19:29:29 DEBUG } 19:29:29 DEBUG } 19:29:29 INFO [http_cmd]: http status code OK 19:29:29 DEBUG --- status code --- 19:29:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:29 DEBUG --- http response --- 19:29:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:29:29 DEBUG ----- output ----- 19:29:29 DEBUG { 19:29:29 DEBUG "status": "success", 19:29:29 DEBUG "data": { 19:29:29 DEBUG "resultType": "vector", 19:29:29 DEBUG "result": [ 19:29:29 DEBUG { 19:29:29 DEBUG "metric": { 19:29:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:29:29 DEBUG }, 19:29:29 DEBUG "value": [ 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG [...] 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:29:29 DEBUG }, 19:29:29 DEBUG "value": [ 19:29:29 DEBUG 1713551369, 19:29:29 DEBUG "419931" 19:29:29 DEBUG ] 19:29:29 DEBUG } 19:29:29 DEBUG ] 19:29:29 DEBUG } 19:29:29 DEBUG } 19:29:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551369" 19:29:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551369" 19:29:29 INFO [http_cmd]: http status code OK 19:29:29 DEBUG --- status code --- 19:29:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:29 DEBUG --- http response --- 19:29:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:29:29 DEBUG ----- output ----- 19:29:29 DEBUG { 19:29:29 DEBUG "status": "success", 19:29:29 DEBUG "data": { 19:29:29 DEBUG "resultType": "vector", 19:29:29 DEBUG "result": [ 19:29:29 DEBUG { 19:29:29 DEBUG "metric": { 19:29:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:29:29 DEBUG }, 19:29:29 DEBUG "value": [ 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG [...] 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:29:29 DEBUG }, 19:29:29 DEBUG "value": [ 19:29:29 DEBUG 1713551369, 19:29:29 DEBUG "2328.3" 19:29:29 DEBUG ] 19:29:29 DEBUG } 19:29:29 DEBUG ] 19:29:29 DEBUG } 19:29:29 DEBUG } 19:29:29 INFO [http_cmd]: http status code OK 19:29:29 DEBUG --- status code --- 19:29:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:29 DEBUG --- http response --- 19:29:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:29:29 DEBUG ----- output ----- 19:29:29 DEBUG { 19:29:29 DEBUG "status": "success", 19:29:29 DEBUG "data": { 19:29:29 DEBUG "resultType": "vector", 19:29:29 DEBUG "result": [ 19:29:29 DEBUG { 19:29:29 DEBUG "metric": { 19:29:29 DEBUG "instance": "10.142.0.107:9100" 19:29:29 DEBUG }, 19:29:29 DEBUG "value": [ 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG [...] 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG "instance": "10.142.0.43:9100" 19:29:29 DEBUG }, 19:29:29 DEBUG "value": [ 19:29:29 DEBUG 1713551369, 19:29:29 DEBUG "0.1857333333333088" 19:29:29 DEBUG ] 19:29:29 DEBUG } 19:29:29 DEBUG ] 19:29:29 DEBUG } 19:29:29 DEBUG } 19:29:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551369" 19:29:29 INFO [http_cmd]: http status code OK 19:29:29 DEBUG --- status code --- 19:29:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:29 DEBUG --- http response --- 19:29:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:29:29 DEBUG ----- output ----- 19:29:29 DEBUG { 19:29:29 DEBUG "status": "success", 19:29:29 DEBUG "data": { 19:29:29 DEBUG "resultType": "vector", 19:29:29 DEBUG "result": [ 19:29:29 DEBUG { 19:29:29 DEBUG "metric": { 19:29:29 DEBUG "pod": "ds-idrepo-1" 19:29:29 DEBUG }, 19:29:29 DEBUG "value": [ 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG [...] 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 19:29:29 DEBUG }, 19:29:29 DEBUG "value": [ 19:29:29 DEBUG 1713551369, 19:29:29 DEBUG "0" 19:29:29 DEBUG ] 19:29:29 DEBUG } 19:29:29 DEBUG ] 19:29:29 DEBUG } 19:29:29 DEBUG } 19:29:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551369" 19:29:29 INFO [http_cmd]: http status code OK 19:29:29 DEBUG --- status code --- 19:29:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:29 DEBUG --- http response --- 19:29:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:29:29 DEBUG ----- output ----- 19:29:29 DEBUG { 19:29:29 DEBUG "status": "success", 19:29:29 DEBUG "data": { 19:29:29 DEBUG "resultType": "vector", 19:29:29 DEBUG "result": [ 19:29:29 DEBUG { 19:29:29 DEBUG "metric": { 19:29:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:29:29 DEBUG }, 19:29:29 DEBUG "value": [ 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG [...] 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:29:29 DEBUG }, 19:29:29 DEBUG "value": [ 19:29:29 DEBUG 1713551369, 19:29:29 DEBUG "2329.6" 19:29:29 DEBUG ] 19:29:29 DEBUG } 19:29:29 DEBUG ] 19:29:29 DEBUG } 19:29:29 DEBUG } 19:29:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551369" 19:29:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551369" 19:29:29 INFO [http_cmd]: http status code OK 19:29:29 DEBUG --- status code --- 19:29:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:29 DEBUG --- http response --- 19:29:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:29:29 DEBUG ----- output ----- 19:29:29 DEBUG { 19:29:29 DEBUG "status": "success", 19:29:29 DEBUG "data": { 19:29:29 DEBUG "resultType": "vector", 19:29:29 DEBUG "result": [ 19:29:29 DEBUG { 19:29:29 DEBUG "metric": { 19:29:29 DEBUG "instance": "10.142.0.107:9100" 19:29:29 DEBUG }, 19:29:29 DEBUG "value": [ 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG [...] 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG "instance": "10.142.0.43:9100" 19:29:29 DEBUG }, 19:29:29 DEBUG "value": [ 19:29:29 DEBUG 1713551369, 19:29:29 DEBUG "70528510.6" 19:29:29 DEBUG ] 19:29:29 DEBUG } 19:29:29 DEBUG ] 19:29:29 DEBUG } 19:29:29 DEBUG } 19:29:29 INFO [http_cmd]: http status code OK 19:29:29 DEBUG --- status code --- 19:29:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:29 DEBUG --- http response --- 19:29:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:29:29 DEBUG ----- output ----- 19:29:29 DEBUG { 19:29:29 DEBUG "status": "success", 19:29:29 DEBUG "data": { 19:29:29 DEBUG "resultType": "vector", 19:29:29 DEBUG "result": [ 19:29:29 DEBUG { 19:29:29 DEBUG "metric": { 19:29:29 DEBUG "instance": "10.142.0.107:9100" 19:29:29 DEBUG }, 19:29:29 DEBUG "value": [ 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG [...] 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG "instance": "10.142.0.43:9100" 19:29:29 DEBUG }, 19:29:29 DEBUG "value": [ 19:29:29 DEBUG 1713551369, 19:29:29 DEBUG "0.086333333333323" 19:29:29 DEBUG ] 19:29:29 DEBUG } 19:29:29 DEBUG ] 19:29:29 DEBUG } 19:29:29 DEBUG } 19:29:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551369" 19:29:29 INFO [http_cmd]: http status code OK 19:29:29 DEBUG --- status code --- 19:29:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:29 DEBUG --- http response --- 19:29:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:29:29 DEBUG ----- output ----- 19:29:29 DEBUG { 19:29:29 DEBUG "status": "success", 19:29:29 DEBUG "data": { 19:29:29 DEBUG "resultType": "vector", 19:29:29 DEBUG "result": [ 19:29:29 DEBUG { 19:29:29 DEBUG "metric": { 19:29:29 DEBUG "pod": "ds-idrepo-0" 19:29:29 DEBUG }, 19:29:29 DEBUG "value": [ 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG [...] 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG "pod": "ds-idrepo-2" 19:29:29 DEBUG }, 19:29:29 DEBUG "value": [ 19:29:29 DEBUG 1713551369, 19:29:29 DEBUG "0" 19:29:29 DEBUG ] 19:29:29 DEBUG } 19:29:29 DEBUG ] 19:29:29 DEBUG } 19:29:29 DEBUG } 19:29:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551369" 19:29:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551369" 19:29:29 INFO [http_cmd]: http status code OK 19:29:29 DEBUG --- status code --- 19:29:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:29 DEBUG --- http response --- 19:29:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:29:29 DEBUG ----- output ----- 19:29:29 DEBUG { 19:29:29 DEBUG "status": "success", 19:29:29 DEBUG "data": { 19:29:29 DEBUG "resultType": "vector", 19:29:29 DEBUG "result": [ 19:29:29 DEBUG { 19:29:29 DEBUG "metric": { 19:29:29 DEBUG "pod": "ds-cts-0" 19:29:29 DEBUG }, 19:29:29 DEBUG "value": [ 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG [...] 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG "pod": "ds-cts-2" 19:29:29 DEBUG }, 19:29:29 DEBUG "value": [ 19:29:29 DEBUG 1713551369, 19:29:29 DEBUG "29.966666666666665" 19:29:29 DEBUG ] 19:29:29 DEBUG } 19:29:29 DEBUG ] 19:29:29 DEBUG } 19:29:29 DEBUG } 19:29:29 INFO [http_cmd]: http status code OK 19:29:29 DEBUG --- status code --- 19:29:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:29 DEBUG --- http response --- 19:29:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:29:29 DEBUG ----- output ----- 19:29:29 DEBUG { 19:29:29 DEBUG "status": "success", 19:29:29 DEBUG "data": { 19:29:29 DEBUG "resultType": "vector", 19:29:29 DEBUG "result": [ 19:29:29 DEBUG { 19:29:29 DEBUG "metric": { 19:29:29 DEBUG "pod": "ds-cts-0" 19:29:29 DEBUG }, 19:29:29 DEBUG "value": [ 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG [...] 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG "pod": "ds-cts-2" 19:29:29 DEBUG }, 19:29:29 DEBUG "value": [ 19:29:29 DEBUG 1713551369, 19:29:29 DEBUG "0.0001" 19:29:29 DEBUG ] 19:29:29 DEBUG } 19:29:29 DEBUG ] 19:29:29 DEBUG } 19:29:29 DEBUG } 19:29:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713551369" 19:29:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551369" 19:29:29 INFO [http_cmd]: http status code OK 19:29:29 DEBUG --- status code --- 19:29:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:29 DEBUG --- http response --- 19:29:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:29:29 DEBUG ----- output ----- 19:29:29 DEBUG { 19:29:29 DEBUG "status": "success", 19:29:29 DEBUG "data": { 19:29:29 DEBUG "resultType": "vector", 19:29:29 DEBUG "result": [ 19:29:29 DEBUG { 19:29:29 DEBUG "metric": { 19:29:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:29:29 DEBUG }, 19:29:29 DEBUG "value": [ 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG [...] 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:29:29 DEBUG }, 19:29:29 DEBUG "value": [ 19:29:29 DEBUG 1713551369, 19:29:29 DEBUG "0.007" 19:29:29 DEBUG ] 19:29:29 DEBUG } 19:29:29 DEBUG ] 19:29:29 DEBUG } 19:29:29 DEBUG } 19:29:29 INFO [http_cmd]: http status code OK 19:29:29 DEBUG --- status code --- 19:29:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:29 DEBUG --- http response --- 19:29:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:29:29 DEBUG ----- output ----- 19:29:29 DEBUG { 19:29:29 DEBUG "status": "success", 19:29:29 DEBUG "data": { 19:29:29 DEBUG "resultType": "vector", 19:29:29 DEBUG "result": [ 19:29:29 DEBUG { 19:29:29 DEBUG "metric": { 19:29:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:29:29 DEBUG }, 19:29:29 DEBUG "value": [ 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG [...] 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:29:29 DEBUG }, 19:29:29 DEBUG "value": [ 19:29:29 DEBUG 1713551369, 19:29:29 DEBUG "2328.1666666666665" 19:29:29 DEBUG ] 19:29:29 DEBUG } 19:29:29 DEBUG ] 19:29:29 DEBUG } 19:29:29 DEBUG } 19:29:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551369" 19:29:29 INFO [http_cmd]: http status code OK 19:29:29 DEBUG --- status code --- 19:29:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:29 DEBUG --- http response --- 19:29:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:29:29 DEBUG ----- output ----- 19:29:29 DEBUG { 19:29:29 DEBUG "status": "success", 19:29:29 DEBUG "data": { 19:29:29 DEBUG "resultType": "vector", 19:29:29 DEBUG "result": [ 19:29:29 DEBUG { 19:29:29 DEBUG "metric": { 19:29:29 DEBUG "pod": "ds-idrepo-1" 19:29:29 DEBUG }, 19:29:29 DEBUG "value": [ 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG [...] 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 19:29:29 DEBUG }, 19:29:29 DEBUG "value": [ 19:29:29 DEBUG 1713551369, 19:29:29 DEBUG "0" 19:29:29 DEBUG ] 19:29:29 DEBUG } 19:29:29 DEBUG ] 19:29:29 DEBUG } 19:29:29 DEBUG } 19:29:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551369" 19:29:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713551369" 19:29:29 INFO [http_cmd]: http status code OK 19:29:29 DEBUG --- status code --- 19:29:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:29 DEBUG --- http response --- 19:29:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:29:29 DEBUG ----- output ----- 19:29:29 DEBUG { 19:29:29 DEBUG "status": "success", 19:29:29 DEBUG "data": { 19:29:29 DEBUG "resultType": "vector", 19:29:29 DEBUG "result": [ 19:29:29 DEBUG { 19:29:29 DEBUG "metric": { 19:29:29 DEBUG "instance": "10.142.0.107:9100" 19:29:29 DEBUG }, 19:29:29 DEBUG "value": [ 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG [...] 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG "instance": "10.142.0.43:9100" 19:29:29 DEBUG }, 19:29:29 DEBUG "value": [ 19:29:29 DEBUG 1713551369, 19:29:29 DEBUG "70525533.76666667" 19:29:29 DEBUG ] 19:29:29 DEBUG } 19:29:29 DEBUG ] 19:29:29 DEBUG } 19:29:29 DEBUG } 19:29:29 INFO [http_cmd]: http status code OK 19:29:29 DEBUG --- status code --- 19:29:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:29 DEBUG --- http response --- 19:29:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:29:29 DEBUG ----- output ----- 19:29:29 DEBUG { 19:29:29 DEBUG "status": "success", 19:29:29 DEBUG "data": { 19:29:29 DEBUG "resultType": "vector", 19:29:29 DEBUG "result": [ 19:29:29 DEBUG { 19:29:29 DEBUG "metric": { 19:29:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:29:29 DEBUG }, 19:29:29 DEBUG "value": [ 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG [...] 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:29:29 DEBUG }, 19:29:29 DEBUG "value": [ 19:29:29 DEBUG 1713551369, 19:29:29 DEBUG "3394019328" 19:29:29 DEBUG ] 19:29:29 DEBUG } 19:29:29 DEBUG ] 19:29:29 DEBUG } 19:29:29 DEBUG } 19:29:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551369" 19:29:29 INFO [http_cmd]: http status code OK 19:29:29 DEBUG --- status code --- 19:29:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:29 DEBUG --- http response --- 19:29:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:29:29 DEBUG ----- output ----- 19:29:29 DEBUG { 19:29:29 DEBUG "status": "success", 19:29:29 DEBUG "data": { 19:29:29 DEBUG "resultType": "vector", 19:29:29 DEBUG "result": [ 19:29:29 DEBUG { 19:29:29 DEBUG "metric": { 19:29:29 DEBUG "pod": "ds-idrepo-1" 19:29:29 DEBUG }, 19:29:29 DEBUG "value": [ 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG [...] 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG 19:29:29 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 19:29:29 DEBUG }, 19:29:29 DEBUG "value": [ 19:29:29 DEBUG 1713551369, 19:29:29 DEBUG "0.000019741948260911894" 19:29:29 DEBUG ] 19:29:29 DEBUG } 19:29:29 DEBUG ] 19:29:29 DEBUG } 19:29:29 DEBUG } 19:29:29 INFO 19:29:29 INFO [loop_until]: kubectl --namespace=xlou top pods 19:29:29 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:29:29 INFO [loop_until]: OK (rc = 0) 19:29:29 DEBUG --- stdout --- 19:29:29 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 14267m 8992Mi am-767f6b67b5-qcqz2 14184m 9003Mi am-767f6b67b5-vbltz 15486m 8982Mi ds-cts-0 15196m 10854Mi ds-cts-1 14640m 11109Mi ds-cts-2 15734m 10860Mi ds-idrepo-0 6398m 3654Mi ds-idrepo-1 6380m 3529Mi ds-idrepo-2 6257m 3492Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7650m 3144Mi 19:29:29 DEBUG --- stderr --- 19:29:29 DEBUG 19:29:30 INFO 19:29:30 INFO [loop_until]: kubectl --namespace=xlou top node 19:29:30 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:29:30 INFO [loop_until]: OK (rc = 0) 19:29:30 DEBUG --- stdout --- 19:29:30 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 73m 0% 6257Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 6509m 21% 7240Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 14260m 47% 11703Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 85m 0% 3339Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 15204m 50% 13895Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 156m 0% 5194Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 14497m 48% 11878Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12325m 41% 2997Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 16428m 55% 11200Mi 10% gke-xlou-cdm-ds-0f601751-j488 6945m 23% 6729Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 13997m 46% 15930Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 6968m 23% 7411Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 16153m 54% 13373Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 7859m 49% 6950Mi 11% 19:29:30 DEBUG --- stderr --- 19:29:30 DEBUG 19:30:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551429" 19:30:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713551429" 19:30:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551429" 19:30:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551429" 19:30:29 INFO [http_cmd]: http status code OK 19:30:29 DEBUG --- status code --- 19:30:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:29 DEBUG --- http response --- 19:30:29 INFO [http_cmd]: http status code OK 19:30:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:30:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713551429" 19:30:29 INFO [http_cmd]: http status code OK 19:30:29 DEBUG --- status code --- 19:30:29 DEBUG ----- output ----- 19:30:29 DEBUG --- status code --- 19:30:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:29 DEBUG { 19:30:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:29 DEBUG --- http response --- 19:30:29 DEBUG "status": "success", 19:30:29 DEBUG --- http response --- 19:30:29 DEBUG "data": { 19:30:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:30:29 DEBUG "resultType": "vector", 19:30:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:30:29 DEBUG ----- output ----- 19:30:29 DEBUG "result": [ 19:30:29 DEBUG ----- output ----- 19:30:29 DEBUG { 19:30:29 DEBUG { 19:30:29 DEBUG { 19:30:29 DEBUG "status": "success", 19:30:29 DEBUG "metric": { 19:30:29 DEBUG "status": "success", 19:30:29 DEBUG "data": { 19:30:29 DEBUG "instance": "10.142.0.107:9100" 19:30:29 DEBUG "data": { 19:30:29 DEBUG "resultType": "vector", 19:30:29 DEBUG }, 19:30:29 DEBUG "resultType": "vector", 19:30:29 DEBUG "result": [ 19:30:29 DEBUG "value": [ 19:30:29 DEBUG "result": [ 19:30:29 DEBUG { 19:30:29 DEBUG 19:30:29 DEBUG { 19:30:29 DEBUG "metric": { 19:30:29 DEBUG 19:30:29 DEBUG "metric": { 19:30:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:30:29 DEBUG 19:30:29 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:30:29 DEBUG }, 19:30:29 DEBUG [...] 19:30:29 DEBUG }, 19:30:29 DEBUG "value": [ 19:30:29 DEBUG 19:30:29 DEBUG "value": [ 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG "instance": "10.142.0.43:9100" 19:30:29 DEBUG 19:30:29 DEBUG [...] 19:30:29 DEBUG }, 19:30:29 DEBUG [...] 19:30:29 DEBUG 19:30:29 DEBUG "value": [ 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 1713551429, 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG "1897137030144" 19:30:29 DEBUG 19:30:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:30:29 DEBUG ] 19:30:29 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 19:30:29 DEBUG }, 19:30:29 DEBUG } 19:30:29 DEBUG }, 19:30:29 DEBUG "value": [ 19:30:29 DEBUG ] 19:30:29 DEBUG "value": [ 19:30:29 DEBUG 1713551429, 19:30:29 DEBUG } 19:30:29 DEBUG 1713551429, 19:30:29 DEBUG "7.673382037008776" 19:30:29 DEBUG } 19:30:29 DEBUG "0" 19:30:29 DEBUG ] 19:30:29 DEBUG ] 19:30:29 DEBUG } 19:30:29 DEBUG } 19:30:29 DEBUG ] 19:30:29 DEBUG ] 19:30:29 DEBUG } 19:30:29 DEBUG } 19:30:29 DEBUG } 19:30:29 DEBUG } 19:30:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551429" 19:30:29 ERROR [http_cmd]: ERROR 19:30:29 ERROR --- status code --- 19:30:29 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:30:29 ERROR --- http response --- 19:30:29 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:30:29 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:30:29 INFO [http_cmd]: http status code OK 19:30:29 INFO [http_cmd]: http status code OK 19:30:29 DEBUG --- status code --- 19:30:29 DEBUG --- status code --- 19:30:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:29 DEBUG --- http response --- 19:30:29 DEBUG --- http response --- 19:30:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:30:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:30:29 DEBUG ----- output ----- 19:30:29 DEBUG ----- output ----- 19:30:29 DEBUG { 19:30:29 DEBUG { 19:30:29 DEBUG "status": "success", 19:30:29 DEBUG "status": "success", 19:30:29 DEBUG "data": { 19:30:29 DEBUG "data": { 19:30:29 DEBUG "resultType": "vector", 19:30:29 DEBUG "resultType": "vector", 19:30:29 DEBUG "result": [ 19:30:29 DEBUG "result": [ 19:30:29 DEBUG { 19:30:29 DEBUG { 19:30:29 DEBUG "metric": { 19:30:29 DEBUG "metric": { 19:30:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:30:29 DEBUG "pod": "ds-idrepo-0" 19:30:29 DEBUG }, 19:30:29 DEBUG }, 19:30:29 DEBUG "value": [ 19:30:29 DEBUG "value": [ 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG [...] 19:30:29 DEBUG [...] 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:30:29 DEBUG "pod": "ds-idrepo-2" 19:30:29 DEBUG }, 19:30:29 DEBUG }, 19:30:29 DEBUG "value": [ 19:30:29 DEBUG "value": [ 19:30:29 DEBUG 1713551429, 19:30:29 DEBUG 1713551429, 19:30:29 DEBUG "0" 19:30:29 DEBUG "0" 19:30:29 DEBUG ] 19:30:29 DEBUG ] 19:30:29 DEBUG } 19:30:29 DEBUG } 19:30:29 DEBUG ] 19:30:29 DEBUG ] 19:30:29 DEBUG } 19:30:29 DEBUG } 19:30:29 DEBUG } 19:30:29 DEBUG } 19:30:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551429" 19:30:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713551429" 19:30:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551429" 19:30:29 INFO [http_cmd]: http status code OK 19:30:29 INFO [http_cmd]: http status code OK 19:30:29 DEBUG --- status code --- 19:30:29 DEBUG --- status code --- 19:30:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:29 DEBUG --- http response --- 19:30:29 DEBUG --- http response --- 19:30:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:30:29 DEBUG ----- output ----- 19:30:29 DEBUG { 19:30:29 DEBUG "status": "success", 19:30:29 DEBUG "data": { 19:30:29 DEBUG "resultType": "vector", 19:30:29 DEBUG "result": [ 19:30:29 DEBUG { 19:30:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:30:29 DEBUG "metric": { 19:30:29 DEBUG ----- output ----- 19:30:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:30:29 DEBUG { 19:30:29 DEBUG }, 19:30:29 DEBUG "status": "success", 19:30:29 DEBUG "value": [ 19:30:29 DEBUG "data": { 19:30:29 DEBUG 19:30:29 DEBUG "resultType": "vector", 19:30:29 DEBUG 19:30:29 DEBUG "result": [ 19:30:29 DEBUG 19:30:29 DEBUG { 19:30:29 DEBUG [...] 19:30:29 DEBUG "metric": { 19:30:29 DEBUG 19:30:29 DEBUG "pod": "ds-idrepo-0" 19:30:29 DEBUG 19:30:29 DEBUG }, 19:30:29 DEBUG 19:30:29 DEBUG "value": [ 19:30:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:30:29 DEBUG 19:30:29 DEBUG }, 19:30:29 DEBUG 19:30:29 DEBUG "value": [ 19:30:29 DEBUG 19:30:29 DEBUG 1713551429, 19:30:29 DEBUG [...] 19:30:29 DEBUG "2328.866666666667" 19:30:29 DEBUG 19:30:29 DEBUG ] 19:30:29 DEBUG 19:30:29 DEBUG } 19:30:29 DEBUG 19:30:29 DEBUG ] 19:30:29 DEBUG "pod": "ds-cts-2" 19:30:29 DEBUG } 19:30:29 DEBUG }, 19:30:29 DEBUG } 19:30:29 DEBUG "value": [ 19:30:29 DEBUG 1713551429, 19:30:29 DEBUG "0" 19:30:29 DEBUG ] 19:30:29 DEBUG } 19:30:29 DEBUG ] 19:30:29 DEBUG } 19:30:29 DEBUG } 19:30:29 INFO [http_cmd]: http status code OK 19:30:29 DEBUG --- status code --- 19:30:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:29 DEBUG --- http response --- 19:30:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:30:29 DEBUG ----- output ----- 19:30:29 DEBUG { 19:30:29 DEBUG "status": "success", 19:30:29 DEBUG "data": { 19:30:29 DEBUG "resultType": "vector", 19:30:29 DEBUG "result": [ 19:30:29 DEBUG { 19:30:29 DEBUG "metric": { 19:30:29 DEBUG "pod": "ds-cts-0" 19:30:29 DEBUG }, 19:30:29 DEBUG "value": [ 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG [...] 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG "pod": "ds-cts-2" 19:30:29 DEBUG }, 19:30:29 DEBUG "value": [ 19:30:29 DEBUG 1713551429, 19:30:29 DEBUG "0" 19:30:29 DEBUG ] 19:30:29 DEBUG } 19:30:29 DEBUG ] 19:30:29 DEBUG } 19:30:29 DEBUG } 19:30:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551429" 19:30:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551429" 19:30:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551429" 19:30:29 INFO [http_cmd]: http status code OK 19:30:29 DEBUG --- status code --- 19:30:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:29 DEBUG --- http response --- 19:30:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:30:29 DEBUG ----- output ----- 19:30:29 DEBUG { 19:30:29 DEBUG "status": "success", 19:30:29 DEBUG "data": { 19:30:29 DEBUG "resultType": "vector", 19:30:29 DEBUG "result": [ 19:30:29 DEBUG { 19:30:29 DEBUG "metric": { 19:30:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:30:29 DEBUG }, 19:30:29 DEBUG "value": [ 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG [...] 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:30:29 DEBUG }, 19:30:29 DEBUG "value": [ 19:30:29 DEBUG 1713551429, 19:30:29 DEBUG "2334.6" 19:30:29 DEBUG ] 19:30:29 DEBUG } 19:30:29 DEBUG ] 19:30:29 DEBUG } 19:30:29 DEBUG } 19:30:29 INFO [http_cmd]: http status code OK 19:30:29 DEBUG --- status code --- 19:30:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:29 DEBUG --- http response --- 19:30:29 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:30:29 INFO [http_cmd]: http status code OK 19:30:29 DEBUG --- status code --- 19:30:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:29 DEBUG --- http response --- 19:30:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:30:29 DEBUG ----- output ----- 19:30:29 DEBUG { 19:30:29 DEBUG "status": "success", 19:30:29 DEBUG "data": { 19:30:29 DEBUG "resultType": "vector", 19:30:29 DEBUG "result": [ 19:30:29 DEBUG { 19:30:29 DEBUG "metric": { 19:30:29 DEBUG "pod": "ds-idrepo-0" 19:30:29 DEBUG }, 19:30:29 DEBUG "value": [ 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG [...] 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG "pod": "ds-idrepo-2" 19:30:29 DEBUG }, 19:30:29 DEBUG "value": [ 19:30:29 DEBUG 1713551429, 19:30:29 DEBUG "0" 19:30:29 DEBUG ] 19:30:29 DEBUG } 19:30:29 DEBUG ] 19:30:29 DEBUG } 19:30:29 DEBUG } 19:30:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551429" 19:30:29 INFO [http_cmd]: http status code OK 19:30:29 DEBUG --- status code --- 19:30:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:29 DEBUG --- http response --- 19:30:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:30:29 DEBUG ----- output ----- 19:30:29 DEBUG { 19:30:29 DEBUG "status": "success", 19:30:29 DEBUG "data": { 19:30:29 DEBUG "resultType": "vector", 19:30:29 DEBUG "result": [ 19:30:29 DEBUG { 19:30:29 DEBUG "metric": { 19:30:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:30:29 DEBUG }, 19:30:29 DEBUG "value": [ 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG [...] 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:30:29 DEBUG }, 19:30:29 DEBUG "value": [ 19:30:29 DEBUG 1713551429, 19:30:29 DEBUG "419814.5" 19:30:29 DEBUG ] 19:30:29 DEBUG } 19:30:29 DEBUG ] 19:30:29 DEBUG } 19:30:29 DEBUG } 19:30:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551429" 19:30:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551429" 19:30:29 INFO [http_cmd]: http status code OK 19:30:29 DEBUG --- status code --- 19:30:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:29 DEBUG --- http response --- 19:30:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:30:29 DEBUG ----- output ----- 19:30:29 DEBUG { 19:30:29 DEBUG "status": "success", 19:30:29 DEBUG "data": { 19:30:29 DEBUG "resultType": "vector", 19:30:29 DEBUG "result": [ 19:30:29 DEBUG { 19:30:29 DEBUG "metric": { 19:30:29 DEBUG "instance": "10.142.0.107:9100" 19:30:29 DEBUG }, 19:30:29 DEBUG "value": [ 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG [...] 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG "instance": "10.142.0.43:9100" 19:30:29 DEBUG }, 19:30:29 DEBUG "value": [ 19:30:29 DEBUG 1713551429, 19:30:29 DEBUG "0.16246666666667883" 19:30:29 DEBUG ] 19:30:29 DEBUG } 19:30:29 DEBUG ] 19:30:29 DEBUG } 19:30:29 DEBUG } 19:30:29 INFO [http_cmd]: http status code OK 19:30:29 DEBUG --- status code --- 19:30:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:29 DEBUG --- http response --- 19:30:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:30:29 DEBUG ----- output ----- 19:30:29 DEBUG { 19:30:29 DEBUG "status": "success", 19:30:29 DEBUG "data": { 19:30:29 DEBUG "resultType": "vector", 19:30:29 DEBUG "result": [ 19:30:29 DEBUG { 19:30:29 DEBUG "metric": { 19:30:29 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:30:29 DEBUG }, 19:30:29 DEBUG "value": [ 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG [...] 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 19:30:29 DEBUG }, 19:30:29 DEBUG "value": [ 19:30:29 DEBUG 1713551429, 19:30:29 DEBUG "0" 19:30:29 DEBUG ] 19:30:29 DEBUG } 19:30:29 DEBUG ] 19:30:29 DEBUG } 19:30:29 DEBUG } 19:30:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551429" 19:30:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551429" 19:30:29 INFO [http_cmd]: http status code OK 19:30:29 DEBUG --- status code --- 19:30:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:29 DEBUG --- http response --- 19:30:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:30:29 DEBUG ----- output ----- 19:30:29 DEBUG { 19:30:29 DEBUG "status": "success", 19:30:29 DEBUG "data": { 19:30:29 DEBUG "resultType": "vector", 19:30:29 DEBUG "result": [ 19:30:29 DEBUG { 19:30:29 DEBUG "metric": { 19:30:29 DEBUG "pod": "ds-idrepo-0" 19:30:29 DEBUG }, 19:30:29 DEBUG "value": [ 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG [...] 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG "pod": "ds-idrepo-2" 19:30:29 DEBUG }, 19:30:29 DEBUG "value": [ 19:30:29 DEBUG 1713551429, 19:30:29 DEBUG "0" 19:30:29 DEBUG ] 19:30:29 DEBUG } 19:30:29 DEBUG ] 19:30:29 DEBUG } 19:30:29 DEBUG } 19:30:29 INFO [http_cmd]: http status code OK 19:30:29 DEBUG --- status code --- 19:30:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:29 DEBUG --- http response --- 19:30:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:30:29 DEBUG ----- output ----- 19:30:29 DEBUG { 19:30:29 DEBUG "status": "success", 19:30:29 DEBUG "data": { 19:30:29 DEBUG "resultType": "vector", 19:30:29 DEBUG "result": [ 19:30:29 DEBUG { 19:30:29 DEBUG "metric": { 19:30:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:30:29 DEBUG }, 19:30:29 DEBUG "value": [ 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG [...] 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:30:29 DEBUG }, 19:30:29 DEBUG "value": [ 19:30:29 DEBUG 1713551429, 19:30:29 DEBUG "2336.233333333333" 19:30:29 DEBUG ] 19:30:29 DEBUG } 19:30:29 DEBUG ] 19:30:29 DEBUG } 19:30:29 DEBUG } 19:30:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551429" 19:30:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551429" 19:30:29 INFO [http_cmd]: http status code OK 19:30:29 DEBUG --- status code --- 19:30:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:29 DEBUG --- http response --- 19:30:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:30:29 DEBUG ----- output ----- 19:30:29 DEBUG { 19:30:29 DEBUG "status": "success", 19:30:29 DEBUG "data": { 19:30:29 DEBUG "resultType": "vector", 19:30:29 DEBUG "result": [ 19:30:29 DEBUG { 19:30:29 DEBUG "metric": { 19:30:29 DEBUG "instance": "10.142.0.107:9100" 19:30:29 DEBUG }, 19:30:29 DEBUG "value": [ 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG [...] 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG "instance": "10.142.0.43:9100" 19:30:29 DEBUG }, 19:30:29 DEBUG "value": [ 19:30:29 DEBUG 1713551429, 19:30:29 DEBUG "70705526.43333334" 19:30:29 DEBUG ] 19:30:29 DEBUG } 19:30:29 DEBUG ] 19:30:29 DEBUG } 19:30:29 DEBUG } 19:30:29 INFO [http_cmd]: http status code OK 19:30:29 DEBUG --- status code --- 19:30:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:29 DEBUG --- http response --- 19:30:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:30:29 DEBUG ----- output ----- 19:30:29 DEBUG { 19:30:29 DEBUG "status": "success", 19:30:29 DEBUG "data": { 19:30:29 DEBUG "resultType": "vector", 19:30:29 DEBUG "result": [ 19:30:29 DEBUG { 19:30:29 DEBUG "metric": { 19:30:29 DEBUG "instance": "10.142.0.107:9100" 19:30:29 DEBUG }, 19:30:29 DEBUG "value": [ 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG [...] 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG "instance": "10.142.0.43:9100" 19:30:29 DEBUG }, 19:30:29 DEBUG "value": [ 19:30:29 DEBUG 1713551429, 19:30:29 DEBUG "0.07466666666667074" 19:30:29 DEBUG ] 19:30:29 DEBUG } 19:30:29 DEBUG ] 19:30:29 DEBUG } 19:30:29 DEBUG } 19:30:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551429" 19:30:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551429" 19:30:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551429" 19:30:29 INFO [http_cmd]: http status code OK 19:30:29 DEBUG --- status code --- 19:30:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:29 DEBUG --- http response --- 19:30:29 INFO [http_cmd]: http status code OK 19:30:29 INFO [http_cmd]: http status code OK 19:30:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:30:29 DEBUG --- status code --- 19:30:29 DEBUG --- status code --- 19:30:29 DEBUG ----- output ----- 19:30:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:29 DEBUG { 19:30:29 DEBUG --- http response --- 19:30:29 DEBUG --- http response --- 19:30:29 DEBUG "status": "success", 19:30:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:30:29 DEBUG "data": { 19:30:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:30:29 DEBUG ----- output ----- 19:30:29 DEBUG "resultType": "vector", 19:30:29 DEBUG ----- output ----- 19:30:29 DEBUG { 19:30:29 DEBUG "result": [ 19:30:29 DEBUG { 19:30:29 DEBUG "status": "success", 19:30:29 DEBUG { 19:30:29 DEBUG "status": "success", 19:30:29 DEBUG "data": { 19:30:29 DEBUG "metric": { 19:30:29 DEBUG "data": { 19:30:29 DEBUG "resultType": "vector", 19:30:29 DEBUG "pod": "ds-cts-0" 19:30:29 DEBUG "resultType": "vector", 19:30:29 DEBUG "result": [ 19:30:29 DEBUG }, 19:30:29 DEBUG "result": [ 19:30:29 DEBUG { 19:30:29 DEBUG "value": [ 19:30:29 DEBUG { 19:30:29 DEBUG "metric": { 19:30:29 DEBUG 19:30:29 DEBUG "metric": { 19:30:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:30:29 DEBUG 19:30:29 DEBUG "pod": "ds-cts-0" 19:30:29 DEBUG }, 19:30:29 DEBUG 19:30:29 DEBUG }, 19:30:29 DEBUG "value": [ 19:30:29 DEBUG [...] 19:30:29 DEBUG "value": [ 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG [...] 19:30:29 DEBUG "pod": "ds-cts-2" 19:30:29 DEBUG [...] 19:30:29 DEBUG 19:30:29 DEBUG }, 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG "value": [ 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 1713551429, 19:30:29 DEBUG 19:30:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:30:29 DEBUG "29.966666666666665" 19:30:29 DEBUG "pod": "ds-cts-2" 19:30:29 DEBUG }, 19:30:29 DEBUG ] 19:30:29 DEBUG }, 19:30:29 DEBUG "value": [ 19:30:29 DEBUG } 19:30:29 DEBUG "value": [ 19:30:29 DEBUG 1713551429, 19:30:29 DEBUG ] 19:30:29 DEBUG 1713551429, 19:30:29 DEBUG "2335.7" 19:30:29 DEBUG } 19:30:29 DEBUG "0.000019882222222222222" 19:30:29 DEBUG ] 19:30:29 DEBUG } 19:30:29 DEBUG ] 19:30:29 DEBUG } 19:30:29 DEBUG } 19:30:29 DEBUG ] 19:30:29 DEBUG ] 19:30:29 DEBUG } 19:30:29 DEBUG } 19:30:29 DEBUG } 19:30:29 DEBUG } 19:30:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551429" 19:30:29 INFO [http_cmd]: http status code OK 19:30:29 DEBUG --- status code --- 19:30:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:29 DEBUG --- http response --- 19:30:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:30:29 DEBUG ----- output ----- 19:30:29 DEBUG { 19:30:29 DEBUG "status": "success", 19:30:29 DEBUG "data": { 19:30:29 DEBUG "resultType": "vector", 19:30:29 DEBUG "result": [ 19:30:29 DEBUG { 19:30:29 DEBUG "metric": { 19:30:29 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:30:29 DEBUG }, 19:30:29 DEBUG "value": [ 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG [...] 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 19:30:29 DEBUG }, 19:30:29 DEBUG "value": [ 19:30:29 DEBUG 1713551429, 19:30:29 DEBUG "0" 19:30:29 DEBUG ] 19:30:29 DEBUG } 19:30:29 DEBUG ] 19:30:29 DEBUG } 19:30:29 DEBUG } 19:30:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713551429" 19:30:29 INFO [http_cmd]: http status code OK 19:30:29 DEBUG --- status code --- 19:30:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:29 DEBUG --- http response --- 19:30:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:30:29 DEBUG ----- output ----- 19:30:29 DEBUG { 19:30:29 DEBUG "status": "success", 19:30:29 DEBUG "data": { 19:30:29 DEBUG "resultType": "vector", 19:30:29 DEBUG "result": [ 19:30:29 DEBUG { 19:30:29 DEBUG "metric": { 19:30:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:30:29 DEBUG }, 19:30:29 DEBUG "value": [ 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG [...] 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:30:29 DEBUG }, 19:30:29 DEBUG "value": [ 19:30:29 DEBUG 1713551429, 19:30:29 DEBUG "0.007" 19:30:29 DEBUG ] 19:30:29 DEBUG } 19:30:29 DEBUG ] 19:30:29 DEBUG } 19:30:29 DEBUG } 19:30:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551429" 19:30:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713551429" 19:30:29 INFO [http_cmd]: http status code OK 19:30:29 DEBUG --- status code --- 19:30:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:29 DEBUG --- http response --- 19:30:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:30:29 DEBUG ----- output ----- 19:30:29 DEBUG { 19:30:29 DEBUG "status": "success", 19:30:29 DEBUG "data": { 19:30:29 DEBUG "resultType": "vector", 19:30:29 DEBUG "result": [ 19:30:29 DEBUG { 19:30:29 DEBUG "metric": { 19:30:29 DEBUG "instance": "10.142.0.107:9100" 19:30:29 DEBUG }, 19:30:29 DEBUG "value": [ 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG [...] 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG "instance": "10.142.0.43:9100" 19:30:29 DEBUG }, 19:30:29 DEBUG "value": [ 19:30:29 DEBUG 1713551429, 19:30:29 DEBUG "70703774.5" 19:30:29 DEBUG ] 19:30:29 DEBUG } 19:30:29 DEBUG ] 19:30:29 DEBUG } 19:30:29 DEBUG } 19:30:29 INFO [http_cmd]: http status code OK 19:30:29 DEBUG --- status code --- 19:30:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:29 DEBUG --- http response --- 19:30:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:30:29 DEBUG ----- output ----- 19:30:29 DEBUG { 19:30:29 DEBUG "status": "success", 19:30:29 DEBUG "data": { 19:30:29 DEBUG "resultType": "vector", 19:30:29 DEBUG "result": [ 19:30:29 DEBUG { 19:30:29 DEBUG "metric": { 19:30:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:30:29 DEBUG }, 19:30:29 DEBUG "value": [ 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG [...] 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:30:29 DEBUG }, 19:30:29 DEBUG "value": [ 19:30:29 DEBUG 1713551429, 19:30:29 DEBUG "3396055040" 19:30:29 DEBUG ] 19:30:29 DEBUG } 19:30:29 DEBUG ] 19:30:29 DEBUG } 19:30:29 DEBUG } 19:30:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551429" 19:30:29 INFO [http_cmd]: http status code OK 19:30:29 DEBUG --- status code --- 19:30:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:29 DEBUG --- http response --- 19:30:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:30:29 DEBUG ----- output ----- 19:30:29 DEBUG { 19:30:29 DEBUG "status": "success", 19:30:29 DEBUG "data": { 19:30:29 DEBUG "resultType": "vector", 19:30:29 DEBUG "result": [ 19:30:29 DEBUG { 19:30:29 DEBUG "metric": { 19:30:29 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:30:29 DEBUG }, 19:30:29 DEBUG "value": [ 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG [...] 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG 19:30:29 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 19:30:29 DEBUG }, 19:30:29 DEBUG "value": [ 19:30:29 DEBUG 1713551429, 19:30:29 DEBUG "0.00003065265402485" 19:30:29 DEBUG ] 19:30:29 DEBUG } 19:30:29 DEBUG ] 19:30:29 DEBUG } 19:30:29 DEBUG } 19:30:29 INFO 19:30:29 INFO [loop_until]: kubectl --namespace=xlou top pods 19:30:29 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:30:29 INFO [loop_until]: OK (rc = 0) 19:30:29 DEBUG --- stdout --- 19:30:29 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 14611m 8991Mi am-767f6b67b5-qcqz2 14765m 9001Mi am-767f6b67b5-vbltz 15271m 8983Mi ds-cts-0 15701m 10871Mi ds-cts-1 14200m 11121Mi ds-cts-2 16155m 10862Mi ds-idrepo-0 6422m 3658Mi ds-idrepo-1 6235m 3532Mi ds-idrepo-2 6353m 3492Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7633m 3146Mi 19:30:29 DEBUG --- stderr --- 19:30:29 DEBUG 19:30:30 INFO 19:30:30 INFO [loop_until]: kubectl --namespace=xlou top node 19:30:30 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:30:30 INFO [loop_until]: OK (rc = 0) 19:30:30 DEBUG --- stdout --- 19:30:30 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 72m 0% 6250Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 6533m 21% 7237Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 14290m 47% 11697Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 88m 0% 3339Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 15303m 51% 13899Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 158m 0% 5212Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 14523m 48% 11878Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12348m 41% 3000Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 16455m 55% 11195Mi 10% gke-xlou-cdm-ds-0f601751-j488 6985m 23% 6737Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 14006m 46% 15927Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 7017m 23% 7415Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 15702m 52% 13363Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 7891m 49% 6954Mi 11% 19:30:30 DEBUG --- stderr --- 19:30:30 DEBUG 19:31:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713551489" 19:31:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551489" 19:31:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551489" 19:31:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551489" 19:31:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551489" 19:31:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713551489" 19:31:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551489" 19:31:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551489" 19:31:29 INFO [http_cmd]: http status code OK 19:31:29 INFO [http_cmd]: http status code OK 19:31:29 INFO [http_cmd]: http status code OK 19:31:29 ERROR [http_cmd]: ERROR 19:31:29 INFO [http_cmd]: http status code OK 19:31:29 INFO [http_cmd]: http status code OK 19:31:29 DEBUG --- status code --- 19:31:29 DEBUG --- status code --- 19:31:29 DEBUG --- status code --- 19:31:29 ERROR --- status code --- 19:31:29 DEBUG --- status code --- 19:31:29 DEBUG --- status code --- 19:31:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:29 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:31:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:29 DEBUG --- http response --- 19:31:29 DEBUG --- http response --- 19:31:29 DEBUG --- http response --- 19:31:29 ERROR --- http response --- 19:31:29 DEBUG --- http response --- 19:31:29 DEBUG --- http response --- 19:31:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:31:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:31:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:31:29 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:31:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:31:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:31:29 DEBUG ----- output ----- 19:31:29 DEBUG ----- output ----- 19:31:29 DEBUG ----- output ----- 19:31:29 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:31:29 DEBUG ----- output ----- 19:31:29 DEBUG ----- output ----- 19:31:29 DEBUG { 19:31:29 DEBUG { 19:31:29 DEBUG { 19:31:29 DEBUG { 19:31:29 DEBUG { 19:31:29 DEBUG "status": "success", 19:31:29 DEBUG "status": "success", 19:31:29 DEBUG "status": "success", 19:31:29 DEBUG "status": "success", 19:31:29 DEBUG "status": "success", 19:31:29 DEBUG "data": { 19:31:29 DEBUG "data": { 19:31:29 DEBUG "data": { 19:31:29 DEBUG "data": { 19:31:29 DEBUG "data": { 19:31:29 DEBUG "resultType": "vector", 19:31:29 DEBUG "resultType": "vector", 19:31:29 DEBUG "resultType": "vector", 19:31:29 DEBUG "resultType": "vector", 19:31:29 DEBUG "resultType": "vector", 19:31:29 DEBUG "result": [ 19:31:29 DEBUG "result": [ 19:31:29 DEBUG "result": [ 19:31:29 INFO [http_cmd]: http status code OK 19:31:29 DEBUG "result": [ 19:31:29 DEBUG "result": [ 19:31:29 DEBUG { 19:31:29 DEBUG { 19:31:29 DEBUG { 19:31:29 DEBUG --- status code --- 19:31:29 DEBUG { 19:31:29 DEBUG { 19:31:29 DEBUG "metric": { 19:31:29 DEBUG "metric": { 19:31:29 DEBUG "metric": { 19:31:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:29 DEBUG "metric": { 19:31:29 DEBUG "metric": { 19:31:29 DEBUG "pod": "ds-cts-0" 19:31:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:31:29 DEBUG "instance": "10.142.0.107:9100" 19:31:29 DEBUG --- http response --- 19:31:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:31:29 DEBUG "pod": "ds-idrepo-1" 19:31:29 DEBUG }, 19:31:29 DEBUG }, 19:31:29 DEBUG }, 19:31:29 DEBUG }, 19:31:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:31:29 DEBUG }, 19:31:29 DEBUG "value": [ 19:31:29 DEBUG "value": [ 19:31:29 DEBUG "value": [ 19:31:29 DEBUG "value": [ 19:31:29 DEBUG ----- output ----- 19:31:29 DEBUG "value": [ 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG { 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG "status": "success", 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG "data": { 19:31:29 DEBUG 19:31:29 DEBUG [...] 19:31:29 DEBUG [...] 19:31:29 DEBUG [...] 19:31:29 DEBUG [...] 19:31:29 DEBUG "resultType": "vector", 19:31:29 DEBUG [...] 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG "result": [ 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG { 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG "metric": { 19:31:29 DEBUG 19:31:29 DEBUG "pod": "ds-cts-2" 19:31:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:31:29 DEBUG "instance": "10.142.0.43:9100" 19:31:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:31:29 DEBUG "pod": "ds-idrepo-0" 19:31:29 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:31:29 DEBUG }, 19:31:29 DEBUG }, 19:31:29 DEBUG }, 19:31:29 DEBUG }, 19:31:29 DEBUG }, 19:31:29 DEBUG }, 19:31:29 DEBUG "value": [ 19:31:29 DEBUG "value": [ 19:31:29 DEBUG "value": [ 19:31:29 DEBUG "value": [ 19:31:29 DEBUG "value": [ 19:31:29 DEBUG "value": [ 19:31:29 DEBUG 1713551489, 19:31:29 DEBUG 1713551489, 19:31:29 DEBUG 1713551489, 19:31:29 DEBUG 1713551489, 19:31:29 DEBUG 19:31:29 DEBUG 1713551489, 19:31:29 DEBUG "0" 19:31:29 DEBUG "0" 19:31:29 DEBUG "1900779156480" 19:31:29 DEBUG "7.724283590762695" 19:31:29 DEBUG 19:31:29 DEBUG "0" 19:31:29 DEBUG ] 19:31:29 DEBUG ] 19:31:29 DEBUG ] 19:31:29 DEBUG ] 19:31:29 DEBUG 19:31:29 DEBUG ] 19:31:29 DEBUG } 19:31:29 DEBUG } 19:31:29 DEBUG } 19:31:29 DEBUG } 19:31:29 DEBUG [...] 19:31:29 DEBUG } 19:31:29 DEBUG ] 19:31:29 DEBUG ] 19:31:29 DEBUG ] 19:31:29 DEBUG ] 19:31:29 DEBUG 19:31:29 DEBUG ] 19:31:29 DEBUG } 19:31:29 DEBUG } 19:31:29 DEBUG } 19:31:29 DEBUG } 19:31:29 DEBUG 19:31:29 DEBUG } 19:31:29 DEBUG } 19:31:29 DEBUG } 19:31:29 DEBUG } 19:31:29 DEBUG } 19:31:29 DEBUG 19:31:29 DEBUG } 19:31:29 DEBUG "pod": "ds-idrepo-2" 19:31:29 DEBUG }, 19:31:29 DEBUG "value": [ 19:31:29 DEBUG 1713551489, 19:31:29 DEBUG "0" 19:31:29 DEBUG ] 19:31:29 DEBUG } 19:31:29 DEBUG ] 19:31:29 DEBUG } 19:31:29 DEBUG } 19:31:29 INFO [http_cmd]: http status code OK 19:31:29 DEBUG --- status code --- 19:31:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:29 DEBUG --- http response --- 19:31:29 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:31:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551489" 19:31:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551489" 19:31:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713551489" 19:31:29 INFO [http_cmd]: http status code OK 19:31:29 DEBUG --- status code --- 19:31:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:29 DEBUG --- http response --- 19:31:29 INFO [http_cmd]: http status code OK 19:31:29 INFO [http_cmd]: http status code OK 19:31:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:31:29 DEBUG --- status code --- 19:31:29 DEBUG --- status code --- 19:31:29 DEBUG ----- output ----- 19:31:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:29 DEBUG { 19:31:29 DEBUG --- http response --- 19:31:29 DEBUG --- http response --- 19:31:29 DEBUG "status": "success", 19:31:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:31:29 DEBUG "data": { 19:31:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:31:29 DEBUG ----- output ----- 19:31:29 DEBUG "resultType": "vector", 19:31:29 DEBUG ----- output ----- 19:31:29 DEBUG { 19:31:29 DEBUG "result": [ 19:31:29 DEBUG { 19:31:29 DEBUG "status": "success", 19:31:29 DEBUG { 19:31:29 DEBUG "status": "success", 19:31:29 DEBUG "data": { 19:31:29 DEBUG "metric": { 19:31:29 DEBUG "data": { 19:31:29 DEBUG "resultType": "vector", 19:31:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:31:29 DEBUG "resultType": "vector", 19:31:29 DEBUG "result": [ 19:31:29 DEBUG }, 19:31:29 DEBUG "result": [ 19:31:29 DEBUG { 19:31:29 DEBUG "value": [ 19:31:29 DEBUG { 19:31:29 DEBUG "metric": { 19:31:29 DEBUG 19:31:29 DEBUG "metric": { 19:31:29 DEBUG "pod": "ds-idrepo-0" 19:31:29 DEBUG 19:31:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:31:29 DEBUG }, 19:31:29 DEBUG 19:31:29 DEBUG }, 19:31:29 DEBUG "value": [ 19:31:29 DEBUG [...] 19:31:29 DEBUG "value": [ 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG [...] 19:31:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:31:29 DEBUG [...] 19:31:29 DEBUG 19:31:29 DEBUG }, 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG "value": [ 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 1713551489, 19:31:29 DEBUG 19:31:29 DEBUG "pod": "ds-cts-2" 19:31:29 DEBUG "2332.5" 19:31:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:31:29 DEBUG }, 19:31:29 DEBUG ] 19:31:29 DEBUG }, 19:31:29 DEBUG "value": [ 19:31:29 DEBUG } 19:31:29 DEBUG "value": [ 19:31:29 DEBUG 1713551489, 19:31:29 DEBUG ] 19:31:29 DEBUG 1713551489, 19:31:29 DEBUG "0" 19:31:29 DEBUG } 19:31:29 DEBUG "2333.5" 19:31:29 DEBUG ] 19:31:29 DEBUG } 19:31:29 DEBUG ] 19:31:29 DEBUG } 19:31:29 DEBUG } 19:31:29 DEBUG ] 19:31:29 DEBUG ] 19:31:29 DEBUG } 19:31:29 DEBUG } 19:31:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551489" 19:31:29 DEBUG } 19:31:29 DEBUG } 19:31:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551489" 19:31:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551489" 19:31:29 INFO [http_cmd]: http status code OK 19:31:29 DEBUG --- status code --- 19:31:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:29 DEBUG --- http response --- 19:31:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:31:29 DEBUG ----- output ----- 19:31:29 DEBUG { 19:31:29 DEBUG "status": "success", 19:31:29 DEBUG "data": { 19:31:29 DEBUG "resultType": "vector", 19:31:29 DEBUG "result": [ 19:31:29 DEBUG { 19:31:29 DEBUG "metric": { 19:31:29 DEBUG "pod": "ds-idrepo-0" 19:31:29 DEBUG }, 19:31:29 DEBUG "value": [ 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG [...] 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG "pod": "ds-idrepo-2" 19:31:29 DEBUG }, 19:31:29 DEBUG "value": [ 19:31:29 DEBUG 1713551489, 19:31:29 DEBUG "0" 19:31:29 DEBUG ] 19:31:29 DEBUG } 19:31:29 DEBUG ] 19:31:29 DEBUG } 19:31:29 DEBUG } 19:31:29 INFO [http_cmd]: http status code OK 19:31:29 DEBUG --- status code --- 19:31:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:29 INFO [http_cmd]: http status code OK 19:31:29 DEBUG --- http response --- 19:31:29 DEBUG --- status code --- 19:31:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:29 DEBUG --- http response --- 19:31:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:31:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:31:29 DEBUG ----- output ----- 19:31:29 DEBUG ----- output ----- 19:31:29 DEBUG { 19:31:29 DEBUG { 19:31:29 DEBUG "status": "success", 19:31:29 DEBUG "status": "success", 19:31:29 DEBUG "data": { 19:31:29 DEBUG "data": { 19:31:29 DEBUG "resultType": "vector", 19:31:29 DEBUG "resultType": "vector", 19:31:29 DEBUG "result": [ 19:31:29 DEBUG "result": [ 19:31:29 DEBUG { 19:31:29 DEBUG { 19:31:29 DEBUG "metric": { 19:31:29 DEBUG "metric": { 19:31:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:31:29 DEBUG "instance": "10.142.0.107:9100" 19:31:29 DEBUG }, 19:31:29 DEBUG }, 19:31:29 DEBUG "value": [ 19:31:29 DEBUG "value": [ 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG [...] 19:31:29 DEBUG [...] 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:31:29 DEBUG "instance": "10.142.0.43:9100" 19:31:29 DEBUG }, 19:31:29 DEBUG }, 19:31:29 DEBUG "value": [ 19:31:29 DEBUG "value": [ 19:31:29 DEBUG 1713551489, 19:31:29 DEBUG 1713551489, 19:31:29 DEBUG "419847" 19:31:29 DEBUG "0.24369999999996897" 19:31:29 DEBUG ] 19:31:29 DEBUG ] 19:31:29 DEBUG } 19:31:29 DEBUG } 19:31:29 DEBUG ] 19:31:29 DEBUG ] 19:31:29 DEBUG } 19:31:29 DEBUG } 19:31:29 DEBUG } 19:31:29 DEBUG } 19:31:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551489" 19:31:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551489" 19:31:29 INFO [http_cmd]: http status code OK 19:31:29 INFO [http_cmd]: http status code OK 19:31:29 DEBUG --- status code --- 19:31:29 DEBUG --- status code --- 19:31:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:29 DEBUG --- http response --- 19:31:29 DEBUG --- http response --- 19:31:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:31:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:31:29 DEBUG ----- output ----- 19:31:29 DEBUG ----- output ----- 19:31:29 DEBUG { 19:31:29 DEBUG { 19:31:29 DEBUG "status": "success", 19:31:29 DEBUG "status": "success", 19:31:29 DEBUG "data": { 19:31:29 DEBUG "data": { 19:31:29 DEBUG "resultType": "vector", 19:31:29 DEBUG "resultType": "vector", 19:31:29 DEBUG "result": [ 19:31:29 DEBUG "result": [ 19:31:29 DEBUG { 19:31:29 DEBUG { 19:31:29 DEBUG "metric": { 19:31:29 DEBUG "metric": { 19:31:29 DEBUG "pod": "ds-idrepo-0" 19:31:29 DEBUG "pod": "ds-idrepo-1" 19:31:29 DEBUG }, 19:31:29 DEBUG }, 19:31:29 DEBUG "value": [ 19:31:29 DEBUG "value": [ 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG [...] 19:31:29 DEBUG [...] 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG "pod": "ds-idrepo-2" 19:31:29 DEBUG "pod": "idm-b85f9d978-txpwv" 19:31:29 DEBUG }, 19:31:29 DEBUG }, 19:31:29 DEBUG "value": [ 19:31:29 DEBUG "value": [ 19:31:29 DEBUG 1713551489, 19:31:29 DEBUG 1713551489, 19:31:29 DEBUG "0" 19:31:29 DEBUG "0" 19:31:29 DEBUG ] 19:31:29 DEBUG ] 19:31:29 DEBUG } 19:31:29 DEBUG } 19:31:29 DEBUG ] 19:31:29 DEBUG ] 19:31:29 DEBUG } 19:31:29 DEBUG } 19:31:29 DEBUG } 19:31:29 DEBUG } 19:31:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551489" 19:31:29 INFO [http_cmd]: http status code OK 19:31:29 DEBUG --- status code --- 19:31:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:29 DEBUG --- http response --- 19:31:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:31:29 DEBUG ----- output ----- 19:31:29 DEBUG { 19:31:29 DEBUG "status": "success", 19:31:29 DEBUG "data": { 19:31:29 DEBUG "resultType": "vector", 19:31:29 DEBUG "result": [ 19:31:29 DEBUG { 19:31:29 DEBUG "metric": { 19:31:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:31:29 DEBUG }, 19:31:29 DEBUG "value": [ 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG [...] 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:31:29 DEBUG }, 19:31:29 DEBUG "value": [ 19:31:29 DEBUG 1713551489, 19:31:29 DEBUG "2332.6666666666665" 19:31:29 DEBUG ] 19:31:29 DEBUG } 19:31:29 DEBUG ] 19:31:29 DEBUG } 19:31:29 DEBUG } 19:31:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551489" 19:31:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551489" 19:31:29 INFO [http_cmd]: http status code OK 19:31:29 DEBUG --- status code --- 19:31:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:29 DEBUG --- http response --- 19:31:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:31:29 DEBUG ----- output ----- 19:31:29 DEBUG { 19:31:29 DEBUG "status": "success", 19:31:29 DEBUG "data": { 19:31:29 DEBUG "resultType": "vector", 19:31:29 DEBUG "result": [ 19:31:29 DEBUG { 19:31:29 DEBUG "metric": { 19:31:29 DEBUG "instance": "10.142.0.107:9100" 19:31:29 DEBUG }, 19:31:29 DEBUG "value": [ 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG [...] 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG "instance": "10.142.0.43:9100" 19:31:29 DEBUG }, 19:31:29 DEBUG "value": [ 19:31:29 DEBUG 1713551489, 19:31:29 DEBUG "70733670.76666667" 19:31:29 DEBUG ] 19:31:29 DEBUG } 19:31:29 DEBUG ] 19:31:29 DEBUG } 19:31:29 DEBUG } 19:31:29 INFO [http_cmd]: http status code OK 19:31:29 DEBUG --- status code --- 19:31:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:29 DEBUG --- http response --- 19:31:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:31:29 DEBUG ----- output ----- 19:31:29 DEBUG { 19:31:29 DEBUG "status": "success", 19:31:29 DEBUG "data": { 19:31:29 DEBUG "resultType": "vector", 19:31:29 DEBUG "result": [ 19:31:29 DEBUG { 19:31:29 DEBUG "metric": { 19:31:29 DEBUG "instance": "10.142.0.107:9100" 19:31:29 DEBUG }, 19:31:29 DEBUG "value": [ 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG [...] 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG "instance": "10.142.0.43:9100" 19:31:29 DEBUG }, 19:31:29 DEBUG "value": [ 19:31:29 DEBUG 1713551489, 19:31:29 DEBUG "0.09833333333333676" 19:31:29 DEBUG ] 19:31:29 DEBUG } 19:31:29 DEBUG ] 19:31:29 DEBUG } 19:31:29 DEBUG } 19:31:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551489" 19:31:29 INFO [http_cmd]: http status code OK 19:31:29 DEBUG --- status code --- 19:31:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:29 DEBUG --- http response --- 19:31:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:31:29 DEBUG ----- output ----- 19:31:29 DEBUG { 19:31:29 DEBUG "status": "success", 19:31:29 DEBUG "data": { 19:31:29 DEBUG "resultType": "vector", 19:31:29 DEBUG "result": [ 19:31:29 DEBUG { 19:31:29 DEBUG "metric": { 19:31:29 DEBUG "pod": "ds-cts-0" 19:31:29 DEBUG }, 19:31:29 DEBUG "value": [ 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG [...] 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG "pod": "ds-cts-2" 19:31:29 DEBUG }, 19:31:29 DEBUG "value": [ 19:31:29 DEBUG 1713551489, 19:31:29 DEBUG "0.000033333333333333335" 19:31:29 DEBUG ] 19:31:29 DEBUG } 19:31:29 DEBUG ] 19:31:29 DEBUG } 19:31:29 DEBUG } 19:31:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551489" 19:31:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551489" 19:31:29 INFO [http_cmd]: http status code OK 19:31:29 DEBUG --- status code --- 19:31:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:29 DEBUG --- http response --- 19:31:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:31:29 DEBUG ----- output ----- 19:31:29 DEBUG { 19:31:29 DEBUG "status": "success", 19:31:29 DEBUG "data": { 19:31:29 DEBUG "resultType": "vector", 19:31:29 DEBUG "result": [ 19:31:29 DEBUG { 19:31:29 DEBUG "metric": { 19:31:29 DEBUG "pod": "ds-cts-0" 19:31:29 DEBUG }, 19:31:29 DEBUG "value": [ 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG [...] 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG "pod": "ds-cts-2" 19:31:29 DEBUG }, 19:31:29 DEBUG "value": [ 19:31:29 DEBUG 1713551489, 19:31:29 DEBUG "29.9" 19:31:29 DEBUG ] 19:31:29 DEBUG } 19:31:29 DEBUG ] 19:31:29 DEBUG } 19:31:29 DEBUG } 19:31:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551489" 19:31:29 INFO [http_cmd]: http status code OK 19:31:29 DEBUG --- status code --- 19:31:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:29 DEBUG --- http response --- 19:31:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:31:29 DEBUG ----- output ----- 19:31:29 DEBUG { 19:31:29 DEBUG "status": "success", 19:31:29 DEBUG "data": { 19:31:29 DEBUG "resultType": "vector", 19:31:29 DEBUG "result": [ 19:31:29 DEBUG { 19:31:29 DEBUG "metric": { 19:31:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:31:29 DEBUG }, 19:31:29 DEBUG "value": [ 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG [...] 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:31:29 DEBUG }, 19:31:29 DEBUG "value": [ 19:31:29 DEBUG 1713551489, 19:31:29 DEBUG "2331.9666666666667" 19:31:29 DEBUG ] 19:31:29 DEBUG } 19:31:29 DEBUG ] 19:31:29 DEBUG } 19:31:29 DEBUG } 19:31:29 INFO [http_cmd]: http status code OK 19:31:29 DEBUG --- status code --- 19:31:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:29 DEBUG --- http response --- 19:31:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:31:29 DEBUG ----- output ----- 19:31:29 DEBUG { 19:31:29 DEBUG "status": "success", 19:31:29 DEBUG "data": { 19:31:29 DEBUG "resultType": "vector", 19:31:29 DEBUG "result": [ 19:31:29 DEBUG { 19:31:29 DEBUG "metric": { 19:31:29 DEBUG "pod": "ds-idrepo-1" 19:31:29 DEBUG }, 19:31:29 DEBUG "value": [ 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG [...] 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:31:29 DEBUG }, 19:31:29 DEBUG "value": [ 19:31:29 DEBUG 1713551489, 19:31:29 DEBUG "0" 19:31:29 DEBUG ] 19:31:29 DEBUG } 19:31:29 DEBUG ] 19:31:29 DEBUG } 19:31:29 DEBUG } 19:31:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713551489" 19:31:29 INFO [http_cmd]: http status code OK 19:31:29 DEBUG --- status code --- 19:31:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:29 DEBUG --- http response --- 19:31:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:31:29 DEBUG ----- output ----- 19:31:29 DEBUG { 19:31:29 DEBUG "status": "success", 19:31:29 DEBUG "data": { 19:31:29 DEBUG "resultType": "vector", 19:31:29 DEBUG "result": [ 19:31:29 DEBUG { 19:31:29 DEBUG "metric": { 19:31:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:31:29 DEBUG }, 19:31:29 DEBUG "value": [ 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG [...] 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:31:29 DEBUG }, 19:31:29 DEBUG "value": [ 19:31:29 DEBUG 1713551489, 19:31:29 DEBUG "0.007" 19:31:29 DEBUG ] 19:31:29 DEBUG } 19:31:29 DEBUG ] 19:31:29 DEBUG } 19:31:29 DEBUG } 19:31:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713551489" 19:31:29 INFO [http_cmd]: http status code OK 19:31:29 DEBUG --- status code --- 19:31:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:29 DEBUG --- http response --- 19:31:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:31:29 DEBUG ----- output ----- 19:31:29 DEBUG { 19:31:29 DEBUG "status": "success", 19:31:29 DEBUG "data": { 19:31:29 DEBUG "resultType": "vector", 19:31:29 DEBUG "result": [ 19:31:29 DEBUG { 19:31:29 DEBUG "metric": { 19:31:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:31:29 DEBUG }, 19:31:29 DEBUG "value": [ 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG [...] 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:31:29 DEBUG }, 19:31:29 DEBUG "value": [ 19:31:29 DEBUG 1713551489, 19:31:29 DEBUG "3399090176" 19:31:29 DEBUG ] 19:31:29 DEBUG } 19:31:29 DEBUG ] 19:31:29 DEBUG } 19:31:29 DEBUG } 19:31:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551489" 19:31:29 INFO [http_cmd]: http status code OK 19:31:29 DEBUG --- status code --- 19:31:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:29 DEBUG --- http response --- 19:31:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:31:29 DEBUG ----- output ----- 19:31:29 DEBUG { 19:31:29 DEBUG "status": "success", 19:31:29 DEBUG "data": { 19:31:29 DEBUG "resultType": "vector", 19:31:29 DEBUG "result": [ 19:31:29 DEBUG { 19:31:29 DEBUG "metric": { 19:31:29 DEBUG "instance": "10.142.0.107:9100" 19:31:29 DEBUG }, 19:31:29 DEBUG "value": [ 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG [...] 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG "instance": "10.142.0.43:9100" 19:31:29 DEBUG }, 19:31:29 DEBUG "value": [ 19:31:29 DEBUG 1713551489, 19:31:29 DEBUG "70730955.96666667" 19:31:29 DEBUG ] 19:31:29 DEBUG } 19:31:29 DEBUG ] 19:31:29 DEBUG } 19:31:29 DEBUG } 19:31:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551489" 19:31:29 INFO [http_cmd]: http status code OK 19:31:29 DEBUG --- status code --- 19:31:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:29 DEBUG --- http response --- 19:31:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:31:29 DEBUG ----- output ----- 19:31:29 DEBUG { 19:31:29 DEBUG "status": "success", 19:31:29 DEBUG "data": { 19:31:29 DEBUG "resultType": "vector", 19:31:29 DEBUG "result": [ 19:31:29 DEBUG { 19:31:29 DEBUG "metric": { 19:31:29 DEBUG "pod": "ds-idrepo-1" 19:31:29 DEBUG }, 19:31:29 DEBUG "value": [ 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG [...] 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG 19:31:29 DEBUG "pod": "idm-b85f9d978-txpwv" 19:31:29 DEBUG }, 19:31:29 DEBUG "value": [ 19:31:29 DEBUG 1713551489, 19:31:29 DEBUG "0" 19:31:29 DEBUG ] 19:31:29 DEBUG } 19:31:29 DEBUG ] 19:31:29 DEBUG } 19:31:29 DEBUG } 19:31:29 INFO 19:31:29 INFO [loop_until]: kubectl --namespace=xlou top pods 19:31:29 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:31:29 INFO [loop_until]: OK (rc = 0) 19:31:29 DEBUG --- stdout --- 19:31:29 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 14446m 8989Mi am-767f6b67b5-qcqz2 14674m 8996Mi am-767f6b67b5-vbltz 15653m 8986Mi ds-cts-0 15559m 10859Mi ds-cts-1 13778m 11122Mi ds-cts-2 15524m 10875Mi ds-idrepo-0 6430m 3658Mi ds-idrepo-1 6015m 3532Mi ds-idrepo-2 6357m 3492Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7772m 3148Mi 19:31:29 DEBUG --- stderr --- 19:31:29 DEBUG 19:31:30 INFO 19:31:30 INFO [loop_until]: kubectl --namespace=xlou top node 19:31:30 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:31:30 INFO [loop_until]: OK (rc = 0) 19:31:30 DEBUG --- stdout --- 19:31:30 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 69m 0% 6253Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 6477m 21% 7242Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 14188m 47% 11688Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 3346Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 15684m 52% 13897Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 138m 0% 5200Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 14378m 48% 11868Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12271m 41% 3009Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 16427m 55% 11183Mi 10% gke-xlou-cdm-ds-0f601751-j488 6955m 23% 6725Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 13854m 46% 15925Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 6898m 23% 7424Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 15602m 52% 13362Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 7853m 49% 6944Mi 11% 19:31:30 DEBUG --- stderr --- 19:31:30 DEBUG 19:32:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713551549" 19:32:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551549" 19:32:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551549" 19:32:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713551549" 19:32:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551549" 19:32:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551549" 19:32:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551549" 19:32:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551549" 19:32:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551549" 19:32:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551549" 19:32:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713551549" 19:32:29 INFO [http_cmd]: http status code OK 19:32:29 INFO [http_cmd]: http status code OK 19:32:29 ERROR [http_cmd]: ERROR 19:32:29 INFO [http_cmd]: http status code OK 19:32:29 INFO [http_cmd]: http status code OK 19:32:29 DEBUG --- status code --- 19:32:29 DEBUG --- status code --- 19:32:29 INFO [http_cmd]: http status code OK 19:32:29 ERROR --- status code --- 19:32:29 DEBUG --- status code --- 19:32:29 DEBUG --- status code --- 19:32:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:29 DEBUG --- status code --- 19:32:29 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:32:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:29 DEBUG --- http response --- 19:32:29 DEBUG --- http response --- 19:32:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:29 INFO [http_cmd]: http status code OK 19:32:29 ERROR --- http response --- 19:32:29 DEBUG --- http response --- 19:32:29 DEBUG --- http response --- 19:32:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:32:29 DEBUG --- http response --- 19:32:29 INFO [http_cmd]: http status code OK 19:32:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:32:29 DEBUG --- status code --- 19:32:29 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:32:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:32:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:32:29 DEBUG ----- output ----- 19:32:29 DEBUG --- status code --- 19:32:29 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:32:29 DEBUG ----- output ----- 19:32:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:29 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:32:29 DEBUG ----- output ----- 19:32:29 DEBUG ----- output ----- 19:32:29 DEBUG { 19:32:29 DEBUG "status": "success", 19:32:29 DEBUG "data": { 19:32:29 DEBUG --- http response --- 19:32:29 DEBUG { 19:32:29 DEBUG { 19:32:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:29 DEBUG { 19:32:29 DEBUG "resultType": "vector", 19:32:29 INFO [http_cmd]: http status code OK 19:32:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:32:29 INFO [http_cmd]: http status code OK 19:32:29 DEBUG "status": "success", 19:32:29 DEBUG "status": "success", 19:32:29 DEBUG --- http response --- 19:32:29 DEBUG "status": "success", 19:32:29 DEBUG "result": [ 19:32:29 DEBUG --- status code --- 19:32:29 DEBUG ----- output ----- 19:32:29 DEBUG --- status code --- 19:32:29 DEBUG "data": { 19:32:29 DEBUG "data": { 19:32:29 DEBUG "data": { 19:32:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:32:29 DEBUG { 19:32:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:29 INFO [http_cmd]: http status code OK 19:32:29 DEBUG { 19:32:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:29 DEBUG "resultType": "vector", 19:32:29 DEBUG "resultType": "vector", 19:32:29 DEBUG "resultType": "vector", 19:32:29 DEBUG ----- output ----- 19:32:29 DEBUG "metric": { 19:32:29 DEBUG --- http response --- 19:32:29 DEBUG --- status code --- 19:32:29 DEBUG "status": "success", 19:32:29 DEBUG --- http response --- 19:32:29 DEBUG "result": [ 19:32:29 DEBUG "result": [ 19:32:29 DEBUG "result": [ 19:32:29 DEBUG { 19:32:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:32:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:32:29 DEBUG "data": { 19:32:29 DEBUG { 19:32:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 19:32:29 DEBUG { 19:32:29 DEBUG { 19:32:29 DEBUG "status": "success", 19:32:29 DEBUG }, 19:32:29 DEBUG --- http response --- 19:32:29 DEBUG ----- output ----- 19:32:29 DEBUG "resultType": "vector", 19:32:29 DEBUG "metric": { 19:32:29 DEBUG ----- output ----- 19:32:29 DEBUG "metric": { 19:32:29 DEBUG "metric": { 19:32:29 DEBUG "data": { 19:32:29 DEBUG "value": [ 19:32:29 DEBUG { 19:32:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:32:29 DEBUG "result": [ 19:32:29 DEBUG "pod": "ds-idrepo-0" 19:32:29 DEBUG { 19:32:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:32:29 DEBUG "instance": "10.142.0.107:9100" 19:32:29 DEBUG "resultType": "vector", 19:32:29 DEBUG 19:32:29 DEBUG "status": "success", 19:32:29 DEBUG ----- output ----- 19:32:29 DEBUG { 19:32:29 DEBUG }, 19:32:29 DEBUG "status": "success", 19:32:29 DEBUG }, 19:32:29 DEBUG }, 19:32:29 DEBUG "result": [ 19:32:29 DEBUG 19:32:29 DEBUG "data": { 19:32:29 DEBUG { 19:32:29 DEBUG "metric": { 19:32:29 DEBUG "value": [ 19:32:29 DEBUG "data": { 19:32:29 DEBUG "value": [ 19:32:29 DEBUG "value": [ 19:32:29 DEBUG { 19:32:29 DEBUG 19:32:29 DEBUG "resultType": "vector", 19:32:29 DEBUG "status": "success", 19:32:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:32:29 DEBUG 19:32:29 DEBUG "resultType": "vector", 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG "metric": { 19:32:29 DEBUG [...] 19:32:29 DEBUG "result": [ 19:32:29 DEBUG "data": { 19:32:29 DEBUG }, 19:32:29 DEBUG 19:32:29 DEBUG "result": [ 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:32:29 DEBUG 19:32:29 DEBUG { 19:32:29 DEBUG "resultType": "vector", 19:32:29 DEBUG "value": [ 19:32:29 DEBUG 19:32:29 DEBUG { 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG }, 19:32:29 DEBUG 19:32:29 DEBUG "metric": { 19:32:29 DEBUG "result": [ 19:32:29 DEBUG 19:32:29 DEBUG [...] 19:32:29 DEBUG "metric": { 19:32:29 DEBUG [...] 19:32:29 DEBUG [...] 19:32:29 DEBUG "value": [ 19:32:29 DEBUG 19:32:29 DEBUG "pod": "ds-cts-0" 19:32:29 DEBUG { 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:32:29 DEBUG }, 19:32:29 DEBUG "metric": { 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG }, 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG }, 19:32:29 DEBUG "value": [ 19:32:29 DEBUG "pod": "ds-idrepo-0" 19:32:29 DEBUG [...] 19:32:29 DEBUG 19:32:29 DEBUG "value": [ 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG "value": [ 19:32:29 DEBUG 19:32:29 DEBUG }, 19:32:29 DEBUG 19:32:29 DEBUG "pod": "ds-idrepo-2" 19:32:29 DEBUG 19:32:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:32:29 DEBUG "instance": "10.142.0.43:9100" 19:32:29 DEBUG [...] 19:32:29 DEBUG 1713551549, 19:32:29 DEBUG 19:32:29 DEBUG "value": [ 19:32:29 DEBUG 19:32:29 DEBUG }, 19:32:29 DEBUG 19:32:29 DEBUG }, 19:32:29 DEBUG }, 19:32:29 DEBUG 19:32:29 DEBUG "7.69150764023705" 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG "value": [ 19:32:29 DEBUG 19:32:29 DEBUG "value": [ 19:32:29 DEBUG "value": [ 19:32:29 DEBUG 19:32:29 DEBUG ] 19:32:29 DEBUG [...] 19:32:29 DEBUG 19:32:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:32:29 DEBUG 1713551549, 19:32:29 DEBUG [...] 19:32:29 DEBUG 1713551549, 19:32:29 DEBUG 1713551549, 19:32:29 DEBUG 19:32:29 DEBUG } 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG }, 19:32:29 DEBUG "0" 19:32:29 DEBUG 19:32:29 DEBUG "2330.3" 19:32:29 DEBUG "1904163568640" 19:32:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:32:29 DEBUG ] 19:32:29 DEBUG 19:32:29 DEBUG [...] 19:32:29 DEBUG "value": [ 19:32:29 DEBUG ] 19:32:29 DEBUG 19:32:29 DEBUG ] 19:32:29 DEBUG ] 19:32:29 DEBUG }, 19:32:29 DEBUG } 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG 1713551549, 19:32:29 DEBUG } 19:32:29 DEBUG 19:32:29 DEBUG } 19:32:29 DEBUG } 19:32:29 DEBUG "value": [ 19:32:29 DEBUG } 19:32:29 DEBUG "pod": "ds-cts-2" 19:32:29 DEBUG 19:32:29 DEBUG "420176.5" 19:32:29 DEBUG ] 19:32:29 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:32:29 DEBUG ] 19:32:29 DEBUG ] 19:32:29 DEBUG 1713551549, 19:32:29 DEBUG }, 19:32:29 DEBUG 19:32:29 DEBUG ] 19:32:29 DEBUG } 19:32:29 DEBUG }, 19:32:29 DEBUG } 19:32:29 DEBUG } 19:32:29 DEBUG "0" 19:32:29 DEBUG "value": [ 19:32:29 DEBUG "pod": "ds-cts-2" 19:32:29 DEBUG } 19:32:29 DEBUG } 19:32:29 DEBUG "value": [ 19:32:29 DEBUG } 19:32:29 DEBUG } 19:32:29 DEBUG ] 19:32:29 DEBUG 1713551549, 19:32:29 DEBUG }, 19:32:29 DEBUG ] 19:32:29 DEBUG 1713551549, 19:32:29 DEBUG } 19:32:29 DEBUG "0" 19:32:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551549" 19:32:29 DEBUG "value": [ 19:32:29 DEBUG } 19:32:29 DEBUG "0" 19:32:29 DEBUG ] 19:32:29 DEBUG ] 19:32:29 DEBUG 1713551549, 19:32:29 DEBUG } 19:32:29 DEBUG ] 19:32:29 DEBUG } 19:32:29 DEBUG } 19:32:29 DEBUG "0" 19:32:29 DEBUG } 19:32:29 DEBUG } 19:32:29 DEBUG ] 19:32:29 DEBUG ] 19:32:29 DEBUG ] 19:32:29 DEBUG } 19:32:29 DEBUG } 19:32:29 DEBUG } 19:32:29 DEBUG } 19:32:29 DEBUG ] 19:32:29 DEBUG } 19:32:29 DEBUG } 19:32:29 DEBUG } 19:32:29 INFO [http_cmd]: http status code OK 19:32:29 DEBUG --- status code --- 19:32:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:29 DEBUG --- http response --- 19:32:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:32:29 DEBUG ----- output ----- 19:32:29 DEBUG { 19:32:29 DEBUG "status": "success", 19:32:29 DEBUG "data": { 19:32:29 DEBUG "resultType": "vector", 19:32:29 DEBUG "result": [ 19:32:29 DEBUG { 19:32:29 DEBUG "metric": { 19:32:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:32:29 DEBUG }, 19:32:29 DEBUG "value": [ 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG [...] 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:32:29 DEBUG }, 19:32:29 DEBUG "value": [ 19:32:29 DEBUG 1713551549, 19:32:29 DEBUG "2339.4" 19:32:29 DEBUG ] 19:32:29 DEBUG } 19:32:29 DEBUG ] 19:32:29 DEBUG } 19:32:29 DEBUG } 19:32:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551549" 19:32:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551549" 19:32:29 INFO [http_cmd]: http status code OK 19:32:29 DEBUG --- status code --- 19:32:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:29 DEBUG --- http response --- 19:32:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:32:29 DEBUG ----- output ----- 19:32:29 DEBUG { 19:32:29 DEBUG "status": "success", 19:32:29 DEBUG "data": { 19:32:29 DEBUG "resultType": "vector", 19:32:29 DEBUG "result": [ 19:32:29 DEBUG { 19:32:29 DEBUG "metric": { 19:32:29 DEBUG "pod": "ds-idrepo-0" 19:32:29 DEBUG }, 19:32:29 DEBUG "value": [ 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG [...] 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG "pod": "ds-idrepo-2" 19:32:29 DEBUG }, 19:32:29 DEBUG "value": [ 19:32:29 DEBUG 1713551549, 19:32:29 DEBUG "0" 19:32:29 DEBUG ] 19:32:29 DEBUG } 19:32:29 DEBUG ] 19:32:29 DEBUG } 19:32:29 DEBUG } 19:32:29 INFO [http_cmd]: http status code OK 19:32:29 DEBUG --- status code --- 19:32:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:29 DEBUG --- http response --- 19:32:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:32:29 DEBUG ----- output ----- 19:32:29 DEBUG { 19:32:29 DEBUG "status": "success", 19:32:29 DEBUG "data": { 19:32:29 DEBUG "resultType": "vector", 19:32:29 DEBUG "result": [ 19:32:29 DEBUG { 19:32:29 DEBUG "metric": { 19:32:29 DEBUG "instance": "10.142.0.107:9100" 19:32:29 DEBUG }, 19:32:29 DEBUG "value": [ 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG [...] 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG "instance": "10.142.0.43:9100" 19:32:29 DEBUG }, 19:32:29 DEBUG "value": [ 19:32:29 DEBUG 1713551549, 19:32:29 DEBUG "0.15463333333336018" 19:32:29 DEBUG ] 19:32:29 DEBUG } 19:32:29 DEBUG ] 19:32:29 DEBUG } 19:32:29 DEBUG } 19:32:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551549" 19:32:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551549" 19:32:29 INFO [http_cmd]: http status code OK 19:32:29 DEBUG --- status code --- 19:32:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:29 DEBUG --- http response --- 19:32:29 INFO [http_cmd]: http status code OK 19:32:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:32:29 DEBUG --- status code --- 19:32:29 DEBUG ----- output ----- 19:32:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:29 DEBUG { 19:32:29 DEBUG --- http response --- 19:32:29 DEBUG "status": "success", 19:32:29 DEBUG "data": { 19:32:29 DEBUG "resultType": "vector", 19:32:29 DEBUG "result": [ 19:32:29 DEBUG { 19:32:29 DEBUG "metric": { 19:32:29 DEBUG "pod": "ds-idrepo-0" 19:32:29 DEBUG }, 19:32:29 DEBUG "value": [ 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG [...] 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 19:32:29 DEBUG 19:32:29 DEBUG ----- output ----- 19:32:29 DEBUG "pod": "ds-idrepo-2" 19:32:29 DEBUG { 19:32:29 DEBUG }, 19:32:29 DEBUG "status": "success", 19:32:29 DEBUG "value": [ 19:32:29 DEBUG "data": { 19:32:29 DEBUG 1713551549, 19:32:29 DEBUG "resultType": "vector", 19:32:29 DEBUG "0" 19:32:29 DEBUG "result": [ 19:32:29 DEBUG ] 19:32:29 DEBUG { 19:32:29 DEBUG } 19:32:29 DEBUG "metric": { 19:32:29 DEBUG ] 19:32:29 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:32:29 DEBUG } 19:32:29 DEBUG }, 19:32:29 DEBUG } 19:32:29 DEBUG "value": [ 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG [...] 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:32:29 DEBUG }, 19:32:29 DEBUG "value": [ 19:32:29 DEBUG 1713551549, 19:32:29 DEBUG "92486618.9852814" 19:32:29 DEBUG ] 19:32:29 DEBUG } 19:32:29 DEBUG ] 19:32:29 DEBUG } 19:32:29 DEBUG } 19:32:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551549" 19:32:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551549" 19:32:29 INFO [http_cmd]: http status code OK 19:32:29 DEBUG --- status code --- 19:32:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:29 DEBUG --- http response --- 19:32:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:32:29 DEBUG ----- output ----- 19:32:29 DEBUG { 19:32:29 DEBUG "status": "success", 19:32:29 DEBUG "data": { 19:32:29 DEBUG "resultType": "vector", 19:32:29 DEBUG "result": [ 19:32:29 DEBUG { 19:32:29 DEBUG "metric": { 19:32:29 DEBUG "instance": "10.142.0.107:9100" 19:32:29 DEBUG }, 19:32:29 DEBUG "value": [ 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG [...] 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG "instance": "10.142.0.43:9100" 19:32:29 DEBUG }, 19:32:29 DEBUG "value": [ 19:32:29 DEBUG 1713551549, 19:32:29 DEBUG "70826753.03333333" 19:32:29 DEBUG ] 19:32:29 DEBUG } 19:32:29 DEBUG ] 19:32:29 DEBUG } 19:32:29 DEBUG } 19:32:29 INFO [http_cmd]: http status code OK 19:32:29 DEBUG --- status code --- 19:32:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:29 DEBUG --- http response --- 19:32:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:32:29 DEBUG ----- output ----- 19:32:29 DEBUG { 19:32:29 DEBUG "status": "success", 19:32:29 DEBUG "data": { 19:32:29 DEBUG "resultType": "vector", 19:32:29 DEBUG "result": [ 19:32:29 DEBUG { 19:32:29 DEBUG "metric": { 19:32:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:32:29 DEBUG }, 19:32:29 DEBUG "value": [ 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG [...] 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:32:29 DEBUG }, 19:32:29 DEBUG "value": [ 19:32:29 DEBUG 1713551549, 19:32:29 DEBUG "2339.5333333333333" 19:32:29 DEBUG ] 19:32:29 DEBUG } 19:32:29 DEBUG ] 19:32:29 DEBUG } 19:32:29 DEBUG } 19:32:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551549" 19:32:29 INFO [http_cmd]: http status code OK 19:32:29 DEBUG --- status code --- 19:32:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:29 DEBUG --- http response --- 19:32:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:32:29 DEBUG ----- output ----- 19:32:29 DEBUG { 19:32:29 DEBUG "status": "success", 19:32:29 DEBUG "data": { 19:32:29 DEBUG "resultType": "vector", 19:32:29 DEBUG "result": [ 19:32:29 DEBUG { 19:32:29 DEBUG "metric": { 19:32:29 DEBUG "instance": "10.142.0.107:9100" 19:32:29 DEBUG }, 19:32:29 DEBUG "value": [ 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG [...] 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG "instance": "10.142.0.43:9100" 19:32:29 DEBUG }, 19:32:29 DEBUG "value": [ 19:32:29 DEBUG 1713551549, 19:32:29 DEBUG "0.06866666666666862" 19:32:29 DEBUG ] 19:32:29 DEBUG } 19:32:29 DEBUG ] 19:32:29 DEBUG } 19:32:29 DEBUG } 19:32:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551549" 19:32:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551549" 19:32:29 INFO [http_cmd]: http status code OK 19:32:29 INFO [http_cmd]: http status code OK 19:32:29 DEBUG --- status code --- 19:32:29 DEBUG --- status code --- 19:32:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:29 DEBUG --- http response --- 19:32:29 DEBUG --- http response --- 19:32:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:32:29 DEBUG ----- output ----- 19:32:29 DEBUG { 19:32:29 DEBUG "status": "success", 19:32:29 DEBUG "data": { 19:32:29 DEBUG "resultType": "vector", 19:32:29 DEBUG "result": [ 19:32:29 DEBUG { 19:32:29 DEBUG "metric": { 19:32:29 DEBUG "pod": "ds-cts-0" 19:32:29 DEBUG }, 19:32:29 DEBUG "value": [ 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG [...] 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG "pod": "ds-cts-2" 19:32:29 DEBUG }, 19:32:29 DEBUG "value": [ 19:32:29 DEBUG 1713551549, 19:32:29 DEBUG "0.000019882222222222222" 19:32:29 DEBUG ] 19:32:29 DEBUG } 19:32:29 DEBUG ] 19:32:29 DEBUG } 19:32:29 DEBUG } 19:32:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:32:29 DEBUG ----- output ----- 19:32:29 DEBUG { 19:32:29 DEBUG "status": "success", 19:32:29 DEBUG "data": { 19:32:29 DEBUG "resultType": "vector", 19:32:29 DEBUG "result": [ 19:32:29 DEBUG { 19:32:29 DEBUG "metric": { 19:32:29 DEBUG "pod": "ds-cts-0" 19:32:29 DEBUG }, 19:32:29 DEBUG "value": [ 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG [...] 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG "pod": "ds-cts-2" 19:32:29 DEBUG }, 19:32:29 DEBUG "value": [ 19:32:29 DEBUG 1713551549, 19:32:29 DEBUG "29.966666666666665" 19:32:29 DEBUG ] 19:32:29 DEBUG } 19:32:29 DEBUG ] 19:32:29 DEBUG } 19:32:29 DEBUG } 19:32:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551549" 19:32:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551549" 19:32:29 INFO [http_cmd]: http status code OK 19:32:29 DEBUG --- status code --- 19:32:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:29 DEBUG --- http response --- 19:32:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:32:29 DEBUG ----- output ----- 19:32:29 DEBUG { 19:32:29 DEBUG "status": "success", 19:32:29 DEBUG "data": { 19:32:29 DEBUG "resultType": "vector", 19:32:29 DEBUG "result": [ 19:32:29 DEBUG { 19:32:29 DEBUG "metric": { 19:32:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:32:29 DEBUG }, 19:32:29 DEBUG "value": [ 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG [...] 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:32:29 DEBUG }, 19:32:29 DEBUG "value": [ 19:32:29 DEBUG 1713551549, 19:32:29 DEBUG "2341.0333333333333" 19:32:29 DEBUG ] 19:32:29 DEBUG } 19:32:29 DEBUG ] 19:32:29 DEBUG } 19:32:29 DEBUG } 19:32:29 INFO [http_cmd]: http status code OK 19:32:29 DEBUG --- status code --- 19:32:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:29 DEBUG --- http response --- 19:32:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 19:32:29 DEBUG ----- output ----- 19:32:29 DEBUG { 19:32:29 DEBUG "status": "success", 19:32:29 DEBUG "data": { 19:32:29 DEBUG "resultType": "vector", 19:32:29 DEBUG "result": [ 19:32:29 DEBUG { 19:32:29 DEBUG "metric": { 19:32:29 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:32:29 DEBUG }, 19:32:29 DEBUG "value": [ 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG [...] 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:32:29 DEBUG }, 19:32:29 DEBUG "value": [ 19:32:29 DEBUG 1713551549, 19:32:29 DEBUG "0" 19:32:29 DEBUG ] 19:32:29 DEBUG } 19:32:29 DEBUG ] 19:32:29 DEBUG } 19:32:29 DEBUG } 19:32:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713551549" 19:32:29 INFO [http_cmd]: http status code OK 19:32:29 DEBUG --- status code --- 19:32:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:29 DEBUG --- http response --- 19:32:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:32:29 DEBUG ----- output ----- 19:32:29 DEBUG { 19:32:29 DEBUG "status": "success", 19:32:29 DEBUG "data": { 19:32:29 DEBUG "resultType": "vector", 19:32:29 DEBUG "result": [ 19:32:29 DEBUG { 19:32:29 DEBUG "metric": { 19:32:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:32:29 DEBUG }, 19:32:29 DEBUG "value": [ 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG [...] 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:32:29 DEBUG }, 19:32:29 DEBUG "value": [ 19:32:29 DEBUG 1713551549, 19:32:29 DEBUG "0.0075" 19:32:29 DEBUG ] 19:32:29 DEBUG } 19:32:29 DEBUG ] 19:32:29 DEBUG } 19:32:29 DEBUG } 19:32:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713551549" 19:32:29 INFO [http_cmd]: http status code OK 19:32:29 DEBUG --- status code --- 19:32:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:29 DEBUG --- http response --- 19:32:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:32:29 DEBUG ----- output ----- 19:32:29 DEBUG { 19:32:29 DEBUG "status": "success", 19:32:29 DEBUG "data": { 19:32:29 DEBUG "resultType": "vector", 19:32:29 DEBUG "result": [ 19:32:29 DEBUG { 19:32:29 DEBUG "metric": { 19:32:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:32:29 DEBUG }, 19:32:29 DEBUG "value": [ 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG [...] 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:32:29 DEBUG }, 19:32:29 DEBUG "value": [ 19:32:29 DEBUG 1713551549, 19:32:29 DEBUG "3398696960" 19:32:29 DEBUG ] 19:32:29 DEBUG } 19:32:29 DEBUG ] 19:32:29 DEBUG } 19:32:29 DEBUG } 19:32:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551549" 19:32:29 INFO [http_cmd]: http status code OK 19:32:29 DEBUG --- status code --- 19:32:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:29 DEBUG --- http response --- 19:32:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:32:29 DEBUG ----- output ----- 19:32:29 DEBUG { 19:32:29 DEBUG "status": "success", 19:32:29 DEBUG "data": { 19:32:29 DEBUG "resultType": "vector", 19:32:29 DEBUG "result": [ 19:32:29 DEBUG { 19:32:29 DEBUG "metric": { 19:32:29 DEBUG "instance": "10.142.0.107:9100" 19:32:29 DEBUG }, 19:32:29 DEBUG "value": [ 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG [...] 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG "instance": "10.142.0.43:9100" 19:32:29 DEBUG }, 19:32:29 DEBUG "value": [ 19:32:29 DEBUG 1713551549, 19:32:29 DEBUG "70822570.5" 19:32:29 DEBUG ] 19:32:29 DEBUG } 19:32:29 DEBUG ] 19:32:29 DEBUG } 19:32:29 DEBUG } 19:32:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551549" 19:32:29 INFO [http_cmd]: http status code OK 19:32:29 DEBUG --- status code --- 19:32:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:29 DEBUG --- http response --- 19:32:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 19:32:29 DEBUG ----- output ----- 19:32:29 DEBUG { 19:32:29 DEBUG "status": "success", 19:32:29 DEBUG "data": { 19:32:29 DEBUG "resultType": "vector", 19:32:29 DEBUG "result": [ 19:32:29 DEBUG { 19:32:29 DEBUG "metric": { 19:32:29 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:32:29 DEBUG }, 19:32:29 DEBUG "value": [ 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG [...] 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG 19:32:29 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:32:29 DEBUG }, 19:32:29 DEBUG "value": [ 19:32:29 DEBUG 1713551549, 19:32:29 DEBUG "3.1740459003773513" 19:32:29 DEBUG ] 19:32:29 DEBUG } 19:32:29 DEBUG ] 19:32:29 DEBUG } 19:32:29 DEBUG } 19:32:29 INFO 19:32:29 INFO [loop_until]: kubectl --namespace=xlou top pods 19:32:29 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:32:29 INFO [loop_until]: OK (rc = 0) 19:32:29 DEBUG --- stdout --- 19:32:29 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 14344m 8991Mi am-767f6b67b5-qcqz2 14779m 8996Mi am-767f6b67b5-vbltz 15586m 8984Mi ds-cts-0 15685m 10865Mi ds-cts-1 13714m 11149Mi ds-cts-2 16626m 10869Mi ds-idrepo-0 6516m 3658Mi ds-idrepo-1 6340m 3531Mi ds-idrepo-2 6355m 3492Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7790m 3150Mi 19:32:29 DEBUG --- stderr --- 19:32:29 DEBUG 19:32:30 INFO 19:32:30 INFO [loop_until]: kubectl --namespace=xlou top node 19:32:30 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:32:30 INFO [loop_until]: OK (rc = 0) 19:32:30 DEBUG --- stdout --- 19:32:30 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 69m 0% 6250Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 6518m 21% 7252Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 14182m 47% 11693Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 3351Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 15358m 51% 13896Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 154m 0% 5206Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 14469m 48% 11866Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12398m 41% 3017Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 16310m 54% 11193Mi 10% gke-xlou-cdm-ds-0f601751-j488 6972m 23% 6735Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 13893m 46% 15941Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 6739m 22% 7412Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 15795m 52% 13375Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 7880m 49% 6937Mi 11% 19:32:30 DEBUG --- stderr --- 19:32:30 DEBUG 19:33:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713551609" 19:33:29 INFO [http_cmd]: http status code OK 19:33:29 DEBUG --- status code --- 19:33:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:29 DEBUG --- http response --- 19:33:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:33:29 DEBUG ----- output ----- 19:33:29 DEBUG { 19:33:29 DEBUG "status": "success", 19:33:29 DEBUG "data": { 19:33:29 DEBUG "resultType": "vector", 19:33:29 DEBUG "result": [ 19:33:29 DEBUG { 19:33:29 DEBUG "metric": { 19:33:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:33:29 DEBUG }, 19:33:29 DEBUG "value": [ 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG [...] 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:33:29 DEBUG }, 19:33:29 DEBUG "value": [ 19:33:29 DEBUG 1713551609, 19:33:29 DEBUG "7.719329075056067" 19:33:29 DEBUG ] 19:33:29 DEBUG } 19:33:29 DEBUG ] 19:33:29 DEBUG } 19:33:29 DEBUG } 19:33:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551609" 19:33:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713551609" 19:33:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551609" 19:33:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551609" 19:33:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551609" 19:33:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551609" 19:33:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551609" 19:33:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551609" 19:33:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551609" 19:33:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551609" 19:33:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713551609" 19:33:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551609" 19:33:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551609" 19:33:29 INFO [http_cmd]: http status code OK 19:33:29 INFO [http_cmd]: http status code OK 19:33:29 DEBUG --- status code --- 19:33:29 DEBUG --- status code --- 19:33:29 ERROR [http_cmd]: ERROR 19:33:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:29 INFO [http_cmd]: http status code OK 19:33:29 INFO [http_cmd]: http status code OK 19:33:29 INFO [http_cmd]: http status code OK 19:33:29 INFO [http_cmd]: http status code OK 19:33:29 INFO [http_cmd]: http status code OK 19:33:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:29 ERROR --- status code --- 19:33:29 DEBUG --- http response --- 19:33:29 DEBUG --- status code --- 19:33:29 DEBUG --- status code --- 19:33:29 DEBUG --- status code --- 19:33:29 DEBUG --- status code --- 19:33:29 DEBUG --- status code --- 19:33:29 DEBUG --- http response --- 19:33:29 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:33:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:33:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:29 INFO [http_cmd]: http status code OK 19:33:29 INFO [http_cmd]: http status code OK 19:33:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:33:29 DEBUG ----- output ----- 19:33:29 DEBUG ----- output ----- 19:33:29 DEBUG --- http response --- 19:33:29 DEBUG --- status code --- 19:33:29 DEBUG --- status code --- 19:33:29 DEBUG --- http response --- 19:33:29 DEBUG --- http response --- 19:33:29 DEBUG --- http response --- 19:33:29 DEBUG --- http response --- 19:33:29 DEBUG { 19:33:29 ERROR --- http response --- 19:33:29 DEBUG { 19:33:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:33:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:33:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:33:29 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:33:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:33:29 DEBUG "status": "success", 19:33:29 DEBUG "status": "success", 19:33:29 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:33:29 DEBUG --- http response --- 19:33:29 DEBUG ----- output ----- 19:33:29 DEBUG --- http response --- 19:33:29 DEBUG ----- output ----- 19:33:29 DEBUG ----- output ----- 19:33:29 DEBUG ----- output ----- 19:33:29 DEBUG "data": { 19:33:29 DEBUG "data": { 19:33:29 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:33:29 DEBUG { 19:33:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:33:29 DEBUG { 19:33:29 DEBUG { 19:33:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:33:29 DEBUG { 19:33:29 DEBUG "resultType": "vector", 19:33:29 DEBUG "resultType": "vector", 19:33:29 DEBUG "status": "success", 19:33:29 DEBUG ----- output ----- 19:33:29 DEBUG "status": "success", 19:33:29 DEBUG "status": "success", 19:33:29 DEBUG ----- output ----- 19:33:29 INFO [http_cmd]: http status code OK 19:33:29 DEBUG "status": "success", 19:33:29 DEBUG "result": [ 19:33:29 DEBUG "result": [ 19:33:29 DEBUG "data": { 19:33:29 DEBUG { 19:33:29 DEBUG "data": { 19:33:29 DEBUG "data": { 19:33:29 DEBUG { 19:33:29 DEBUG --- status code --- 19:33:29 DEBUG "data": { 19:33:29 DEBUG { 19:33:29 DEBUG { 19:33:29 DEBUG "resultType": "vector", 19:33:29 DEBUG "status": "success", 19:33:29 DEBUG "resultType": "vector", 19:33:29 DEBUG "resultType": "vector", 19:33:29 DEBUG "status": "success", 19:33:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:29 DEBUG "resultType": "vector", 19:33:29 DEBUG "metric": { 19:33:29 DEBUG "metric": { 19:33:29 DEBUG "result": [ 19:33:29 DEBUG "data": { 19:33:29 DEBUG "result": [ 19:33:29 DEBUG "result": [ 19:33:29 DEBUG "data": { 19:33:29 DEBUG --- http response --- 19:33:29 DEBUG "result": [ 19:33:29 DEBUG "pod": "ds-idrepo-0" 19:33:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:33:29 DEBUG { 19:33:29 DEBUG "resultType": "vector", 19:33:29 DEBUG { 19:33:29 DEBUG { 19:33:29 DEBUG "resultType": "vector", 19:33:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:33:29 DEBUG { 19:33:29 DEBUG }, 19:33:29 DEBUG }, 19:33:29 DEBUG "metric": { 19:33:29 DEBUG "result": [ 19:33:29 DEBUG "metric": { 19:33:29 DEBUG "metric": { 19:33:29 DEBUG "result": [ 19:33:29 DEBUG ----- output ----- 19:33:29 DEBUG "metric": { 19:33:29 DEBUG "value": [ 19:33:29 DEBUG "value": [ 19:33:29 DEBUG "pod": "ds-idrepo-1" 19:33:29 DEBUG { 19:33:29 INFO [http_cmd]: http status code OK 19:33:29 INFO [http_cmd]: http status code OK 19:33:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:33:29 DEBUG "pod": "ds-idrepo-0" 19:33:29 DEBUG { 19:33:29 DEBUG { 19:33:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG }, 19:33:29 DEBUG "metric": { 19:33:29 DEBUG --- status code --- 19:33:29 DEBUG --- status code --- 19:33:29 DEBUG }, 19:33:29 DEBUG }, 19:33:29 DEBUG "metric": { 19:33:29 DEBUG "status": "success", 19:33:29 DEBUG }, 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG "value": [ 19:33:29 DEBUG "instance": "10.142.0.107:9100" 19:33:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:29 DEBUG "value": [ 19:33:29 DEBUG "value": [ 19:33:29 DEBUG "pod": "ds-cts-0" 19:33:29 DEBUG "data": { 19:33:29 DEBUG "value": [ 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG }, 19:33:29 DEBUG --- http response --- 19:33:29 DEBUG --- http response --- 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG }, 19:33:29 DEBUG "resultType": "vector", 19:33:29 DEBUG 19:33:29 DEBUG [...] 19:33:29 DEBUG [...] 19:33:29 DEBUG 19:33:29 DEBUG "value": [ 19:33:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:33:29 DEBUG 19:33:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:33:29 DEBUG 19:33:29 DEBUG "value": [ 19:33:29 DEBUG "result": [ 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG ----- output ----- 19:33:29 DEBUG 19:33:29 DEBUG ----- output ----- 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG { 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG [...] 19:33:29 DEBUG 19:33:29 DEBUG { 19:33:29 DEBUG [...] 19:33:29 DEBUG { 19:33:29 DEBUG [...] 19:33:29 DEBUG 19:33:29 DEBUG "metric": { 19:33:29 DEBUG [...] 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG "status": "success", 19:33:29 DEBUG 19:33:29 DEBUG "status": "success", 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG "instance": "10.142.0.107:9100" 19:33:29 DEBUG 19:33:29 DEBUG "pod": "ds-idrepo-2" 19:33:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:33:29 DEBUG 19:33:29 DEBUG [...] 19:33:29 DEBUG "data": { 19:33:29 DEBUG 19:33:29 DEBUG "data": { 19:33:29 DEBUG 19:33:29 DEBUG [...] 19:33:29 DEBUG }, 19:33:29 DEBUG 19:33:29 DEBUG }, 19:33:29 DEBUG }, 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG "resultType": "vector", 19:33:29 DEBUG 19:33:29 DEBUG "resultType": "vector", 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG "value": [ 19:33:29 DEBUG 19:33:29 DEBUG "value": [ 19:33:29 DEBUG "value": [ 19:33:29 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 19:33:29 DEBUG 19:33:29 DEBUG "result": [ 19:33:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:33:29 DEBUG "result": [ 19:33:29 DEBUG "pod": "ds-idrepo-2" 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:33:29 DEBUG 1713551609, 19:33:29 DEBUG 1713551609, 19:33:29 DEBUG }, 19:33:29 DEBUG 19:33:29 DEBUG { 19:33:29 DEBUG }, 19:33:29 DEBUG { 19:33:29 DEBUG }, 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG }, 19:33:29 DEBUG "0" 19:33:29 DEBUG "2339.866666666667" 19:33:29 DEBUG "value": [ 19:33:29 DEBUG "instance": "10.142.0.43:9100" 19:33:29 DEBUG "metric": { 19:33:29 DEBUG "value": [ 19:33:29 DEBUG "metric": { 19:33:29 DEBUG "value": [ 19:33:29 DEBUG "pod": "ds-cts-2" 19:33:29 DEBUG 19:33:29 DEBUG "value": [ 19:33:29 DEBUG ] 19:33:29 DEBUG ] 19:33:29 DEBUG 1713551609, 19:33:29 DEBUG }, 19:33:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:33:29 DEBUG 1713551609, 19:33:29 DEBUG "pod": "ds-idrepo-0" 19:33:29 DEBUG 1713551609, 19:33:29 DEBUG }, 19:33:29 DEBUG [...] 19:33:29 DEBUG 1713551609, 19:33:29 DEBUG } 19:33:29 DEBUG } 19:33:29 DEBUG "0" 19:33:29 DEBUG "value": [ 19:33:29 DEBUG }, 19:33:29 DEBUG "0" 19:33:29 DEBUG }, 19:33:29 DEBUG "0" 19:33:29 DEBUG "value": [ 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG ] 19:33:29 DEBUG ] 19:33:29 DEBUG ] 19:33:29 DEBUG 1713551609, 19:33:29 DEBUG "value": [ 19:33:29 DEBUG ] 19:33:29 DEBUG "value": [ 19:33:29 DEBUG ] 19:33:29 DEBUG 1713551609, 19:33:29 DEBUG "2333" 19:33:29 DEBUG 19:33:29 DEBUG } 19:33:29 DEBUG } 19:33:29 DEBUG } 19:33:29 DEBUG "0.16536666666671163" 19:33:29 DEBUG 19:33:29 DEBUG } 19:33:29 DEBUG 19:33:29 DEBUG } 19:33:29 DEBUG "0" 19:33:29 DEBUG ] 19:33:29 DEBUG "instance": "10.142.0.43:9100" 19:33:29 DEBUG } 19:33:29 DEBUG } 19:33:29 DEBUG ] 19:33:29 DEBUG ] 19:33:29 DEBUG 19:33:29 DEBUG ] 19:33:29 DEBUG 19:33:29 DEBUG ] 19:33:29 DEBUG ] 19:33:29 DEBUG } 19:33:29 DEBUG ] 19:33:29 DEBUG }, 19:33:29 DEBUG } 19:33:29 DEBUG 19:33:29 DEBUG } 19:33:29 DEBUG 19:33:29 DEBUG } 19:33:29 DEBUG } 19:33:29 DEBUG } 19:33:29 DEBUG } 19:33:29 DEBUG "value": [ 19:33:29 DEBUG ] 19:33:29 DEBUG [...] 19:33:29 DEBUG } 19:33:29 DEBUG [...] 19:33:29 DEBUG } 19:33:29 DEBUG ] 19:33:29 DEBUG } 19:33:29 DEBUG } 19:33:29 DEBUG 1713551609, 19:33:29 DEBUG } 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG "1907563297792" 19:33:29 DEBUG } 19:33:29 DEBUG 19:33:29 DEBUG } 19:33:29 DEBUG 19:33:29 DEBUG ] 19:33:29 DEBUG 19:33:29 DEBUG } 19:33:29 DEBUG "pod": "ds-cts-2" 19:33:29 DEBUG } 19:33:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:33:29 DEBUG }, 19:33:29 DEBUG ] 19:33:29 DEBUG }, 19:33:29 DEBUG "value": [ 19:33:29 DEBUG } 19:33:29 DEBUG "value": [ 19:33:29 DEBUG 1713551609, 19:33:29 DEBUG } 19:33:29 DEBUG 1713551609, 19:33:29 DEBUG "0" 19:33:29 DEBUG "420403" 19:33:29 DEBUG ] 19:33:29 DEBUG ] 19:33:29 DEBUG } 19:33:29 DEBUG } 19:33:29 DEBUG ] 19:33:29 DEBUG ] 19:33:29 DEBUG } 19:33:29 DEBUG } 19:33:29 DEBUG } 19:33:29 DEBUG } 19:33:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551609" 19:33:29 INFO [http_cmd]: http status code OK 19:33:29 DEBUG --- status code --- 19:33:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:29 DEBUG --- http response --- 19:33:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 19:33:29 DEBUG ----- output ----- 19:33:29 DEBUG { 19:33:29 DEBUG "status": "success", 19:33:29 DEBUG "data": { 19:33:29 DEBUG "resultType": "vector", 19:33:29 DEBUG "result": [ 19:33:29 DEBUG { 19:33:29 DEBUG "metric": { 19:33:29 DEBUG "pod": "ds-idrepo-1" 19:33:29 DEBUG }, 19:33:29 DEBUG "value": [ 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG [...] 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 19:33:29 DEBUG }, 19:33:29 DEBUG "value": [ 19:33:29 DEBUG 1713551609, 19:33:29 DEBUG "0" 19:33:29 DEBUG ] 19:33:29 DEBUG } 19:33:29 DEBUG ] 19:33:29 DEBUG } 19:33:29 DEBUG } 19:33:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551609" 19:33:29 INFO [http_cmd]: http status code OK 19:33:29 DEBUG --- status code --- 19:33:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:29 DEBUG --- http response --- 19:33:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:33:29 DEBUG ----- output ----- 19:33:29 DEBUG { 19:33:29 DEBUG "status": "success", 19:33:29 DEBUG "data": { 19:33:29 DEBUG "resultType": "vector", 19:33:29 DEBUG "result": [ 19:33:29 DEBUG { 19:33:29 DEBUG "metric": { 19:33:29 DEBUG "pod": "ds-idrepo-0" 19:33:29 DEBUG }, 19:33:29 DEBUG "value": [ 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG [...] 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG "pod": "ds-idrepo-2" 19:33:29 DEBUG }, 19:33:29 DEBUG "value": [ 19:33:29 DEBUG 1713551609, 19:33:29 DEBUG "0" 19:33:29 DEBUG ] 19:33:29 DEBUG } 19:33:29 DEBUG ] 19:33:29 DEBUG } 19:33:29 DEBUG } 19:33:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551609" 19:33:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551609" 19:33:29 INFO [http_cmd]: http status code OK 19:33:29 DEBUG --- status code --- 19:33:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:29 DEBUG --- http response --- 19:33:29 INFO [http_cmd]: http status code OK 19:33:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:33:29 DEBUG --- status code --- 19:33:29 DEBUG ----- output ----- 19:33:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:29 DEBUG { 19:33:29 DEBUG --- http response --- 19:33:29 DEBUG "status": "success", 19:33:29 DEBUG "data": { 19:33:29 DEBUG "resultType": "vector", 19:33:29 DEBUG "result": [ 19:33:29 DEBUG { 19:33:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:33:29 DEBUG "metric": { 19:33:29 DEBUG ----- output ----- 19:33:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:33:29 DEBUG { 19:33:29 DEBUG }, 19:33:29 DEBUG "status": "success", 19:33:29 DEBUG "value": [ 19:33:29 DEBUG "data": { 19:33:29 DEBUG 19:33:29 DEBUG "resultType": "vector", 19:33:29 DEBUG 19:33:29 DEBUG "result": [ 19:33:29 DEBUG 19:33:29 DEBUG { 19:33:29 DEBUG [...] 19:33:29 DEBUG "metric": { 19:33:29 DEBUG 19:33:29 DEBUG "instance": "10.142.0.107:9100" 19:33:29 DEBUG 19:33:29 DEBUG }, 19:33:29 DEBUG 19:33:29 DEBUG "value": [ 19:33:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:33:29 DEBUG 19:33:29 DEBUG }, 19:33:29 DEBUG 19:33:29 DEBUG "value": [ 19:33:29 DEBUG 19:33:29 DEBUG 1713551609, 19:33:29 DEBUG [...] 19:33:29 DEBUG "2340.1666666666665" 19:33:29 DEBUG 19:33:29 DEBUG ] 19:33:29 DEBUG 19:33:29 DEBUG } 19:33:29 DEBUG 19:33:29 DEBUG ] 19:33:29 DEBUG "instance": "10.142.0.43:9100" 19:33:29 DEBUG } 19:33:29 DEBUG }, 19:33:29 DEBUG } 19:33:29 DEBUG "value": [ 19:33:29 DEBUG 1713551609, 19:33:29 DEBUG "70656896.43333334" 19:33:29 DEBUG ] 19:33:29 DEBUG } 19:33:29 DEBUG ] 19:33:29 DEBUG } 19:33:29 DEBUG } 19:33:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551609" 19:33:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551609" 19:33:29 INFO [http_cmd]: http status code OK 19:33:29 INFO [http_cmd]: http status code OK 19:33:29 DEBUG --- status code --- 19:33:29 DEBUG --- status code --- 19:33:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:29 DEBUG --- http response --- 19:33:29 DEBUG --- http response --- 19:33:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:33:29 DEBUG ----- output ----- 19:33:29 DEBUG { 19:33:29 DEBUG "status": "success", 19:33:29 DEBUG "data": { 19:33:29 DEBUG "resultType": "vector", 19:33:29 DEBUG "result": [ 19:33:29 DEBUG { 19:33:29 DEBUG "metric": { 19:33:29 DEBUG "pod": "ds-cts-0" 19:33:29 DEBUG }, 19:33:29 DEBUG "value": [ 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG [...] 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG "pod": "ds-cts-2" 19:33:29 DEBUG }, 19:33:29 DEBUG "value": [ 19:33:29 DEBUG 1713551609, 19:33:29 DEBUG "29.933333333333334" 19:33:29 DEBUG ] 19:33:29 DEBUG } 19:33:29 DEBUG ] 19:33:29 DEBUG } 19:33:29 DEBUG } 19:33:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:33:29 DEBUG ----- output ----- 19:33:29 DEBUG { 19:33:29 DEBUG "status": "success", 19:33:29 DEBUG "data": { 19:33:29 DEBUG "resultType": "vector", 19:33:29 DEBUG "result": [ 19:33:29 DEBUG { 19:33:29 DEBUG "metric": { 19:33:29 DEBUG "pod": "ds-cts-0" 19:33:29 DEBUG }, 19:33:29 DEBUG "value": [ 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG [...] 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG "pod": "ds-cts-2" 19:33:29 DEBUG }, 19:33:29 DEBUG "value": [ 19:33:29 DEBUG 1713551609, 19:33:29 DEBUG "0.00013596" 19:33:29 DEBUG ] 19:33:29 DEBUG } 19:33:29 DEBUG ] 19:33:29 DEBUG } 19:33:29 DEBUG } 19:33:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551609" 19:33:29 INFO [http_cmd]: http status code OK 19:33:29 DEBUG --- status code --- 19:33:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:29 DEBUG --- http response --- 19:33:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:33:29 DEBUG ----- output ----- 19:33:29 DEBUG { 19:33:29 DEBUG "status": "success", 19:33:29 DEBUG "data": { 19:33:29 DEBUG "resultType": "vector", 19:33:29 DEBUG "result": [ 19:33:29 DEBUG { 19:33:29 DEBUG "metric": { 19:33:29 DEBUG "instance": "10.142.0.107:9100" 19:33:29 DEBUG }, 19:33:29 DEBUG "value": [ 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG [...] 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG "instance": "10.142.0.43:9100" 19:33:29 DEBUG }, 19:33:29 DEBUG "value": [ 19:33:29 DEBUG 1713551609, 19:33:29 DEBUG "0.07300000000000373" 19:33:29 DEBUG ] 19:33:29 DEBUG } 19:33:29 DEBUG ] 19:33:29 DEBUG } 19:33:29 DEBUG } 19:33:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551609" 19:33:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551609" 19:33:29 INFO [http_cmd]: http status code OK 19:33:29 DEBUG --- status code --- 19:33:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:29 DEBUG --- http response --- 19:33:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:33:29 DEBUG ----- output ----- 19:33:29 DEBUG { 19:33:29 DEBUG "status": "success", 19:33:29 DEBUG "data": { 19:33:29 DEBUG "resultType": "vector", 19:33:29 DEBUG "result": [ 19:33:29 DEBUG { 19:33:29 DEBUG "metric": { 19:33:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:33:29 DEBUG }, 19:33:29 DEBUG "value": [ 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG [...] 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:33:29 DEBUG }, 19:33:29 DEBUG "value": [ 19:33:29 DEBUG 1713551609, 19:33:29 DEBUG "2340.1666666666665" 19:33:29 DEBUG ] 19:33:29 DEBUG } 19:33:29 DEBUG ] 19:33:29 DEBUG } 19:33:29 DEBUG } 19:33:29 INFO [http_cmd]: http status code OK 19:33:29 DEBUG --- status code --- 19:33:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:29 DEBUG --- http response --- 19:33:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:33:29 DEBUG ----- output ----- 19:33:29 DEBUG { 19:33:29 DEBUG "status": "success", 19:33:29 DEBUG "data": { 19:33:29 DEBUG "resultType": "vector", 19:33:29 DEBUG "result": [ 19:33:29 DEBUG { 19:33:29 DEBUG "metric": { 19:33:29 DEBUG "pod": "ds-idrepo-1" 19:33:29 DEBUG }, 19:33:29 DEBUG "value": [ 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG [...] 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 19:33:29 DEBUG }, 19:33:29 DEBUG "value": [ 19:33:29 DEBUG 1713551609, 19:33:29 DEBUG "0" 19:33:29 DEBUG ] 19:33:29 DEBUG } 19:33:29 DEBUG ] 19:33:29 DEBUG } 19:33:29 DEBUG } 19:33:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713551609" 19:33:29 INFO [http_cmd]: http status code OK 19:33:29 DEBUG --- status code --- 19:33:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:29 DEBUG --- http response --- 19:33:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:33:29 DEBUG ----- output ----- 19:33:29 DEBUG { 19:33:29 DEBUG "status": "success", 19:33:29 DEBUG "data": { 19:33:29 DEBUG "resultType": "vector", 19:33:29 DEBUG "result": [ 19:33:29 DEBUG { 19:33:29 DEBUG "metric": { 19:33:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:33:29 DEBUG }, 19:33:29 DEBUG "value": [ 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG [...] 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:33:29 DEBUG }, 19:33:29 DEBUG "value": [ 19:33:29 DEBUG 1713551609, 19:33:29 DEBUG "0.0085" 19:33:29 DEBUG ] 19:33:29 DEBUG } 19:33:29 DEBUG ] 19:33:29 DEBUG } 19:33:29 DEBUG } 19:33:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713551609" 19:33:29 INFO [http_cmd]: http status code OK 19:33:29 DEBUG --- status code --- 19:33:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:29 DEBUG --- http response --- 19:33:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:33:29 DEBUG ----- output ----- 19:33:29 DEBUG { 19:33:29 DEBUG "status": "success", 19:33:29 DEBUG "data": { 19:33:29 DEBUG "resultType": "vector", 19:33:29 DEBUG "result": [ 19:33:29 DEBUG { 19:33:29 DEBUG "metric": { 19:33:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:33:29 DEBUG }, 19:33:29 DEBUG "value": [ 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG [...] 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:33:29 DEBUG }, 19:33:29 DEBUG "value": [ 19:33:29 DEBUG 1713551609, 19:33:29 DEBUG "3401539584" 19:33:29 DEBUG ] 19:33:29 DEBUG } 19:33:29 DEBUG ] 19:33:29 DEBUG } 19:33:29 DEBUG } 19:33:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551609" 19:33:29 INFO [http_cmd]: http status code OK 19:33:29 DEBUG --- status code --- 19:33:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:29 DEBUG --- http response --- 19:33:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:33:29 DEBUG ----- output ----- 19:33:29 DEBUG { 19:33:29 DEBUG "status": "success", 19:33:29 DEBUG "data": { 19:33:29 DEBUG "resultType": "vector", 19:33:29 DEBUG "result": [ 19:33:29 DEBUG { 19:33:29 DEBUG "metric": { 19:33:29 DEBUG "instance": "10.142.0.107:9100" 19:33:29 DEBUG }, 19:33:29 DEBUG "value": [ 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG [...] 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG "instance": "10.142.0.43:9100" 19:33:29 DEBUG }, 19:33:29 DEBUG "value": [ 19:33:29 DEBUG 1713551609, 19:33:29 DEBUG "70653019.93333334" 19:33:29 DEBUG ] 19:33:29 DEBUG } 19:33:29 DEBUG ] 19:33:29 DEBUG } 19:33:29 DEBUG } 19:33:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551609" 19:33:29 INFO [http_cmd]: http status code OK 19:33:29 DEBUG --- status code --- 19:33:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:29 DEBUG --- http response --- 19:33:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 19:33:29 DEBUG ----- output ----- 19:33:29 DEBUG { 19:33:29 DEBUG "status": "success", 19:33:29 DEBUG "data": { 19:33:29 DEBUG "resultType": "vector", 19:33:29 DEBUG "result": [ 19:33:29 DEBUG { 19:33:29 DEBUG "metric": { 19:33:29 DEBUG "pod": "ds-idrepo-1" 19:33:29 DEBUG }, 19:33:29 DEBUG "value": [ 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG [...] 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG 19:33:29 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 19:33:29 DEBUG }, 19:33:29 DEBUG "value": [ 19:33:29 DEBUG 1713551609, 19:33:29 DEBUG "0.000016198703484839104" 19:33:29 DEBUG ] 19:33:29 DEBUG } 19:33:29 DEBUG ] 19:33:29 DEBUG } 19:33:29 DEBUG } 19:33:29 INFO 19:33:29 INFO [loop_until]: kubectl --namespace=xlou top pods 19:33:29 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:33:30 INFO [loop_until]: OK (rc = 0) 19:33:30 DEBUG --- stdout --- 19:33:30 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 14423m 8993Mi am-767f6b67b5-qcqz2 14687m 8996Mi am-767f6b67b5-vbltz 15423m 8978Mi ds-cts-0 15363m 10866Mi ds-cts-1 14077m 11129Mi ds-cts-2 15758m 10873Mi ds-idrepo-0 6429m 3658Mi ds-idrepo-1 6406m 3531Mi ds-idrepo-2 6434m 3493Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7607m 3151Mi 19:33:30 DEBUG --- stderr --- 19:33:30 DEBUG 19:33:30 INFO 19:33:30 INFO [loop_until]: kubectl --namespace=xlou top node 19:33:30 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:33:30 INFO [loop_until]: OK (rc = 0) 19:33:30 DEBUG --- stdout --- 19:33:30 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 69m 0% 6254Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 6552m 21% 7246Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 14294m 47% 11703Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 87m 0% 3350Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 15422m 51% 13890Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 137m 0% 5212Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 14506m 48% 11864Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12350m 41% 3000Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 16420m 54% 11189Mi 10% gke-xlou-cdm-ds-0f601751-j488 6988m 23% 6732Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 14332m 48% 15937Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 6992m 23% 7429Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 15954m 53% 13362Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 7898m 49% 6935Mi 11% 19:33:30 DEBUG --- stderr --- 19:33:30 DEBUG 19:34:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713551669" 19:34:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551669" 19:34:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713551669" 19:34:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551669" 19:34:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551669" 19:34:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551669" 19:34:29 INFO [http_cmd]: http status code OK 19:34:29 DEBUG --- status code --- 19:34:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:29 DEBUG --- http response --- 19:34:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:34:29 DEBUG ----- output ----- 19:34:29 DEBUG { 19:34:29 DEBUG "status": "success", 19:34:29 DEBUG "data": { 19:34:29 DEBUG "resultType": "vector", 19:34:29 DEBUG "result": [ 19:34:29 DEBUG { 19:34:29 DEBUG "metric": { 19:34:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:34:29 DEBUG }, 19:34:29 DEBUG "value": [ 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG [...] 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:34:29 DEBUG }, 19:34:29 DEBUG "value": [ 19:34:29 DEBUG 1713551669, 19:34:29 DEBUG "7.721872216001139" 19:34:29 DEBUG ] 19:34:29 DEBUG } 19:34:29 DEBUG ] 19:34:29 DEBUG } 19:34:29 DEBUG } 19:34:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713551669" 19:34:29 INFO [http_cmd]: http status code OK 19:34:29 DEBUG --- status code --- 19:34:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:29 DEBUG --- http response --- 19:34:29 INFO [http_cmd]: http status code OK 19:34:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:34:29 DEBUG --- status code --- 19:34:29 DEBUG ----- output ----- 19:34:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:29 DEBUG { 19:34:29 DEBUG --- http response --- 19:34:29 DEBUG "status": "success", 19:34:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 19:34:29 DEBUG "data": { 19:34:29 DEBUG ----- output ----- 19:34:29 DEBUG "resultType": "vector", 19:34:29 DEBUG { 19:34:29 DEBUG "result": [ 19:34:29 DEBUG "status": "success", 19:34:29 DEBUG { 19:34:29 INFO [http_cmd]: http status code OK 19:34:29 DEBUG "data": { 19:34:29 INFO [http_cmd]: http status code OK 19:34:29 INFO [http_cmd]: http status code OK 19:34:29 DEBUG "metric": { 19:34:29 DEBUG --- status code --- 19:34:29 DEBUG "resultType": "vector", 19:34:29 DEBUG --- status code --- 19:34:29 DEBUG --- status code --- 19:34:29 DEBUG "pod": "ds-idrepo-0" 19:34:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:29 DEBUG "result": [ 19:34:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:29 DEBUG }, 19:34:29 DEBUG --- http response --- 19:34:29 DEBUG { 19:34:29 DEBUG --- http response --- 19:34:29 DEBUG --- http response --- 19:34:29 DEBUG "value": [ 19:34:29 DEBUG "metric": { 19:34:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:34:29 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:34:29 DEBUG 19:34:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:34:29 DEBUG ----- output ----- 19:34:29 DEBUG { 19:34:29 DEBUG "status": "success", 19:34:29 DEBUG "pod": "ds-idrepo-1" 19:34:29 DEBUG ----- output ----- 19:34:29 DEBUG 19:34:29 DEBUG "data": { 19:34:29 DEBUG }, 19:34:29 DEBUG { 19:34:29 DEBUG 19:34:29 DEBUG "resultType": "vector", 19:34:29 DEBUG "value": [ 19:34:29 DEBUG "status": "success", 19:34:29 DEBUG [...] 19:34:29 DEBUG "result": [ 19:34:29 DEBUG 19:34:29 DEBUG "data": { 19:34:29 DEBUG 19:34:29 DEBUG { 19:34:29 DEBUG 19:34:29 DEBUG "resultType": "vector", 19:34:29 DEBUG 19:34:29 DEBUG "metric": { 19:34:29 DEBUG 19:34:29 DEBUG "result": [ 19:34:29 DEBUG 19:34:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:34:29 DEBUG [...] 19:34:29 DEBUG { 19:34:29 DEBUG "pod": "ds-cts-2" 19:34:29 DEBUG }, 19:34:29 DEBUG 19:34:29 DEBUG "metric": { 19:34:29 DEBUG }, 19:34:29 DEBUG "value": [ 19:34:29 DEBUG 19:34:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:34:29 DEBUG "value": [ 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG }, 19:34:29 DEBUG 1713551669, 19:34:29 DEBUG 19:34:29 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:34:29 DEBUG "value": [ 19:34:29 DEBUG "0" 19:34:29 DEBUG 19:34:29 DEBUG }, 19:34:29 DEBUG 19:34:29 DEBUG ] 19:34:29 DEBUG [...] 19:34:29 DEBUG "value": [ 19:34:29 DEBUG 19:34:29 DEBUG } 19:34:29 DEBUG 19:34:29 DEBUG 1713551669, 19:34:29 DEBUG 19:34:29 DEBUG ] 19:34:29 DEBUG 19:34:29 DEBUG "0" 19:34:29 DEBUG [...] 19:34:29 DEBUG } 19:34:29 DEBUG 19:34:29 DEBUG ] 19:34:29 DEBUG 19:34:29 DEBUG } 19:34:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:34:29 DEBUG } 19:34:29 DEBUG 19:34:29 DEBUG }, 19:34:29 DEBUG ] 19:34:29 DEBUG 19:34:29 DEBUG "value": [ 19:34:29 DEBUG } 19:34:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:34:29 DEBUG 1713551669, 19:34:29 DEBUG } 19:34:29 DEBUG }, 19:34:29 DEBUG "2334.633333333333" 19:34:29 DEBUG "value": [ 19:34:29 DEBUG ] 19:34:29 DEBUG 1713551669, 19:34:29 DEBUG } 19:34:29 DEBUG "0" 19:34:29 DEBUG ] 19:34:29 DEBUG ] 19:34:29 DEBUG } 19:34:29 DEBUG } 19:34:29 DEBUG } 19:34:29 DEBUG ] 19:34:29 DEBUG } 19:34:29 DEBUG } 19:34:29 ERROR [http_cmd]: ERROR 19:34:29 ERROR --- status code --- 19:34:29 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:34:29 ERROR --- http response --- 19:34:29 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:34:29 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:34:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551669" 19:34:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551669" 19:34:29 INFO [http_cmd]: http status code OK 19:34:29 DEBUG --- status code --- 19:34:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:29 DEBUG --- http response --- 19:34:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:34:29 DEBUG ----- output ----- 19:34:29 DEBUG { 19:34:29 DEBUG "status": "success", 19:34:29 DEBUG "data": { 19:34:29 DEBUG "resultType": "vector", 19:34:29 DEBUG "result": [ 19:34:29 DEBUG { 19:34:29 DEBUG "metric": { 19:34:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:34:29 DEBUG }, 19:34:29 DEBUG "value": [ 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG [...] 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:34:29 DEBUG }, 19:34:29 DEBUG "value": [ 19:34:29 DEBUG 1713551669, 19:34:29 DEBUG "420331.5" 19:34:29 DEBUG ] 19:34:29 DEBUG } 19:34:29 DEBUG ] 19:34:29 DEBUG } 19:34:29 DEBUG } 19:34:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551669" 19:34:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551669" 19:34:29 INFO [http_cmd]: http status code OK 19:34:29 DEBUG --- status code --- 19:34:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:29 DEBUG --- http response --- 19:34:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551669" 19:34:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:34:29 DEBUG ----- output ----- 19:34:29 DEBUG { 19:34:29 DEBUG "status": "success", 19:34:29 DEBUG "data": { 19:34:29 DEBUG "resultType": "vector", 19:34:29 DEBUG "result": [ 19:34:29 DEBUG { 19:34:29 DEBUG "metric": { 19:34:29 DEBUG "instance": "10.142.0.107:9100" 19:34:29 DEBUG }, 19:34:29 DEBUG "value": [ 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG [...] 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG "instance": "10.142.0.43:9100" 19:34:29 DEBUG }, 19:34:29 DEBUG "value": [ 19:34:29 DEBUG 1713551669, 19:34:29 DEBUG "1911315012608" 19:34:29 DEBUG ] 19:34:29 DEBUG } 19:34:29 DEBUG ] 19:34:29 DEBUG } 19:34:29 DEBUG } 19:34:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551669" 19:34:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551669" 19:34:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551669" 19:34:29 INFO [http_cmd]: http status code OK 19:34:29 INFO [http_cmd]: http status code OK 19:34:29 DEBUG --- status code --- 19:34:29 DEBUG --- status code --- 19:34:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:29 DEBUG --- http response --- 19:34:29 DEBUG --- http response --- 19:34:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:34:29 DEBUG ----- output ----- 19:34:29 DEBUG { 19:34:29 DEBUG "status": "success", 19:34:29 DEBUG "data": { 19:34:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:34:29 DEBUG "resultType": "vector", 19:34:29 DEBUG ----- output ----- 19:34:29 DEBUG "result": [ 19:34:29 INFO [http_cmd]: http status code OK 19:34:29 DEBUG { 19:34:29 DEBUG { 19:34:29 DEBUG --- status code --- 19:34:29 DEBUG "status": "success", 19:34:29 DEBUG "metric": { 19:34:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:29 DEBUG "data": { 19:34:29 DEBUG "pod": "ds-idrepo-0" 19:34:29 DEBUG --- http response --- 19:34:29 DEBUG "resultType": "vector", 19:34:29 DEBUG }, 19:34:29 DEBUG "result": [ 19:34:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:34:29 DEBUG "value": [ 19:34:29 DEBUG { 19:34:29 DEBUG ----- output ----- 19:34:29 DEBUG 19:34:29 DEBUG "metric": { 19:34:29 DEBUG { 19:34:29 DEBUG 19:34:29 DEBUG "pod": "ds-idrepo-0" 19:34:29 DEBUG "status": "success", 19:34:29 DEBUG 19:34:29 DEBUG }, 19:34:29 DEBUG "data": { 19:34:29 DEBUG [...] 19:34:29 DEBUG "value": [ 19:34:29 DEBUG "resultType": "vector", 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG "result": [ 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG { 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG "metric": { 19:34:29 DEBUG "pod": "ds-idrepo-2" 19:34:29 DEBUG [...] 19:34:29 DEBUG "pod": "ds-idrepo-0" 19:34:29 DEBUG }, 19:34:29 INFO [http_cmd]: http status code OK 19:34:29 INFO [http_cmd]: http status code OK 19:34:29 DEBUG 19:34:29 INFO [http_cmd]: http status code OK 19:34:29 DEBUG }, 19:34:29 DEBUG "value": [ 19:34:29 DEBUG --- status code --- 19:34:29 DEBUG --- status code --- 19:34:29 DEBUG 19:34:29 DEBUG --- status code --- 19:34:29 DEBUG "value": [ 19:34:29 DEBUG 1713551669, 19:34:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:29 DEBUG 19:34:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:29 DEBUG 19:34:29 DEBUG "0" 19:34:29 DEBUG --- http response --- 19:34:29 DEBUG --- http response --- 19:34:29 DEBUG "pod": "ds-idrepo-2" 19:34:29 DEBUG --- http response --- 19:34:29 DEBUG 19:34:29 DEBUG ] 19:34:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:34:29 DEBUG }, 19:34:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:34:29 DEBUG 19:34:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:34:29 DEBUG } 19:34:29 DEBUG ----- output ----- 19:34:29 DEBUG "value": [ 19:34:29 DEBUG ----- output ----- 19:34:29 DEBUG [...] 19:34:29 DEBUG ----- output ----- 19:34:29 DEBUG ] 19:34:29 DEBUG { 19:34:29 DEBUG 1713551669, 19:34:29 DEBUG { 19:34:29 DEBUG 19:34:29 DEBUG { 19:34:29 DEBUG } 19:34:29 DEBUG "status": "success", 19:34:29 DEBUG "0" 19:34:29 DEBUG "status": "success", 19:34:29 DEBUG 19:34:29 DEBUG "status": "success", 19:34:29 DEBUG } 19:34:29 DEBUG "data": { 19:34:29 DEBUG ] 19:34:29 DEBUG "data": { 19:34:29 DEBUG 19:34:29 DEBUG "data": { 19:34:29 DEBUG "resultType": "vector", 19:34:29 DEBUG } 19:34:29 DEBUG "resultType": "vector", 19:34:29 DEBUG "pod": "ds-idrepo-2" 19:34:29 DEBUG "resultType": "vector", 19:34:29 DEBUG "result": [ 19:34:29 DEBUG ] 19:34:29 DEBUG "result": [ 19:34:29 DEBUG }, 19:34:29 DEBUG "result": [ 19:34:29 DEBUG { 19:34:29 DEBUG } 19:34:29 DEBUG { 19:34:29 DEBUG "value": [ 19:34:29 DEBUG { 19:34:29 DEBUG "metric": { 19:34:29 DEBUG } 19:34:29 DEBUG "metric": { 19:34:29 DEBUG 1713551669, 19:34:29 DEBUG "metric": { 19:34:29 DEBUG "instance": "10.142.0.107:9100" 19:34:29 DEBUG "pod": "ds-cts-0" 19:34:29 DEBUG "0" 19:34:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:34:29 DEBUG }, 19:34:29 DEBUG }, 19:34:29 DEBUG ] 19:34:29 DEBUG }, 19:34:29 DEBUG "value": [ 19:34:29 DEBUG "value": [ 19:34:29 DEBUG } 19:34:29 DEBUG "value": [ 19:34:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551669" 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG ] 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG } 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG } 19:34:29 DEBUG 19:34:29 DEBUG [...] 19:34:29 DEBUG [...] 19:34:29 DEBUG [...] 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG "instance": "10.142.0.43:9100" 19:34:29 DEBUG "pod": "ds-cts-2" 19:34:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:34:29 DEBUG }, 19:34:29 DEBUG }, 19:34:29 DEBUG }, 19:34:29 DEBUG "value": [ 19:34:29 DEBUG "value": [ 19:34:29 DEBUG "value": [ 19:34:29 DEBUG 1713551669, 19:34:29 DEBUG 1713551669, 19:34:29 DEBUG 1713551669, 19:34:29 DEBUG "0.27456666666673946" 19:34:29 DEBUG "0" 19:34:29 DEBUG "2333.866666666667" 19:34:29 DEBUG ] 19:34:29 DEBUG ] 19:34:29 DEBUG ] 19:34:29 DEBUG } 19:34:29 DEBUG } 19:34:29 DEBUG } 19:34:29 DEBUG ] 19:34:29 DEBUG ] 19:34:29 DEBUG ] 19:34:29 DEBUG } 19:34:29 DEBUG } 19:34:29 DEBUG } 19:34:29 DEBUG } 19:34:29 DEBUG } 19:34:29 DEBUG } 19:34:29 INFO [http_cmd]: http status code OK 19:34:29 DEBUG --- status code --- 19:34:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:29 DEBUG --- http response --- 19:34:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:34:29 DEBUG ----- output ----- 19:34:29 DEBUG { 19:34:29 DEBUG "status": "success", 19:34:29 DEBUG "data": { 19:34:29 DEBUG "resultType": "vector", 19:34:29 DEBUG "result": [ 19:34:29 DEBUG { 19:34:29 DEBUG "metric": { 19:34:29 DEBUG "pod": "ds-idrepo-1" 19:34:29 DEBUG }, 19:34:29 DEBUG "value": [ 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG [...] 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:34:29 DEBUG }, 19:34:29 DEBUG "value": [ 19:34:29 DEBUG 1713551669, 19:34:29 DEBUG "0" 19:34:29 DEBUG ] 19:34:29 DEBUG } 19:34:29 DEBUG ] 19:34:29 DEBUG } 19:34:29 DEBUG } 19:34:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551669" 19:34:29 INFO [http_cmd]: http status code OK 19:34:29 DEBUG --- status code --- 19:34:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:29 DEBUG --- http response --- 19:34:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:34:29 DEBUG ----- output ----- 19:34:29 DEBUG { 19:34:29 DEBUG "status": "success", 19:34:29 DEBUG "data": { 19:34:29 DEBUG "resultType": "vector", 19:34:29 DEBUG "result": [ 19:34:29 DEBUG { 19:34:29 DEBUG "metric": { 19:34:29 DEBUG "pod": "ds-cts-0" 19:34:29 DEBUG }, 19:34:29 DEBUG "value": [ 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG [...] 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG "pod": "ds-cts-2" 19:34:29 DEBUG }, 19:34:29 DEBUG "value": [ 19:34:29 DEBUG 1713551669, 19:34:29 DEBUG "30" 19:34:29 DEBUG ] 19:34:29 DEBUG } 19:34:29 DEBUG ] 19:34:29 DEBUG } 19:34:29 DEBUG } 19:34:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551669" 19:34:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551669" 19:34:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551669" 19:34:29 INFO [http_cmd]: http status code OK 19:34:29 INFO [http_cmd]: http status code OK 19:34:29 DEBUG --- status code --- 19:34:29 DEBUG --- status code --- 19:34:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:29 DEBUG --- http response --- 19:34:29 DEBUG --- http response --- 19:34:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:34:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:34:29 DEBUG ----- output ----- 19:34:29 DEBUG ----- output ----- 19:34:29 DEBUG { 19:34:29 DEBUG { 19:34:29 DEBUG "status": "success", 19:34:29 DEBUG "status": "success", 19:34:29 DEBUG "data": { 19:34:29 DEBUG "data": { 19:34:29 DEBUG "resultType": "vector", 19:34:29 DEBUG "resultType": "vector", 19:34:29 DEBUG "result": [ 19:34:29 DEBUG "result": [ 19:34:29 DEBUG { 19:34:29 DEBUG { 19:34:29 DEBUG "metric": { 19:34:29 DEBUG "metric": { 19:34:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:34:29 DEBUG "instance": "10.142.0.107:9100" 19:34:29 DEBUG }, 19:34:29 DEBUG }, 19:34:29 DEBUG "value": [ 19:34:29 DEBUG "value": [ 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG [...] 19:34:29 DEBUG [...] 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:34:29 DEBUG "instance": "10.142.0.43:9100" 19:34:29 DEBUG }, 19:34:29 DEBUG }, 19:34:29 DEBUG "value": [ 19:34:29 DEBUG "value": [ 19:34:29 DEBUG 1713551669, 19:34:29 DEBUG 1713551669, 19:34:29 DEBUG "2335.5666666666666" 19:34:29 DEBUG "70598744.93333334" 19:34:29 DEBUG ] 19:34:29 DEBUG ] 19:34:29 DEBUG } 19:34:29 DEBUG } 19:34:29 DEBUG ] 19:34:29 DEBUG ] 19:34:29 DEBUG } 19:34:29 DEBUG } 19:34:29 DEBUG } 19:34:29 DEBUG } 19:34:29 INFO [http_cmd]: http status code OK 19:34:29 DEBUG --- status code --- 19:34:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:29 DEBUG --- http response --- 19:34:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:34:29 DEBUG ----- output ----- 19:34:29 DEBUG { 19:34:29 DEBUG "status": "success", 19:34:29 DEBUG "data": { 19:34:29 DEBUG "resultType": "vector", 19:34:29 DEBUG "result": [ 19:34:29 DEBUG { 19:34:29 DEBUG "metric": { 19:34:29 DEBUG "instance": "10.142.0.107:9100" 19:34:29 DEBUG }, 19:34:29 DEBUG "value": [ 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG [...] 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG "instance": "10.142.0.43:9100" 19:34:29 DEBUG }, 19:34:29 DEBUG "value": [ 19:34:29 DEBUG 1713551669, 19:34:29 DEBUG "0.1213333333333291" 19:34:29 DEBUG ] 19:34:29 DEBUG } 19:34:29 DEBUG ] 19:34:29 DEBUG } 19:34:29 DEBUG } 19:34:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551669" 19:34:29 INFO [http_cmd]: http status code OK 19:34:29 DEBUG --- status code --- 19:34:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:29 DEBUG --- http response --- 19:34:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:34:29 DEBUG ----- output ----- 19:34:29 DEBUG { 19:34:29 DEBUG "status": "success", 19:34:29 DEBUG "data": { 19:34:29 DEBUG "resultType": "vector", 19:34:29 DEBUG "result": [ 19:34:29 DEBUG { 19:34:29 DEBUG "metric": { 19:34:29 DEBUG "pod": "ds-cts-0" 19:34:29 DEBUG }, 19:34:29 DEBUG "value": [ 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG [...] 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG "pod": "ds-cts-2" 19:34:29 DEBUG }, 19:34:29 DEBUG "value": [ 19:34:29 DEBUG 1713551669, 19:34:29 DEBUG "0.000033333333333333335" 19:34:29 DEBUG ] 19:34:29 DEBUG } 19:34:29 DEBUG ] 19:34:29 DEBUG } 19:34:29 DEBUG } 19:34:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551669" 19:34:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551669" 19:34:29 INFO [http_cmd]: http status code OK 19:34:29 DEBUG --- status code --- 19:34:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:29 DEBUG --- http response --- 19:34:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:34:29 DEBUG ----- output ----- 19:34:29 DEBUG { 19:34:29 DEBUG "status": "success", 19:34:29 DEBUG "data": { 19:34:29 DEBUG "resultType": "vector", 19:34:29 DEBUG "result": [ 19:34:29 DEBUG { 19:34:29 DEBUG "metric": { 19:34:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:34:29 DEBUG }, 19:34:29 DEBUG "value": [ 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG [...] 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:34:29 DEBUG }, 19:34:29 DEBUG "value": [ 19:34:29 DEBUG 1713551669, 19:34:29 DEBUG "2338.5333333333333" 19:34:29 DEBUG ] 19:34:29 DEBUG } 19:34:29 DEBUG ] 19:34:29 DEBUG } 19:34:29 DEBUG } 19:34:29 INFO [http_cmd]: http status code OK 19:34:29 DEBUG --- status code --- 19:34:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:29 DEBUG --- http response --- 19:34:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 19:34:29 DEBUG ----- output ----- 19:34:29 DEBUG { 19:34:29 DEBUG "status": "success", 19:34:29 DEBUG "data": { 19:34:29 DEBUG "resultType": "vector", 19:34:29 DEBUG "result": [ 19:34:29 DEBUG { 19:34:29 DEBUG "metric": { 19:34:29 DEBUG "pod": "ds-idrepo-1" 19:34:29 DEBUG }, 19:34:29 DEBUG "value": [ 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG [...] 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:34:29 DEBUG }, 19:34:29 DEBUG "value": [ 19:34:29 DEBUG 1713551669, 19:34:29 DEBUG "0" 19:34:29 DEBUG ] 19:34:29 DEBUG } 19:34:29 DEBUG ] 19:34:29 DEBUG } 19:34:29 DEBUG } 19:34:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713551669" 19:34:29 INFO [http_cmd]: http status code OK 19:34:29 DEBUG --- status code --- 19:34:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:29 DEBUG --- http response --- 19:34:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:34:29 DEBUG ----- output ----- 19:34:29 DEBUG { 19:34:29 DEBUG "status": "success", 19:34:29 DEBUG "data": { 19:34:29 DEBUG "resultType": "vector", 19:34:29 DEBUG "result": [ 19:34:29 DEBUG { 19:34:29 DEBUG "metric": { 19:34:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:34:29 DEBUG }, 19:34:29 DEBUG "value": [ 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG [...] 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:34:29 DEBUG }, 19:34:29 DEBUG "value": [ 19:34:29 DEBUG 1713551669, 19:34:29 DEBUG "0.009" 19:34:29 DEBUG ] 19:34:29 DEBUG } 19:34:29 DEBUG ] 19:34:29 DEBUG } 19:34:29 DEBUG } 19:34:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713551669" 19:34:29 INFO [http_cmd]: http status code OK 19:34:29 DEBUG --- status code --- 19:34:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:29 DEBUG --- http response --- 19:34:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:34:29 DEBUG ----- output ----- 19:34:29 DEBUG { 19:34:29 DEBUG "status": "success", 19:34:29 DEBUG "data": { 19:34:29 DEBUG "resultType": "vector", 19:34:29 DEBUG "result": [ 19:34:29 DEBUG { 19:34:29 DEBUG "metric": { 19:34:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:34:29 DEBUG }, 19:34:29 DEBUG "value": [ 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG [...] 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:34:29 DEBUG }, 19:34:29 DEBUG "value": [ 19:34:29 DEBUG 1713551669, 19:34:29 DEBUG "3403038720" 19:34:29 DEBUG ] 19:34:29 DEBUG } 19:34:29 DEBUG ] 19:34:29 DEBUG } 19:34:29 DEBUG } 19:34:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551669" 19:34:29 INFO [http_cmd]: http status code OK 19:34:29 DEBUG --- status code --- 19:34:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:29 DEBUG --- http response --- 19:34:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:34:29 DEBUG ----- output ----- 19:34:29 DEBUG { 19:34:29 DEBUG "status": "success", 19:34:29 DEBUG "data": { 19:34:29 DEBUG "resultType": "vector", 19:34:29 DEBUG "result": [ 19:34:29 DEBUG { 19:34:29 DEBUG "metric": { 19:34:29 DEBUG "pod": "ds-idrepo-1" 19:34:29 DEBUG }, 19:34:29 DEBUG "value": [ 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG [...] 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:34:29 DEBUG }, 19:34:29 DEBUG "value": [ 19:34:29 DEBUG 1713551669, 19:34:29 DEBUG "0.000029516868792033473" 19:34:29 DEBUG ] 19:34:29 DEBUG } 19:34:29 DEBUG ] 19:34:29 DEBUG } 19:34:29 DEBUG } 19:34:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551669" 19:34:29 INFO [http_cmd]: http status code OK 19:34:29 DEBUG --- status code --- 19:34:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:29 DEBUG --- http response --- 19:34:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:34:29 DEBUG ----- output ----- 19:34:29 DEBUG { 19:34:29 DEBUG "status": "success", 19:34:29 DEBUG "data": { 19:34:29 DEBUG "resultType": "vector", 19:34:29 DEBUG "result": [ 19:34:29 DEBUG { 19:34:29 DEBUG "metric": { 19:34:29 DEBUG "instance": "10.142.0.107:9100" 19:34:29 DEBUG }, 19:34:29 DEBUG "value": [ 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG [...] 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG 19:34:29 DEBUG "instance": "10.142.0.43:9100" 19:34:29 DEBUG }, 19:34:29 DEBUG "value": [ 19:34:29 DEBUG 1713551669, 19:34:29 DEBUG "70596224.33333334" 19:34:29 DEBUG ] 19:34:29 DEBUG } 19:34:29 DEBUG ] 19:34:29 DEBUG } 19:34:29 DEBUG } 19:34:30 INFO 19:34:30 INFO [loop_until]: kubectl --namespace=xlou top pods 19:34:30 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:34:30 INFO [loop_until]: OK (rc = 0) 19:34:30 DEBUG --- stdout --- 19:34:30 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 14406m 8994Mi am-767f6b67b5-qcqz2 14637m 8999Mi am-767f6b67b5-vbltz 15800m 8981Mi ds-cts-0 15349m 10886Mi ds-cts-1 14316m 11117Mi ds-cts-2 16252m 10877Mi ds-idrepo-0 6369m 3658Mi ds-idrepo-1 6402m 3530Mi ds-idrepo-2 6137m 3492Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 4m 1218Mi lodemon-f8459c597-6tj6f 1m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7773m 3153Mi 19:34:30 DEBUG --- stderr --- 19:34:30 DEBUG 19:34:30 INFO 19:34:30 INFO [loop_until]: kubectl --namespace=xlou top node 19:34:30 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:34:30 INFO [loop_until]: OK (rc = 0) 19:34:30 DEBUG --- stdout --- 19:34:30 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 73m 0% 6256Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 6525m 21% 7260Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13790m 46% 11700Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 3347Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 15375m 51% 13898Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 150m 0% 5197Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 14482m 48% 11880Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12341m 41% 3011Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 16385m 54% 11198Mi 10% gke-xlou-cdm-ds-0f601751-j488 7006m 23% 6740Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 14071m 47% 15934Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 6974m 23% 7418Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 16021m 53% 13358Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 7924m 49% 6937Mi 11% 19:34:30 DEBUG --- stderr --- 19:34:30 DEBUG 19:35:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713551729" 19:35:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551729" 19:35:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551729" 19:35:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551729" 19:35:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551729" 19:35:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713551729" 19:35:29 INFO [http_cmd]: http status code OK 19:35:29 DEBUG --- status code --- 19:35:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:29 DEBUG --- http response --- 19:35:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:35:29 DEBUG ----- output ----- 19:35:29 DEBUG { 19:35:29 DEBUG "status": "success", 19:35:29 DEBUG "data": { 19:35:29 DEBUG "resultType": "vector", 19:35:29 DEBUG "result": [ 19:35:29 DEBUG { 19:35:29 DEBUG "metric": { 19:35:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:35:29 DEBUG }, 19:35:29 DEBUG "value": [ 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG [...] 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:35:29 DEBUG }, 19:35:29 DEBUG "value": [ 19:35:29 DEBUG 1713551729, 19:35:29 DEBUG "420411.5" 19:35:29 DEBUG ] 19:35:29 DEBUG } 19:35:29 DEBUG ] 19:35:29 DEBUG } 19:35:29 DEBUG } 19:35:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713551729" 19:35:29 INFO [http_cmd]: http status code OK 19:35:29 DEBUG --- status code --- 19:35:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:29 DEBUG --- http response --- 19:35:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:35:29 DEBUG ----- output ----- 19:35:29 DEBUG { 19:35:29 DEBUG "status": "success", 19:35:29 DEBUG "data": { 19:35:29 DEBUG "resultType": "vector", 19:35:29 DEBUG "result": [ 19:35:29 DEBUG { 19:35:29 DEBUG "metric": { 19:35:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:35:29 DEBUG }, 19:35:29 DEBUG "value": [ 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG [...] 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:35:29 DEBUG }, 19:35:29 DEBUG "value": [ 19:35:29 DEBUG 1713551729, 19:35:29 DEBUG "7.697323137425866" 19:35:29 DEBUG ] 19:35:29 DEBUG } 19:35:29 DEBUG ] 19:35:29 DEBUG } 19:35:29 DEBUG } 19:35:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551729" 19:35:29 INFO [http_cmd]: http status code OK 19:35:29 DEBUG --- status code --- 19:35:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:29 DEBUG --- http response --- 19:35:29 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:35:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551729" 19:35:29 ERROR [http_cmd]: ERROR 19:35:29 INFO [http_cmd]: http status code OK 19:35:29 ERROR --- status code --- 19:35:29 DEBUG --- status code --- 19:35:29 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:35:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:29 ERROR --- http response --- 19:35:29 DEBUG --- http response --- 19:35:29 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:35:29 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:35:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:35:29 DEBUG ----- output ----- 19:35:29 DEBUG { 19:35:29 DEBUG "status": "success", 19:35:29 DEBUG "data": { 19:35:29 DEBUG "resultType": "vector", 19:35:29 DEBUG "result": [ 19:35:29 DEBUG { 19:35:29 DEBUG "metric": { 19:35:29 DEBUG "pod": "ds-idrepo-0" 19:35:29 DEBUG }, 19:35:29 DEBUG "value": [ 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG [...] 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:35:29 DEBUG }, 19:35:29 DEBUG "value": [ 19:35:29 DEBUG 1713551729, 19:35:29 DEBUG "0" 19:35:29 DEBUG ] 19:35:29 DEBUG } 19:35:29 DEBUG ] 19:35:29 DEBUG } 19:35:29 DEBUG } 19:35:29 INFO [http_cmd]: http status code OK 19:35:29 DEBUG --- status code --- 19:35:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:29 DEBUG --- http response --- 19:35:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:35:29 DEBUG ----- output ----- 19:35:29 DEBUG { 19:35:29 DEBUG "status": "success", 19:35:29 DEBUG "data": { 19:35:29 DEBUG "resultType": "vector", 19:35:29 DEBUG "result": [ 19:35:29 DEBUG { 19:35:29 DEBUG "metric": { 19:35:29 DEBUG "instance": "10.142.0.107:9100" 19:35:29 DEBUG }, 19:35:29 DEBUG "value": [ 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG [...] 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG "instance": "10.142.0.43:9100" 19:35:29 DEBUG }, 19:35:29 DEBUG "value": [ 19:35:29 DEBUG 1713551729, 19:35:29 DEBUG "1914753262592" 19:35:29 DEBUG ] 19:35:29 DEBUG } 19:35:29 DEBUG ] 19:35:29 DEBUG } 19:35:29 DEBUG } 19:35:29 INFO [http_cmd]: http status code OK 19:35:29 DEBUG --- status code --- 19:35:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:29 DEBUG --- http response --- 19:35:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:35:29 DEBUG ----- output ----- 19:35:29 DEBUG { 19:35:29 DEBUG "status": "success", 19:35:29 DEBUG "data": { 19:35:29 DEBUG "resultType": "vector", 19:35:29 DEBUG "result": [ 19:35:29 DEBUG { 19:35:29 DEBUG "metric": { 19:35:29 DEBUG "pod": "ds-idrepo-0" 19:35:29 DEBUG }, 19:35:29 DEBUG "value": [ 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG [...] 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG "pod": "ds-cts-2" 19:35:29 DEBUG }, 19:35:29 DEBUG "value": [ 19:35:29 DEBUG 1713551729, 19:35:29 DEBUG "0" 19:35:29 DEBUG ] 19:35:29 DEBUG } 19:35:29 DEBUG ] 19:35:29 DEBUG } 19:35:29 DEBUG } 19:35:29 INFO [http_cmd]: http status code OK 19:35:29 INFO [http_cmd]: http status code OK 19:35:29 DEBUG --- status code --- 19:35:29 DEBUG --- status code --- 19:35:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:29 DEBUG --- http response --- 19:35:29 DEBUG --- http response --- 19:35:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:35:29 DEBUG ----- output ----- 19:35:29 DEBUG { 19:35:29 DEBUG "status": "success", 19:35:29 DEBUG "data": { 19:35:29 DEBUG "resultType": "vector", 19:35:29 DEBUG "result": [ 19:35:29 DEBUG { 19:35:29 DEBUG "metric": { 19:35:29 DEBUG "pod": "ds-idrepo-0" 19:35:29 DEBUG }, 19:35:29 DEBUG "value": [ 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG [...] 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG "pod": "ds-idrepo-2" 19:35:29 DEBUG }, 19:35:29 DEBUG "value": [ 19:35:29 DEBUG 1713551729, 19:35:29 DEBUG "0" 19:35:29 DEBUG ] 19:35:29 DEBUG } 19:35:29 DEBUG ] 19:35:29 DEBUG } 19:35:29 DEBUG } 19:35:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:35:29 DEBUG ----- output ----- 19:35:29 DEBUG { 19:35:29 DEBUG "status": "success", 19:35:29 DEBUG "data": { 19:35:29 DEBUG "resultType": "vector", 19:35:29 DEBUG "result": [ 19:35:29 DEBUG { 19:35:29 DEBUG "metric": { 19:35:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:35:29 DEBUG }, 19:35:29 DEBUG "value": [ 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG [...] 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:35:29 DEBUG }, 19:35:29 DEBUG "value": [ 19:35:29 DEBUG 1713551729, 19:35:29 DEBUG "2337.1" 19:35:29 DEBUG ] 19:35:29 DEBUG } 19:35:29 DEBUG ] 19:35:29 DEBUG } 19:35:29 DEBUG } 19:35:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551729" 19:35:29 INFO [http_cmd]: http status code OK 19:35:29 DEBUG --- status code --- 19:35:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:29 DEBUG --- http response --- 19:35:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:35:29 DEBUG ----- output ----- 19:35:29 DEBUG { 19:35:29 DEBUG "status": "success", 19:35:29 DEBUG "data": { 19:35:29 DEBUG "resultType": "vector", 19:35:29 DEBUG "result": [ 19:35:29 DEBUG { 19:35:29 DEBUG "metric": { 19:35:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:35:29 DEBUG }, 19:35:29 DEBUG "value": [ 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG [...] 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:35:29 DEBUG }, 19:35:29 DEBUG "value": [ 19:35:29 DEBUG 1713551729, 19:35:29 DEBUG "0" 19:35:29 DEBUG ] 19:35:29 DEBUG } 19:35:29 DEBUG ] 19:35:29 DEBUG } 19:35:29 DEBUG } 19:35:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551729" 19:35:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551729" 19:35:29 INFO [http_cmd]: http status code OK 19:35:29 DEBUG --- status code --- 19:35:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:29 DEBUG --- http response --- 19:35:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:35:29 INFO [http_cmd]: http status code OK 19:35:29 DEBUG ----- output ----- 19:35:29 DEBUG --- status code --- 19:35:29 DEBUG { 19:35:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:29 DEBUG "status": "success", 19:35:29 DEBUG --- http response --- 19:35:29 DEBUG "data": { 19:35:29 DEBUG "resultType": "vector", 19:35:29 DEBUG "result": [ 19:35:29 DEBUG { 19:35:29 DEBUG "metric": { 19:35:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:35:29 DEBUG "pod": "ds-idrepo-0" 19:35:29 DEBUG ----- output ----- 19:35:29 DEBUG }, 19:35:29 DEBUG { 19:35:29 DEBUG "value": [ 19:35:29 DEBUG "status": "success", 19:35:29 DEBUG 19:35:29 DEBUG "data": { 19:35:29 DEBUG 19:35:29 DEBUG "resultType": "vector", 19:35:29 DEBUG 19:35:29 DEBUG "result": [ 19:35:29 DEBUG [...] 19:35:29 DEBUG { 19:35:29 DEBUG 19:35:29 DEBUG "metric": { 19:35:29 DEBUG 19:35:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:35:29 DEBUG 19:35:29 DEBUG }, 19:35:29 DEBUG "pod": "ds-idrepo-2" 19:35:29 DEBUG "value": [ 19:35:29 DEBUG }, 19:35:29 DEBUG 19:35:29 DEBUG "value": [ 19:35:29 DEBUG 19:35:29 DEBUG 1713551729, 19:35:29 DEBUG 19:35:29 DEBUG "0" 19:35:29 DEBUG [...] 19:35:29 DEBUG ] 19:35:29 DEBUG 19:35:29 DEBUG } 19:35:29 DEBUG 19:35:29 DEBUG ] 19:35:29 DEBUG 19:35:29 DEBUG } 19:35:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:35:29 DEBUG } 19:35:29 DEBUG }, 19:35:29 DEBUG "value": [ 19:35:29 DEBUG 1713551729, 19:35:29 DEBUG "2339.0666666666666" 19:35:29 DEBUG ] 19:35:29 DEBUG } 19:35:29 DEBUG ] 19:35:29 DEBUG } 19:35:29 DEBUG } 19:35:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551729" 19:35:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551729" 19:35:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551729" 19:35:29 INFO [http_cmd]: http status code OK 19:35:29 DEBUG --- status code --- 19:35:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:29 DEBUG --- http response --- 19:35:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:35:29 DEBUG ----- output ----- 19:35:29 DEBUG { 19:35:29 DEBUG "status": "success", 19:35:29 DEBUG "data": { 19:35:29 DEBUG "resultType": "vector", 19:35:29 DEBUG "result": [ 19:35:29 DEBUG { 19:35:29 DEBUG "metric": { 19:35:29 DEBUG "instance": "10.142.0.107:9100" 19:35:29 DEBUG }, 19:35:29 DEBUG "value": [ 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG [...] 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG "instance": "10.142.0.43:9100" 19:35:29 DEBUG }, 19:35:29 DEBUG "value": [ 19:35:29 DEBUG 1713551729, 19:35:29 DEBUG "0.17076666666669288" 19:35:29 DEBUG ] 19:35:29 DEBUG } 19:35:29 DEBUG ] 19:35:29 DEBUG } 19:35:29 DEBUG } 19:35:29 INFO [http_cmd]: http status code OK 19:35:29 DEBUG --- status code --- 19:35:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:29 DEBUG --- http response --- 19:35:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:35:29 DEBUG ----- output ----- 19:35:29 DEBUG { 19:35:29 DEBUG "status": "success", 19:35:29 DEBUG "data": { 19:35:29 DEBUG "resultType": "vector", 19:35:29 DEBUG "result": [ 19:35:29 DEBUG { 19:35:29 DEBUG "metric": { 19:35:29 DEBUG "pod": "ds-idrepo-0" 19:35:29 DEBUG }, 19:35:29 DEBUG "value": [ 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG [...] 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG "pod": "ds-idrepo-2" 19:35:29 DEBUG }, 19:35:29 DEBUG "value": [ 19:35:29 DEBUG 1713551729, 19:35:29 DEBUG "0" 19:35:29 DEBUG ] 19:35:29 DEBUG } 19:35:29 DEBUG ] 19:35:29 DEBUG } 19:35:29 DEBUG } 19:35:29 INFO [http_cmd]: http status code OK 19:35:29 DEBUG --- status code --- 19:35:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:29 DEBUG --- http response --- 19:35:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:35:29 DEBUG ----- output ----- 19:35:29 DEBUG { 19:35:29 DEBUG "status": "success", 19:35:29 DEBUG "data": { 19:35:29 DEBUG "resultType": "vector", 19:35:29 DEBUG "result": [ 19:35:29 DEBUG { 19:35:29 DEBUG "metric": { 19:35:29 DEBUG "pod": "ds-cts-0" 19:35:29 DEBUG }, 19:35:29 DEBUG "value": [ 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG [...] 19:35:29 DEBUG 19:35:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551729" 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG "pod": "ds-cts-2" 19:35:29 DEBUG }, 19:35:29 DEBUG "value": [ 19:35:29 DEBUG 1713551729, 19:35:29 DEBUG "0" 19:35:29 DEBUG ] 19:35:29 DEBUG } 19:35:29 DEBUG ] 19:35:29 DEBUG } 19:35:29 DEBUG } 19:35:29 INFO [http_cmd]: http status code OK 19:35:29 DEBUG --- status code --- 19:35:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:29 DEBUG --- http response --- 19:35:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 19:35:29 DEBUG ----- output ----- 19:35:29 DEBUG { 19:35:29 DEBUG "status": "success", 19:35:29 DEBUG "data": { 19:35:29 DEBUG "resultType": "vector", 19:35:29 DEBUG "result": [ 19:35:29 DEBUG { 19:35:29 DEBUG "metric": { 19:35:29 DEBUG "pod": "ds-idrepo-0" 19:35:29 DEBUG }, 19:35:29 DEBUG "value": [ 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG [...] 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:35:29 DEBUG }, 19:35:29 DEBUG "value": [ 19:35:29 DEBUG 1713551729, 19:35:29 DEBUG "0" 19:35:29 DEBUG ] 19:35:29 DEBUG } 19:35:29 DEBUG ] 19:35:29 DEBUG } 19:35:29 DEBUG } 19:35:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551729" 19:35:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551729" 19:35:29 INFO [http_cmd]: http status code OK 19:35:29 DEBUG --- status code --- 19:35:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:29 DEBUG --- http response --- 19:35:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:35:29 DEBUG ----- output ----- 19:35:29 DEBUG { 19:35:29 DEBUG "status": "success", 19:35:29 DEBUG "data": { 19:35:29 DEBUG "resultType": "vector", 19:35:29 DEBUG "result": [ 19:35:29 DEBUG { 19:35:29 DEBUG "metric": { 19:35:29 DEBUG "instance": "10.142.0.107:9100" 19:35:29 DEBUG }, 19:35:29 DEBUG "value": [ 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG [...] 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG "instance": "10.142.0.43:9100" 19:35:29 DEBUG }, 19:35:29 DEBUG "value": [ 19:35:29 DEBUG 1713551729, 19:35:29 DEBUG "70611934.8" 19:35:29 DEBUG ] 19:35:29 DEBUG } 19:35:29 DEBUG ] 19:35:29 DEBUG } 19:35:29 DEBUG } 19:35:29 INFO [http_cmd]: http status code OK 19:35:29 DEBUG --- status code --- 19:35:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:29 DEBUG --- http response --- 19:35:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:35:29 DEBUG ----- output ----- 19:35:29 DEBUG { 19:35:29 DEBUG "status": "success", 19:35:29 DEBUG "data": { 19:35:29 DEBUG "resultType": "vector", 19:35:29 DEBUG "result": [ 19:35:29 DEBUG { 19:35:29 DEBUG "metric": { 19:35:29 DEBUG "instance": "10.142.0.107:9100" 19:35:29 DEBUG }, 19:35:29 DEBUG "value": [ 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG [...] 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG "instance": "10.142.0.43:9100" 19:35:29 DEBUG }, 19:35:29 DEBUG "value": [ 19:35:29 DEBUG 1713551729, 19:35:29 DEBUG "0.07933333333334075" 19:35:29 DEBUG ] 19:35:29 DEBUG } 19:35:29 DEBUG ] 19:35:29 DEBUG } 19:35:29 DEBUG } 19:35:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551729" 19:35:29 INFO [http_cmd]: http status code OK 19:35:29 DEBUG --- status code --- 19:35:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:29 DEBUG --- http response --- 19:35:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:35:29 DEBUG ----- output ----- 19:35:29 DEBUG { 19:35:29 DEBUG "status": "success", 19:35:29 DEBUG "data": { 19:35:29 DEBUG "resultType": "vector", 19:35:29 DEBUG "result": [ 19:35:29 DEBUG { 19:35:29 DEBUG "metric": { 19:35:29 DEBUG "pod": "ds-cts-0" 19:35:29 DEBUG }, 19:35:29 DEBUG "value": [ 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG [...] 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG "pod": "ds-cts-2" 19:35:29 DEBUG }, 19:35:29 DEBUG "value": [ 19:35:29 DEBUG 1713551729, 19:35:29 DEBUG "29.9" 19:35:29 DEBUG ] 19:35:29 DEBUG } 19:35:29 DEBUG ] 19:35:29 DEBUG } 19:35:29 DEBUG } 19:35:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551729" 19:35:29 INFO [http_cmd]: http status code OK 19:35:29 DEBUG --- status code --- 19:35:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:29 DEBUG --- http response --- 19:35:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:35:29 DEBUG ----- output ----- 19:35:29 DEBUG { 19:35:29 DEBUG "status": "success", 19:35:29 DEBUG "data": { 19:35:29 DEBUG "resultType": "vector", 19:35:29 DEBUG "result": [ 19:35:29 DEBUG { 19:35:29 DEBUG "metric": { 19:35:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:35:29 DEBUG }, 19:35:29 DEBUG "value": [ 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG [...] 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:35:29 DEBUG }, 19:35:29 DEBUG "value": [ 19:35:29 DEBUG 1713551729, 19:35:29 DEBUG "2337" 19:35:29 DEBUG ] 19:35:29 DEBUG } 19:35:29 DEBUG ] 19:35:29 DEBUG } 19:35:29 DEBUG } 19:35:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551729" 19:35:29 INFO [http_cmd]: http status code OK 19:35:29 DEBUG --- status code --- 19:35:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:29 DEBUG --- http response --- 19:35:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:35:29 DEBUG ----- output ----- 19:35:29 DEBUG { 19:35:29 DEBUG "status": "success", 19:35:29 DEBUG "data": { 19:35:29 DEBUG "resultType": "vector", 19:35:29 DEBUG "result": [ 19:35:29 DEBUG { 19:35:29 DEBUG "metric": { 19:35:29 DEBUG "pod": "ds-cts-0" 19:35:29 DEBUG }, 19:35:29 DEBUG "value": [ 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG [...] 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG "pod": "ds-cts-2" 19:35:29 DEBUG }, 19:35:29 DEBUG "value": [ 19:35:29 DEBUG 1713551729, 19:35:29 DEBUG "0.000019882222222222222" 19:35:29 DEBUG ] 19:35:29 DEBUG } 19:35:29 DEBUG ] 19:35:29 DEBUG } 19:35:29 DEBUG } 19:35:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551729" 19:35:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551729" 19:35:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713551729" 19:35:29 INFO [http_cmd]: http status code OK 19:35:29 DEBUG --- status code --- 19:35:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:29 DEBUG --- http response --- 19:35:29 INFO [http_cmd]: http status code OK 19:35:29 INFO [http_cmd]: http status code OK 19:35:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:35:29 DEBUG --- status code --- 19:35:29 DEBUG --- status code --- 19:35:29 DEBUG ----- output ----- 19:35:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:29 DEBUG { 19:35:29 DEBUG --- http response --- 19:35:29 DEBUG --- http response --- 19:35:29 DEBUG "status": "success", 19:35:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:35:29 DEBUG "data": { 19:35:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:35:29 DEBUG ----- output ----- 19:35:29 DEBUG "resultType": "vector", 19:35:29 DEBUG ----- output ----- 19:35:29 DEBUG { 19:35:29 DEBUG "result": [ 19:35:29 DEBUG { 19:35:29 DEBUG "status": "success", 19:35:29 DEBUG { 19:35:29 DEBUG "status": "success", 19:35:29 DEBUG "data": { 19:35:29 DEBUG "metric": { 19:35:29 DEBUG "data": { 19:35:29 DEBUG "resultType": "vector", 19:35:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:35:29 DEBUG "resultType": "vector", 19:35:29 DEBUG "result": [ 19:35:29 DEBUG }, 19:35:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713551729" 19:35:29 DEBUG "result": [ 19:35:29 DEBUG { 19:35:29 DEBUG "value": [ 19:35:29 DEBUG { 19:35:29 DEBUG "metric": { 19:35:29 DEBUG 19:35:29 DEBUG "metric": { 19:35:29 DEBUG "pod": "ds-idrepo-0" 19:35:29 DEBUG 19:35:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:35:29 DEBUG }, 19:35:29 DEBUG 19:35:29 DEBUG }, 19:35:29 DEBUG "value": [ 19:35:29 DEBUG [...] 19:35:29 DEBUG "value": [ 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG [...] 19:35:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:35:29 DEBUG [...] 19:35:29 DEBUG 19:35:29 DEBUG }, 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG "value": [ 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG 1713551729, 19:35:29 DEBUG 19:35:29 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:35:29 DEBUG "2336.8" 19:35:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:35:29 DEBUG }, 19:35:29 DEBUG ] 19:35:29 DEBUG }, 19:35:29 DEBUG "value": [ 19:35:29 DEBUG } 19:35:29 DEBUG "value": [ 19:35:29 DEBUG 1713551729, 19:35:29 DEBUG ] 19:35:29 DEBUG 1713551729, 19:35:29 DEBUG "0" 19:35:29 DEBUG } 19:35:29 DEBUG "0.009" 19:35:29 DEBUG ] 19:35:29 DEBUG } 19:35:29 DEBUG ] 19:35:29 DEBUG } 19:35:29 DEBUG } 19:35:29 DEBUG ] 19:35:29 DEBUG ] 19:35:29 DEBUG } 19:35:29 DEBUG } 19:35:29 DEBUG } 19:35:29 DEBUG } 19:35:29 INFO [http_cmd]: http status code OK 19:35:29 DEBUG --- status code --- 19:35:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:29 DEBUG --- http response --- 19:35:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:35:29 DEBUG ----- output ----- 19:35:29 DEBUG { 19:35:29 DEBUG "status": "success", 19:35:29 DEBUG "data": { 19:35:29 DEBUG "resultType": "vector", 19:35:29 DEBUG "result": [ 19:35:29 DEBUG { 19:35:29 DEBUG "metric": { 19:35:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:35:29 DEBUG }, 19:35:29 DEBUG "value": [ 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG [...] 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:35:29 DEBUG }, 19:35:29 DEBUG "value": [ 19:35:29 DEBUG 1713551729, 19:35:29 DEBUG "3405897728" 19:35:29 DEBUG ] 19:35:29 DEBUG } 19:35:29 DEBUG ] 19:35:29 DEBUG } 19:35:29 DEBUG } 19:35:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551729" 19:35:29 INFO [http_cmd]: http status code OK 19:35:29 DEBUG --- status code --- 19:35:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:29 DEBUG --- http response --- 19:35:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 19:35:29 DEBUG ----- output ----- 19:35:29 DEBUG { 19:35:29 DEBUG "status": "success", 19:35:29 DEBUG "data": { 19:35:29 DEBUG "resultType": "vector", 19:35:29 DEBUG "result": [ 19:35:29 DEBUG { 19:35:29 DEBUG "metric": { 19:35:29 DEBUG "pod": "ds-idrepo-0" 19:35:29 DEBUG }, 19:35:29 DEBUG "value": [ 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG [...] 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:35:29 DEBUG }, 19:35:29 DEBUG "value": [ 19:35:29 DEBUG 1713551729, 19:35:29 DEBUG "0.000012985328011845482" 19:35:29 DEBUG ] 19:35:29 DEBUG } 19:35:29 DEBUG ] 19:35:29 DEBUG } 19:35:29 DEBUG } 19:35:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551729" 19:35:29 INFO [http_cmd]: http status code OK 19:35:29 DEBUG --- status code --- 19:35:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:29 DEBUG --- http response --- 19:35:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:35:29 DEBUG ----- output ----- 19:35:29 DEBUG { 19:35:29 DEBUG "status": "success", 19:35:29 DEBUG "data": { 19:35:29 DEBUG "resultType": "vector", 19:35:29 DEBUG "result": [ 19:35:29 DEBUG { 19:35:29 DEBUG "metric": { 19:35:29 DEBUG "instance": "10.142.0.107:9100" 19:35:29 DEBUG }, 19:35:29 DEBUG "value": [ 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG [...] 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG 19:35:29 DEBUG "instance": "10.142.0.43:9100" 19:35:29 DEBUG }, 19:35:29 DEBUG "value": [ 19:35:29 DEBUG 1713551729, 19:35:29 DEBUG "70609637.96666667" 19:35:29 DEBUG ] 19:35:29 DEBUG } 19:35:29 DEBUG ] 19:35:29 DEBUG } 19:35:29 DEBUG } 19:35:30 INFO 19:35:30 INFO [loop_until]: kubectl --namespace=xlou top pods 19:35:30 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:35:30 INFO [loop_until]: OK (rc = 0) 19:35:30 DEBUG --- stdout --- 19:35:30 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 14456m 8994Mi am-767f6b67b5-qcqz2 14403m 8998Mi am-767f6b67b5-vbltz 15343m 8986Mi ds-cts-0 16084m 10873Mi ds-cts-1 14140m 11135Mi ds-cts-2 15272m 10897Mi ds-idrepo-0 6406m 3658Mi ds-idrepo-1 6229m 3530Mi ds-idrepo-2 6460m 3492Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 1m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7761m 3155Mi 19:35:30 DEBUG --- stderr --- 19:35:30 DEBUG 19:35:30 INFO 19:35:30 INFO [loop_until]: kubectl --namespace=xlou top node 19:35:30 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:35:30 INFO [loop_until]: OK (rc = 0) 19:35:30 DEBUG --- stdout --- 19:35:30 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 71m 0% 6262Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 6535m 21% 7254Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 14222m 47% 11714Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 84m 0% 3347Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 15050m 50% 13905Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 140m 0% 5207Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 14387m 48% 11868Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12329m 41% 2996Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 16467m 55% 11206Mi 10% gke-xlou-cdm-ds-0f601751-j488 7025m 23% 6728Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 14232m 47% 15933Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 6897m 23% 7433Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 15679m 52% 13371Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 7635m 48% 6939Mi 11% 19:35:30 DEBUG --- stderr --- 19:35:30 DEBUG 19:36:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551789" 19:36:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713551789" 19:36:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551789" 19:36:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551789" 19:36:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713551789" 19:36:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551789" 19:36:29 INFO [http_cmd]: http status code OK 19:36:29 DEBUG --- status code --- 19:36:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:29 DEBUG --- http response --- 19:36:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:36:29 DEBUG ----- output ----- 19:36:29 DEBUG { 19:36:29 DEBUG "status": "success", 19:36:29 DEBUG "data": { 19:36:29 DEBUG "resultType": "vector", 19:36:29 DEBUG "result": [ 19:36:29 DEBUG { 19:36:29 DEBUG "metric": { 19:36:29 DEBUG "instance": "10.142.0.107:9100" 19:36:29 DEBUG }, 19:36:29 DEBUG "value": [ 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG [...] 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG "instance": "10.142.0.43:9100" 19:36:29 DEBUG }, 19:36:29 DEBUG "value": [ 19:36:29 DEBUG 1713551789, 19:36:29 DEBUG "1918024092672" 19:36:29 DEBUG ] 19:36:29 DEBUG } 19:36:29 DEBUG ] 19:36:29 DEBUG } 19:36:29 DEBUG } 19:36:29 ERROR [http_cmd]: ERROR 19:36:29 ERROR --- status code --- 19:36:29 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:36:29 ERROR --- http response --- 19:36:29 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:36:29 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:36:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551789" 19:36:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713551789" 19:36:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551789" 19:36:29 INFO [http_cmd]: http status code OK 19:36:29 DEBUG --- status code --- 19:36:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:29 DEBUG --- http response --- 19:36:29 INFO [http_cmd]: http status code OK 19:36:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:36:29 INFO [http_cmd]: http status code OK 19:36:29 INFO [http_cmd]: http status code OK 19:36:29 DEBUG --- status code --- 19:36:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551789" 19:36:29 DEBUG ----- output ----- 19:36:29 DEBUG --- status code --- 19:36:29 DEBUG --- status code --- 19:36:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:29 DEBUG { 19:36:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:29 DEBUG --- http response --- 19:36:29 DEBUG "status": "success", 19:36:29 DEBUG --- http response --- 19:36:29 DEBUG --- http response --- 19:36:29 DEBUG "data": { 19:36:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:36:29 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:36:29 DEBUG "resultType": "vector", 19:36:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:36:29 DEBUG ----- output ----- 19:36:29 DEBUG "result": [ 19:36:29 INFO [http_cmd]: http status code OK 19:36:29 DEBUG ----- output ----- 19:36:29 DEBUG { 19:36:29 DEBUG { 19:36:29 DEBUG --- status code --- 19:36:29 DEBUG { 19:36:29 DEBUG "status": "success", 19:36:29 DEBUG "metric": { 19:36:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:29 DEBUG "status": "success", 19:36:29 DEBUG "data": { 19:36:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:36:29 INFO [http_cmd]: http status code OK 19:36:29 DEBUG --- http response --- 19:36:29 DEBUG "data": { 19:36:29 DEBUG "resultType": "vector", 19:36:29 DEBUG }, 19:36:29 DEBUG --- status code --- 19:36:29 DEBUG "resultType": "vector", 19:36:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:36:29 DEBUG "result": [ 19:36:29 DEBUG "value": [ 19:36:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:29 DEBUG "result": [ 19:36:29 DEBUG ----- output ----- 19:36:29 DEBUG { 19:36:29 DEBUG 19:36:29 DEBUG --- http response --- 19:36:29 DEBUG { 19:36:29 DEBUG { 19:36:29 DEBUG "metric": { 19:36:29 DEBUG 19:36:29 DEBUG "metric": { 19:36:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:36:29 DEBUG "status": "success", 19:36:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:36:29 DEBUG 19:36:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:36:29 DEBUG ----- output ----- 19:36:29 DEBUG "data": { 19:36:29 DEBUG }, 19:36:29 DEBUG [...] 19:36:29 DEBUG }, 19:36:29 DEBUG { 19:36:29 DEBUG "resultType": "vector", 19:36:29 DEBUG "value": [ 19:36:29 DEBUG 19:36:29 DEBUG "value": [ 19:36:29 DEBUG "status": "success", 19:36:29 DEBUG "result": [ 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG "data": { 19:36:29 DEBUG { 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG "resultType": "vector", 19:36:29 DEBUG "metric": { 19:36:29 DEBUG 19:36:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:36:29 DEBUG 19:36:29 DEBUG "result": [ 19:36:29 DEBUG "pod": "ds-idrepo-1" 19:36:29 DEBUG [...] 19:36:29 DEBUG }, 19:36:29 DEBUG [...] 19:36:29 DEBUG { 19:36:29 DEBUG }, 19:36:29 INFO [http_cmd]: http status code OK 19:36:29 DEBUG 19:36:29 DEBUG "value": [ 19:36:29 DEBUG 19:36:29 DEBUG "metric": { 19:36:29 DEBUG "value": [ 19:36:29 DEBUG --- status code --- 19:36:29 DEBUG 19:36:29 DEBUG 1713551789, 19:36:29 DEBUG 19:36:29 DEBUG "pod": "ds-idrepo-0" 19:36:29 DEBUG 19:36:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:29 INFO [http_cmd]: http status code OK 19:36:29 DEBUG 19:36:29 DEBUG "2340.7" 19:36:29 DEBUG 19:36:29 DEBUG }, 19:36:29 DEBUG 19:36:29 DEBUG --- http response --- 19:36:29 DEBUG --- status code --- 19:36:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:36:29 DEBUG ] 19:36:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:36:29 DEBUG "value": [ 19:36:29 DEBUG 19:36:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:36:29 DEBUG }, 19:36:29 DEBUG } 19:36:29 DEBUG }, 19:36:29 DEBUG 19:36:29 DEBUG [...] 19:36:29 DEBUG --- http response --- 19:36:29 DEBUG ----- output ----- 19:36:29 DEBUG "value": [ 19:36:29 DEBUG ] 19:36:29 DEBUG "value": [ 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG { 19:36:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:36:29 DEBUG 1713551789, 19:36:29 DEBUG } 19:36:29 DEBUG 1713551789, 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG "status": "success", 19:36:29 DEBUG ----- output ----- 19:36:29 DEBUG "7.719709844769799" 19:36:29 DEBUG } 19:36:29 DEBUG "420596.5" 19:36:29 DEBUG [...] 19:36:29 DEBUG 19:36:29 DEBUG "data": { 19:36:29 DEBUG { 19:36:29 DEBUG ] 19:36:29 DEBUG ] 19:36:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551789" 19:36:29 DEBUG 19:36:29 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:36:29 DEBUG "resultType": "vector", 19:36:29 DEBUG "status": "success", 19:36:29 DEBUG } 19:36:29 DEBUG } 19:36:29 DEBUG 19:36:29 DEBUG }, 19:36:29 DEBUG "result": [ 19:36:29 DEBUG "data": { 19:36:29 DEBUG ] 19:36:29 DEBUG ] 19:36:29 DEBUG 19:36:29 DEBUG "value": [ 19:36:29 DEBUG { 19:36:29 DEBUG "resultType": "vector", 19:36:29 DEBUG } 19:36:29 DEBUG } 19:36:29 DEBUG "pod": "ds-cts-2" 19:36:29 DEBUG 1713551789, 19:36:29 DEBUG "metric": { 19:36:29 DEBUG "result": [ 19:36:29 DEBUG } 19:36:29 DEBUG } 19:36:29 DEBUG }, 19:36:29 DEBUG "0" 19:36:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:36:29 DEBUG { 19:36:29 DEBUG "value": [ 19:36:29 DEBUG ] 19:36:29 DEBUG }, 19:36:29 DEBUG "metric": { 19:36:29 DEBUG 1713551789, 19:36:29 DEBUG } 19:36:29 DEBUG "value": [ 19:36:29 DEBUG "pod": "ds-idrepo-0" 19:36:29 DEBUG "0" 19:36:29 DEBUG ] 19:36:29 DEBUG 19:36:29 DEBUG }, 19:36:29 DEBUG ] 19:36:29 DEBUG } 19:36:29 DEBUG 19:36:29 DEBUG "value": [ 19:36:29 DEBUG } 19:36:29 DEBUG } 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG ] 19:36:29 DEBUG [...] 19:36:29 DEBUG 19:36:29 DEBUG } 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG } 19:36:29 DEBUG 19:36:29 DEBUG [...] 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:36:29 DEBUG 19:36:29 DEBUG }, 19:36:29 DEBUG 19:36:29 DEBUG "value": [ 19:36:29 DEBUG "pod": "ds-idrepo-2" 19:36:29 DEBUG 1713551789, 19:36:29 DEBUG }, 19:36:29 DEBUG "2345.733333333333" 19:36:29 DEBUG "value": [ 19:36:29 DEBUG ] 19:36:29 DEBUG 1713551789, 19:36:29 DEBUG } 19:36:29 DEBUG "0" 19:36:29 DEBUG ] 19:36:29 DEBUG ] 19:36:29 DEBUG } 19:36:29 DEBUG } 19:36:29 DEBUG } 19:36:29 DEBUG ] 19:36:29 DEBUG } 19:36:29 DEBUG } 19:36:29 INFO [http_cmd]: http status code OK 19:36:29 DEBUG --- status code --- 19:36:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:29 DEBUG --- http response --- 19:36:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:36:29 DEBUG ----- output ----- 19:36:29 DEBUG { 19:36:29 DEBUG "status": "success", 19:36:29 DEBUG "data": { 19:36:29 DEBUG "resultType": "vector", 19:36:29 DEBUG "result": [ 19:36:29 DEBUG { 19:36:29 DEBUG "metric": { 19:36:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:36:29 DEBUG }, 19:36:29 DEBUG "value": [ 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG [...] 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:36:29 DEBUG }, 19:36:29 DEBUG "value": [ 19:36:29 DEBUG 1713551789, 19:36:29 DEBUG "0" 19:36:29 DEBUG ] 19:36:29 DEBUG } 19:36:29 DEBUG ] 19:36:29 DEBUG } 19:36:29 DEBUG } 19:36:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551789" 19:36:29 INFO [http_cmd]: http status code OK 19:36:29 DEBUG --- status code --- 19:36:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:29 DEBUG --- http response --- 19:36:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:36:29 DEBUG ----- output ----- 19:36:29 DEBUG { 19:36:29 DEBUG "status": "success", 19:36:29 DEBUG "data": { 19:36:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551789" 19:36:29 DEBUG "resultType": "vector", 19:36:29 DEBUG "result": [ 19:36:29 DEBUG { 19:36:29 DEBUG "metric": { 19:36:29 DEBUG "pod": "ds-idrepo-0" 19:36:29 DEBUG }, 19:36:29 DEBUG "value": [ 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG [...] 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG "pod": "ds-idrepo-2" 19:36:29 DEBUG }, 19:36:29 DEBUG "value": [ 19:36:29 DEBUG 1713551789, 19:36:29 DEBUG "0" 19:36:29 DEBUG ] 19:36:29 DEBUG } 19:36:29 DEBUG ] 19:36:29 DEBUG } 19:36:29 DEBUG } 19:36:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551789" 19:36:29 INFO [http_cmd]: http status code OK 19:36:29 DEBUG --- status code --- 19:36:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:29 DEBUG --- http response --- 19:36:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:36:29 DEBUG ----- output ----- 19:36:29 DEBUG { 19:36:29 DEBUG "status": "success", 19:36:29 DEBUG "data": { 19:36:29 DEBUG "resultType": "vector", 19:36:29 DEBUG "result": [ 19:36:29 DEBUG { 19:36:29 DEBUG "metric": { 19:36:29 DEBUG "instance": "10.142.0.107:9100" 19:36:29 DEBUG }, 19:36:29 DEBUG "value": [ 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG [...] 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG "instance": "10.142.0.43:9100" 19:36:29 DEBUG }, 19:36:29 DEBUG "value": [ 19:36:29 DEBUG 1713551789, 19:36:29 DEBUG "0.1673333333332939" 19:36:29 DEBUG ] 19:36:29 DEBUG } 19:36:29 DEBUG ] 19:36:29 DEBUG } 19:36:29 DEBUG } 19:36:29 INFO [http_cmd]: http status code OK 19:36:29 DEBUG --- status code --- 19:36:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:29 DEBUG --- http response --- 19:36:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:36:29 DEBUG ----- output ----- 19:36:29 DEBUG { 19:36:29 DEBUG "status": "success", 19:36:29 DEBUG "data": { 19:36:29 DEBUG "resultType": "vector", 19:36:29 DEBUG "result": [ 19:36:29 DEBUG { 19:36:29 DEBUG "metric": { 19:36:29 DEBUG "pod": "ds-cts-0" 19:36:29 DEBUG }, 19:36:29 DEBUG "value": [ 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG [...] 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG "pod": "ds-cts-2" 19:36:29 DEBUG }, 19:36:29 DEBUG "value": [ 19:36:29 DEBUG 1713551789, 19:36:29 DEBUG "0" 19:36:29 DEBUG ] 19:36:29 DEBUG } 19:36:29 DEBUG ] 19:36:29 DEBUG } 19:36:29 DEBUG } 19:36:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551789" 19:36:29 INFO [http_cmd]: http status code OK 19:36:29 DEBUG --- status code --- 19:36:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:29 DEBUG --- http response --- 19:36:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:36:29 DEBUG ----- output ----- 19:36:29 DEBUG { 19:36:29 DEBUG "status": "success", 19:36:29 DEBUG "data": { 19:36:29 DEBUG "resultType": "vector", 19:36:29 DEBUG "result": [ 19:36:29 DEBUG { 19:36:29 DEBUG "metric": { 19:36:29 DEBUG "pod": "ds-idrepo-0" 19:36:29 DEBUG }, 19:36:29 DEBUG "value": [ 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG [...] 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG "pod": "ds-idrepo-2" 19:36:29 DEBUG }, 19:36:29 DEBUG "value": [ 19:36:29 DEBUG 1713551789, 19:36:29 DEBUG "0" 19:36:29 DEBUG ] 19:36:29 DEBUG } 19:36:29 DEBUG ] 19:36:29 DEBUG } 19:36:29 DEBUG } 19:36:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551789" 19:36:29 INFO [http_cmd]: http status code OK 19:36:29 DEBUG --- status code --- 19:36:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:29 DEBUG --- http response --- 19:36:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:36:29 DEBUG ----- output ----- 19:36:29 DEBUG { 19:36:29 DEBUG "status": "success", 19:36:29 DEBUG "data": { 19:36:29 DEBUG "resultType": "vector", 19:36:29 DEBUG "result": [ 19:36:29 DEBUG { 19:36:29 DEBUG "metric": { 19:36:29 DEBUG "pod": "ds-idrepo-1" 19:36:29 DEBUG }, 19:36:29 DEBUG "value": [ 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG [...] 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:36:29 DEBUG }, 19:36:29 DEBUG "value": [ 19:36:29 DEBUG 1713551789, 19:36:29 DEBUG "0" 19:36:29 DEBUG ] 19:36:29 DEBUG } 19:36:29 DEBUG ] 19:36:29 DEBUG } 19:36:29 DEBUG } 19:36:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551789" 19:36:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551789" 19:36:29 INFO [http_cmd]: http status code OK 19:36:29 DEBUG --- status code --- 19:36:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:29 DEBUG --- http response --- 19:36:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:36:29 DEBUG ----- output ----- 19:36:29 DEBUG { 19:36:29 DEBUG "status": "success", 19:36:29 DEBUG "data": { 19:36:29 DEBUG "resultType": "vector", 19:36:29 DEBUG "result": [ 19:36:29 DEBUG { 19:36:29 DEBUG "metric": { 19:36:29 DEBUG "instance": "10.142.0.107:9100" 19:36:29 DEBUG }, 19:36:29 DEBUG "value": [ 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG [...] 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG "instance": "10.142.0.43:9100" 19:36:29 DEBUG }, 19:36:29 DEBUG "value": [ 19:36:29 DEBUG 1713551789, 19:36:29 DEBUG "70748901.6" 19:36:29 DEBUG ] 19:36:29 DEBUG } 19:36:29 DEBUG ] 19:36:29 DEBUG } 19:36:29 DEBUG } 19:36:29 INFO [http_cmd]: http status code OK 19:36:29 DEBUG --- status code --- 19:36:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:29 DEBUG --- http response --- 19:36:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:36:29 DEBUG ----- output ----- 19:36:29 DEBUG { 19:36:29 DEBUG "status": "success", 19:36:29 DEBUG "data": { 19:36:29 DEBUG "resultType": "vector", 19:36:29 DEBUG "result": [ 19:36:29 DEBUG { 19:36:29 DEBUG "metric": { 19:36:29 DEBUG "instance": "10.142.0.107:9100" 19:36:29 DEBUG }, 19:36:29 DEBUG "value": [ 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG [...] 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG "instance": "10.142.0.43:9100" 19:36:29 DEBUG }, 19:36:29 DEBUG "value": [ 19:36:29 DEBUG 1713551789, 19:36:29 DEBUG "0.07833333333332461" 19:36:29 DEBUG ] 19:36:29 DEBUG } 19:36:29 DEBUG ] 19:36:29 DEBUG } 19:36:29 DEBUG } 19:36:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551789" 19:36:29 INFO [http_cmd]: http status code OK 19:36:29 DEBUG --- status code --- 19:36:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:29 DEBUG --- http response --- 19:36:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:36:29 DEBUG ----- output ----- 19:36:29 DEBUG { 19:36:29 DEBUG "status": "success", 19:36:29 DEBUG "data": { 19:36:29 DEBUG "resultType": "vector", 19:36:29 DEBUG "result": [ 19:36:29 DEBUG { 19:36:29 DEBUG "metric": { 19:36:29 DEBUG "pod": "ds-cts-0" 19:36:29 DEBUG }, 19:36:29 DEBUG "value": [ 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG [...] 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG "pod": "ds-cts-2" 19:36:29 DEBUG }, 19:36:29 DEBUG "value": [ 19:36:29 DEBUG 1713551789, 19:36:29 DEBUG "29.933333333333334" 19:36:29 DEBUG ] 19:36:29 DEBUG } 19:36:29 DEBUG ] 19:36:29 DEBUG } 19:36:29 DEBUG } 19:36:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551789" 19:36:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551789" 19:36:29 INFO [http_cmd]: http status code OK 19:36:29 DEBUG --- status code --- 19:36:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:29 DEBUG --- http response --- 19:36:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:36:29 DEBUG ----- output ----- 19:36:29 DEBUG { 19:36:29 DEBUG "status": "success", 19:36:29 DEBUG "data": { 19:36:29 DEBUG "resultType": "vector", 19:36:29 DEBUG "result": [ 19:36:29 DEBUG { 19:36:29 DEBUG "metric": { 19:36:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:36:29 DEBUG }, 19:36:29 DEBUG "value": [ 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG [...] 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:36:29 DEBUG }, 19:36:29 DEBUG "value": [ 19:36:29 DEBUG 1713551789, 19:36:29 DEBUG "2345.8" 19:36:29 DEBUG ] 19:36:29 DEBUG } 19:36:29 DEBUG ] 19:36:29 DEBUG } 19:36:29 DEBUG } 19:36:29 INFO [http_cmd]: http status code OK 19:36:29 DEBUG --- status code --- 19:36:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:29 DEBUG --- http response --- 19:36:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:36:29 DEBUG ----- output ----- 19:36:29 DEBUG { 19:36:29 DEBUG "status": "success", 19:36:29 DEBUG "data": { 19:36:29 DEBUG "resultType": "vector", 19:36:29 DEBUG "result": [ 19:36:29 DEBUG { 19:36:29 DEBUG "metric": { 19:36:29 DEBUG "pod": "ds-cts-0" 19:36:29 DEBUG }, 19:36:29 DEBUG "value": [ 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG [...] 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG "pod": "ds-cts-2" 19:36:29 DEBUG }, 19:36:29 DEBUG "value": [ 19:36:29 DEBUG 1713551789, 19:36:29 DEBUG "0" 19:36:29 DEBUG ] 19:36:29 DEBUG } 19:36:29 DEBUG ] 19:36:29 DEBUG } 19:36:29 DEBUG } 19:36:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551789" 19:36:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551789" 19:36:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713551789" 19:36:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713551789" 19:36:29 INFO [http_cmd]: http status code OK 19:36:29 INFO [http_cmd]: http status code OK 19:36:29 DEBUG --- status code --- 19:36:29 DEBUG --- status code --- 19:36:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:29 DEBUG --- http response --- 19:36:29 DEBUG --- http response --- 19:36:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:36:29 DEBUG ----- output ----- 19:36:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:36:29 INFO [http_cmd]: http status code OK 19:36:29 DEBUG { 19:36:29 DEBUG ----- output ----- 19:36:29 DEBUG --- status code --- 19:36:29 DEBUG "status": "success", 19:36:29 DEBUG { 19:36:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:29 DEBUG "data": { 19:36:29 DEBUG "status": "success", 19:36:29 DEBUG --- http response --- 19:36:29 DEBUG "resultType": "vector", 19:36:29 DEBUG "data": { 19:36:29 DEBUG "result": [ 19:36:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:36:29 DEBUG "resultType": "vector", 19:36:29 DEBUG { 19:36:29 DEBUG ----- output ----- 19:36:29 DEBUG "result": [ 19:36:29 DEBUG "metric": { 19:36:29 DEBUG { 19:36:29 DEBUG { 19:36:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:36:29 DEBUG "status": "success", 19:36:29 DEBUG "metric": { 19:36:29 DEBUG }, 19:36:29 DEBUG "data": { 19:36:29 DEBUG "pod": "ds-idrepo-1" 19:36:29 DEBUG "value": [ 19:36:29 DEBUG "resultType": "vector", 19:36:29 DEBUG }, 19:36:29 DEBUG 19:36:29 DEBUG "result": [ 19:36:29 DEBUG "value": [ 19:36:29 DEBUG 19:36:29 DEBUG { 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG "metric": { 19:36:29 DEBUG 19:36:29 DEBUG [...] 19:36:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG }, 19:36:29 DEBUG [...] 19:36:29 DEBUG 19:36:29 DEBUG "value": [ 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG }, 19:36:29 DEBUG 19:36:29 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:36:29 DEBUG "value": [ 19:36:29 DEBUG [...] 19:36:29 DEBUG }, 19:36:29 DEBUG 1713551789, 19:36:29 DEBUG 19:36:29 DEBUG "value": [ 19:36:29 DEBUG "2345.8333333333335" 19:36:29 DEBUG 19:36:29 DEBUG 1713551789, 19:36:29 DEBUG ] 19:36:29 DEBUG 19:36:29 DEBUG "0" 19:36:29 DEBUG } 19:36:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:36:29 DEBUG ] 19:36:29 DEBUG ] 19:36:29 DEBUG }, 19:36:29 DEBUG } 19:36:29 DEBUG } 19:36:29 DEBUG "value": [ 19:36:29 DEBUG ] 19:36:29 DEBUG } 19:36:29 DEBUG 1713551789, 19:36:29 DEBUG } 19:36:29 DEBUG "0.01" 19:36:29 DEBUG } 19:36:29 DEBUG ] 19:36:29 DEBUG } 19:36:29 DEBUG ] 19:36:29 DEBUG } 19:36:29 DEBUG } 19:36:29 INFO [http_cmd]: http status code OK 19:36:29 DEBUG --- status code --- 19:36:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:29 DEBUG --- http response --- 19:36:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:36:29 DEBUG ----- output ----- 19:36:29 DEBUG { 19:36:29 DEBUG "status": "success", 19:36:29 DEBUG "data": { 19:36:29 DEBUG "resultType": "vector", 19:36:29 DEBUG "result": [ 19:36:29 DEBUG { 19:36:29 DEBUG "metric": { 19:36:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:36:29 DEBUG }, 19:36:29 DEBUG "value": [ 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG [...] 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:36:29 DEBUG }, 19:36:29 DEBUG "value": [ 19:36:29 DEBUG 1713551789, 19:36:29 DEBUG "3407007744" 19:36:29 DEBUG ] 19:36:29 DEBUG } 19:36:29 DEBUG ] 19:36:29 DEBUG } 19:36:29 DEBUG } 19:36:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551789" 19:36:29 INFO [http_cmd]: http status code OK 19:36:29 DEBUG --- status code --- 19:36:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:29 DEBUG --- http response --- 19:36:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:36:29 DEBUG ----- output ----- 19:36:29 DEBUG { 19:36:29 DEBUG "status": "success", 19:36:29 DEBUG "data": { 19:36:29 DEBUG "resultType": "vector", 19:36:29 DEBUG "result": [ 19:36:29 DEBUG { 19:36:29 DEBUG "metric": { 19:36:29 DEBUG "pod": "ds-idrepo-1" 19:36:29 DEBUG }, 19:36:29 DEBUG "value": [ 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG [...] 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:36:29 DEBUG }, 19:36:29 DEBUG "value": [ 19:36:29 DEBUG 1713551789, 19:36:29 DEBUG "0.000031056575089587194" 19:36:29 DEBUG ] 19:36:29 DEBUG } 19:36:29 DEBUG ] 19:36:29 DEBUG } 19:36:29 DEBUG } 19:36:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551789" 19:36:29 INFO [http_cmd]: http status code OK 19:36:29 DEBUG --- status code --- 19:36:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:29 DEBUG --- http response --- 19:36:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:36:29 DEBUG ----- output ----- 19:36:29 DEBUG { 19:36:29 DEBUG "status": "success", 19:36:29 DEBUG "data": { 19:36:29 DEBUG "resultType": "vector", 19:36:29 DEBUG "result": [ 19:36:29 DEBUG { 19:36:29 DEBUG "metric": { 19:36:29 DEBUG "instance": "10.142.0.107:9100" 19:36:29 DEBUG }, 19:36:29 DEBUG "value": [ 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG [...] 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG 19:36:29 DEBUG "instance": "10.142.0.43:9100" 19:36:29 DEBUG }, 19:36:29 DEBUG "value": [ 19:36:29 DEBUG 1713551789, 19:36:29 DEBUG "70746225.86666667" 19:36:29 DEBUG ] 19:36:29 DEBUG } 19:36:29 DEBUG ] 19:36:29 DEBUG } 19:36:29 DEBUG } 19:36:30 INFO 19:36:30 INFO [loop_until]: kubectl --namespace=xlou top pods 19:36:30 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:36:30 INFO [loop_until]: OK (rc = 0) 19:36:30 DEBUG --- stdout --- 19:36:30 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 14444m 8993Mi am-767f6b67b5-qcqz2 14411m 8997Mi am-767f6b67b5-vbltz 15757m 8987Mi ds-cts-0 15570m 10879Mi ds-cts-1 13916m 11143Mi ds-cts-2 15526m 10885Mi ds-idrepo-0 6321m 3659Mi ds-idrepo-1 6238m 3532Mi ds-idrepo-2 6329m 3493Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 4m 1217Mi lodemon-f8459c597-6tj6f 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7780m 3157Mi 19:36:30 DEBUG --- stderr --- 19:36:30 DEBUG 19:36:30 INFO 19:36:30 INFO [loop_until]: kubectl --namespace=xlou top node 19:36:30 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:36:30 INFO [loop_until]: OK (rc = 0) 19:36:30 DEBUG --- stdout --- 19:36:30 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 67m 0% 6263Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 6503m 21% 7253Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 14336m 48% 11705Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 84m 0% 3341Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 15670m 52% 13893Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 160m 0% 5199Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 14465m 48% 11872Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12443m 41% 2989Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 16452m 55% 11207Mi 10% gke-xlou-cdm-ds-0f601751-j488 7013m 23% 6744Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 14120m 47% 15938Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 6916m 23% 7424Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 15701m 52% 13367Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 7922m 49% 6936Mi 11% 19:36:30 DEBUG --- stderr --- 19:36:30 DEBUG 19:37:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713551849" 19:37:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551849" 19:37:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713551849" 19:37:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551849" 19:37:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713551849" 19:37:29 INFO [http_cmd]: http status code OK 19:37:29 DEBUG --- status code --- 19:37:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:29 DEBUG --- http response --- 19:37:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:37:29 DEBUG ----- output ----- 19:37:29 DEBUG { 19:37:29 DEBUG "status": "success", 19:37:29 DEBUG "data": { 19:37:29 DEBUG "resultType": "vector", 19:37:29 DEBUG "result": [ 19:37:29 DEBUG { 19:37:29 DEBUG "metric": { 19:37:29 DEBUG "instance": "10.142.0.107:9100" 19:37:29 DEBUG }, 19:37:29 DEBUG "value": [ 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG [...] 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG "instance": "10.142.0.43:9100" 19:37:29 DEBUG }, 19:37:29 DEBUG "value": [ 19:37:29 DEBUG 1713551849, 19:37:29 DEBUG "1921908950016" 19:37:29 DEBUG ] 19:37:29 DEBUG } 19:37:29 DEBUG ] 19:37:29 DEBUG } 19:37:29 DEBUG } 19:37:29 INFO [http_cmd]: http status code OK 19:37:29 DEBUG --- status code --- 19:37:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:29 DEBUG --- http response --- 19:37:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:37:29 DEBUG ----- output ----- 19:37:29 DEBUG { 19:37:29 DEBUG "status": "success", 19:37:29 DEBUG "data": { 19:37:29 DEBUG "resultType": "vector", 19:37:29 DEBUG "result": [ 19:37:29 DEBUG { 19:37:29 DEBUG "metric": { 19:37:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:37:29 DEBUG }, 19:37:29 DEBUG "value": [ 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG [...] 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:37:29 DEBUG }, 19:37:29 DEBUG "value": [ 19:37:29 DEBUG 1713551849, 19:37:29 DEBUG "7.732057624094471" 19:37:29 DEBUG ] 19:37:29 DEBUG } 19:37:29 DEBUG ] 19:37:29 DEBUG } 19:37:29 DEBUG } 19:37:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551849" 19:37:29 ERROR [http_cmd]: ERROR 19:37:29 INFO [http_cmd]: http status code OK 19:37:29 ERROR --- status code --- 19:37:29 DEBUG --- status code --- 19:37:29 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:37:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:29 ERROR --- http response --- 19:37:29 DEBUG --- http response --- 19:37:29 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:37:29 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:37:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:37:29 DEBUG ----- output ----- 19:37:29 DEBUG { 19:37:29 DEBUG "status": "success", 19:37:29 DEBUG "data": { 19:37:29 DEBUG "resultType": "vector", 19:37:29 DEBUG "result": [ 19:37:29 DEBUG { 19:37:29 DEBUG "metric": { 19:37:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:37:29 DEBUG }, 19:37:29 DEBUG "value": [ 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG [...] 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:37:29 DEBUG }, 19:37:29 DEBUG "value": [ 19:37:29 DEBUG 1713551849, 19:37:29 DEBUG "0" 19:37:29 DEBUG ] 19:37:29 DEBUG } 19:37:29 DEBUG ] 19:37:29 DEBUG } 19:37:29 DEBUG } 19:37:29 INFO [http_cmd]: http status code OK 19:37:29 DEBUG --- status code --- 19:37:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:29 DEBUG --- http response --- 19:37:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:37:29 DEBUG ----- output ----- 19:37:29 DEBUG { 19:37:29 DEBUG "status": "success", 19:37:29 DEBUG "data": { 19:37:29 DEBUG "resultType": "vector", 19:37:29 DEBUG "result": [ 19:37:29 DEBUG { 19:37:29 DEBUG "metric": { 19:37:29 DEBUG "pod": "ds-idrepo-0" 19:37:29 DEBUG }, 19:37:29 DEBUG "value": [ 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG [...] 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG "pod": "ds-cts-2" 19:37:29 DEBUG }, 19:37:29 DEBUG "value": [ 19:37:29 DEBUG 1713551849, 19:37:29 DEBUG "0" 19:37:29 DEBUG ] 19:37:29 DEBUG } 19:37:29 DEBUG ] 19:37:29 DEBUG } 19:37:29 DEBUG } 19:37:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551849" 19:37:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551849" 19:37:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551849" 19:37:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551849" 19:37:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551849" 19:37:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551849" 19:37:29 INFO [http_cmd]: http status code OK 19:37:29 DEBUG --- status code --- 19:37:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:29 DEBUG --- http response --- 19:37:29 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:37:29 INFO [http_cmd]: http status code OK 19:37:29 DEBUG --- status code --- 19:37:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:29 DEBUG --- http response --- 19:37:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:37:29 DEBUG ----- output ----- 19:37:29 DEBUG { 19:37:29 DEBUG "status": "success", 19:37:29 DEBUG "data": { 19:37:29 DEBUG "resultType": "vector", 19:37:29 DEBUG "result": [ 19:37:29 DEBUG { 19:37:29 DEBUG "metric": { 19:37:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:37:29 DEBUG }, 19:37:29 DEBUG "value": [ 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG [...] 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:37:29 DEBUG }, 19:37:29 DEBUG "value": [ 19:37:29 DEBUG 1713551849, 19:37:29 DEBUG "2335.4666666666667" 19:37:29 DEBUG ] 19:37:29 DEBUG } 19:37:29 DEBUG ] 19:37:29 DEBUG } 19:37:29 DEBUG } 19:37:29 INFO [http_cmd]: http status code OK 19:37:29 DEBUG --- status code --- 19:37:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:29 DEBUG --- http response --- 19:37:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:37:29 DEBUG ----- output ----- 19:37:29 DEBUG { 19:37:29 DEBUG "status": "success", 19:37:29 DEBUG "data": { 19:37:29 DEBUG "resultType": "vector", 19:37:29 DEBUG "result": [ 19:37:29 DEBUG { 19:37:29 DEBUG "metric": { 19:37:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:37:29 DEBUG }, 19:37:29 DEBUG "value": [ 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG [...] 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:37:29 DEBUG }, 19:37:29 DEBUG "value": [ 19:37:29 DEBUG 1713551849, 19:37:29 DEBUG "2333.9333333333334" 19:37:29 DEBUG ] 19:37:29 DEBUG } 19:37:29 DEBUG ] 19:37:29 DEBUG } 19:37:29 DEBUG } 19:37:29 INFO [http_cmd]: http status code OK 19:37:29 DEBUG --- status code --- 19:37:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:29 DEBUG --- http response --- 19:37:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:37:29 DEBUG ----- output ----- 19:37:29 DEBUG { 19:37:29 DEBUG "status": "success", 19:37:29 DEBUG "data": { 19:37:29 DEBUG "resultType": "vector", 19:37:29 DEBUG "result": [ 19:37:29 DEBUG { 19:37:29 DEBUG "metric": { 19:37:29 DEBUG "pod": "ds-idrepo-0" 19:37:29 DEBUG }, 19:37:29 DEBUG "value": [ 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG [...] 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG "pod": "ds-idrepo-2" 19:37:29 DEBUG }, 19:37:29 DEBUG "value": [ 19:37:29 DEBUG 1713551849, 19:37:29 DEBUG "0" 19:37:29 DEBUG ] 19:37:29 DEBUG } 19:37:29 DEBUG ] 19:37:29 DEBUG } 19:37:29 DEBUG } 19:37:29 INFO [http_cmd]: http status code OK 19:37:29 DEBUG --- status code --- 19:37:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:29 DEBUG --- http response --- 19:37:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:37:29 DEBUG ----- output ----- 19:37:29 DEBUG { 19:37:29 DEBUG "status": "success", 19:37:29 DEBUG "data": { 19:37:29 DEBUG "resultType": "vector", 19:37:29 DEBUG "result": [ 19:37:29 DEBUG { 19:37:29 DEBUG "metric": { 19:37:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:37:29 DEBUG }, 19:37:29 DEBUG "value": [ 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG [...] 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:37:29 DEBUG }, 19:37:29 DEBUG "value": [ 19:37:29 DEBUG 1713551849, 19:37:29 DEBUG "420795" 19:37:29 DEBUG ] 19:37:29 DEBUG } 19:37:29 DEBUG ] 19:37:29 DEBUG } 19:37:29 DEBUG } 19:37:29 INFO [http_cmd]: http status code OK 19:37:29 INFO [http_cmd]: http status code OK 19:37:29 DEBUG --- status code --- 19:37:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551849" 19:37:29 DEBUG --- status code --- 19:37:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:29 DEBUG --- http response --- 19:37:29 DEBUG --- http response --- 19:37:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:37:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 19:37:29 DEBUG ----- output ----- 19:37:29 DEBUG ----- output ----- 19:37:29 DEBUG { 19:37:29 DEBUG { 19:37:29 DEBUG "status": "success", 19:37:29 DEBUG "status": "success", 19:37:29 DEBUG "data": { 19:37:29 DEBUG "data": { 19:37:29 DEBUG "resultType": "vector", 19:37:29 DEBUG "resultType": "vector", 19:37:29 DEBUG "result": [ 19:37:29 DEBUG "result": [ 19:37:29 DEBUG { 19:37:29 DEBUG { 19:37:29 DEBUG "metric": { 19:37:29 DEBUG "metric": { 19:37:29 DEBUG "instance": "10.142.0.107:9100" 19:37:29 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:37:29 DEBUG }, 19:37:29 DEBUG }, 19:37:29 DEBUG "value": [ 19:37:29 DEBUG "value": [ 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG [...] 19:37:29 DEBUG [...] 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG "instance": "10.142.0.43:9100" 19:37:29 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:37:29 DEBUG }, 19:37:29 DEBUG }, 19:37:29 DEBUG "value": [ 19:37:29 DEBUG "value": [ 19:37:29 DEBUG 1713551849, 19:37:29 DEBUG 1713551849, 19:37:29 DEBUG "0.19410000000005045" 19:37:29 DEBUG "0" 19:37:29 DEBUG ] 19:37:29 DEBUG ] 19:37:29 DEBUG } 19:37:29 DEBUG } 19:37:29 DEBUG ] 19:37:29 DEBUG ] 19:37:29 DEBUG } 19:37:29 DEBUG } 19:37:29 DEBUG } 19:37:29 DEBUG } 19:37:29 INFO [http_cmd]: http status code OK 19:37:29 DEBUG --- status code --- 19:37:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:29 DEBUG --- http response --- 19:37:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:37:29 DEBUG ----- output ----- 19:37:29 DEBUG { 19:37:29 DEBUG "status": "success", 19:37:29 DEBUG "data": { 19:37:29 DEBUG "resultType": "vector", 19:37:29 DEBUG "result": [ 19:37:29 DEBUG { 19:37:29 DEBUG "metric": { 19:37:29 DEBUG "pod": "ds-idrepo-0" 19:37:29 DEBUG }, 19:37:29 DEBUG "value": [ 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG [...] 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG "pod": "ds-idrepo-2" 19:37:29 DEBUG }, 19:37:29 DEBUG "value": [ 19:37:29 DEBUG 1713551849, 19:37:29 DEBUG "0" 19:37:29 DEBUG ] 19:37:29 DEBUG } 19:37:29 DEBUG ] 19:37:29 DEBUG } 19:37:29 DEBUG } 19:37:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551849" 19:37:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551849" 19:37:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551849" 19:37:29 INFO [http_cmd]: http status code OK 19:37:29 DEBUG --- status code --- 19:37:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:29 DEBUG --- http response --- 19:37:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:37:29 DEBUG ----- output ----- 19:37:29 DEBUG { 19:37:29 DEBUG "status": "success", 19:37:29 DEBUG "data": { 19:37:29 DEBUG "resultType": "vector", 19:37:29 DEBUG "result": [ 19:37:29 DEBUG { 19:37:29 DEBUG "metric": { 19:37:29 DEBUG "pod": "ds-cts-0" 19:37:29 DEBUG }, 19:37:29 DEBUG "value": [ 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG [...] 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG "pod": "ds-cts-2" 19:37:29 DEBUG }, 19:37:29 DEBUG "value": [ 19:37:29 DEBUG 1713551849, 19:37:29 DEBUG "0" 19:37:29 DEBUG ] 19:37:29 DEBUG } 19:37:29 DEBUG ] 19:37:29 DEBUG } 19:37:29 DEBUG } 19:37:29 INFO [http_cmd]: http status code OK 19:37:29 DEBUG --- status code --- 19:37:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:29 DEBUG --- http response --- 19:37:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:37:29 DEBUG ----- output ----- 19:37:29 DEBUG { 19:37:29 DEBUG "status": "success", 19:37:29 DEBUG "data": { 19:37:29 DEBUG "resultType": "vector", 19:37:29 DEBUG "result": [ 19:37:29 DEBUG { 19:37:29 DEBUG "metric": { 19:37:29 DEBUG "instance": "10.142.0.107:9100" 19:37:29 DEBUG }, 19:37:29 DEBUG "value": [ 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG [...] 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG "instance": "10.142.0.43:9100" 19:37:29 DEBUG }, 19:37:29 DEBUG "value": [ 19:37:29 DEBUG 1713551849, 19:37:29 DEBUG "70530747.33333334" 19:37:29 DEBUG ] 19:37:29 DEBUG } 19:37:29 DEBUG ] 19:37:29 DEBUG } 19:37:29 DEBUG } 19:37:29 INFO [http_cmd]: http status code OK 19:37:29 DEBUG --- status code --- 19:37:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:29 DEBUG --- http response --- 19:37:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 19:37:29 DEBUG ----- output ----- 19:37:29 DEBUG { 19:37:29 DEBUG "status": "success", 19:37:29 DEBUG "data": { 19:37:29 DEBUG "resultType": "vector", 19:37:29 DEBUG "result": [ 19:37:29 DEBUG { 19:37:29 DEBUG "metric": { 19:37:29 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:37:29 DEBUG }, 19:37:29 DEBUG "value": [ 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG [...] 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:37:29 DEBUG }, 19:37:29 DEBUG "value": [ 19:37:29 DEBUG 1713551849, 19:37:29 DEBUG "0" 19:37:29 DEBUG ] 19:37:29 DEBUG } 19:37:29 DEBUG ] 19:37:29 DEBUG } 19:37:29 DEBUG } 19:37:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551849" 19:37:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551849" 19:37:29 INFO [http_cmd]: http status code OK 19:37:29 DEBUG --- status code --- 19:37:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:29 DEBUG --- http response --- 19:37:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:37:29 DEBUG ----- output ----- 19:37:29 DEBUG { 19:37:29 DEBUG "status": "success", 19:37:29 DEBUG "data": { 19:37:29 DEBUG "resultType": "vector", 19:37:29 DEBUG "result": [ 19:37:29 DEBUG { 19:37:29 DEBUG "metric": { 19:37:29 DEBUG "pod": "ds-idrepo-0" 19:37:29 DEBUG }, 19:37:29 DEBUG "value": [ 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG [...] 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG "pod": "ds-idrepo-2" 19:37:29 DEBUG }, 19:37:29 DEBUG "value": [ 19:37:29 DEBUG 1713551849, 19:37:29 DEBUG "0" 19:37:29 DEBUG ] 19:37:29 DEBUG } 19:37:29 DEBUG ] 19:37:29 DEBUG } 19:37:29 DEBUG } 19:37:29 INFO [http_cmd]: http status code OK 19:37:29 DEBUG --- status code --- 19:37:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:29 DEBUG --- http response --- 19:37:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:37:29 DEBUG ----- output ----- 19:37:29 DEBUG { 19:37:29 DEBUG "status": "success", 19:37:29 DEBUG "data": { 19:37:29 DEBUG "resultType": "vector", 19:37:29 DEBUG "result": [ 19:37:29 DEBUG { 19:37:29 DEBUG "metric": { 19:37:29 DEBUG "instance": "10.142.0.107:9100" 19:37:29 DEBUG }, 19:37:29 DEBUG "value": [ 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG [...] 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG "instance": "10.142.0.43:9100" 19:37:29 DEBUG }, 19:37:29 DEBUG "value": [ 19:37:29 DEBUG 1713551849, 19:37:29 DEBUG "0.09200000000000157" 19:37:29 DEBUG ] 19:37:29 DEBUG } 19:37:29 DEBUG ] 19:37:29 DEBUG } 19:37:29 DEBUG } 19:37:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551849" 19:37:29 INFO [http_cmd]: http status code OK 19:37:29 DEBUG --- status code --- 19:37:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:29 DEBUG --- http response --- 19:37:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:37:29 DEBUG ----- output ----- 19:37:29 DEBUG { 19:37:29 DEBUG "status": "success", 19:37:29 DEBUG "data": { 19:37:29 DEBUG "resultType": "vector", 19:37:29 DEBUG "result": [ 19:37:29 DEBUG { 19:37:29 DEBUG "metric": { 19:37:29 DEBUG "pod": "ds-cts-0" 19:37:29 DEBUG }, 19:37:29 DEBUG "value": [ 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG [...] 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG "pod": "ds-cts-2" 19:37:29 DEBUG }, 19:37:29 DEBUG "value": [ 19:37:29 DEBUG 1713551849, 19:37:29 DEBUG "30" 19:37:29 DEBUG ] 19:37:29 DEBUG } 19:37:29 DEBUG ] 19:37:29 DEBUG } 19:37:29 DEBUG } 19:37:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551849" 19:37:29 INFO [http_cmd]: http status code OK 19:37:29 DEBUG --- status code --- 19:37:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:29 DEBUG --- http response --- 19:37:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:37:29 DEBUG ----- output ----- 19:37:29 DEBUG { 19:37:29 DEBUG "status": "success", 19:37:29 DEBUG "data": { 19:37:29 DEBUG "resultType": "vector", 19:37:29 DEBUG "result": [ 19:37:29 DEBUG { 19:37:29 DEBUG "metric": { 19:37:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:37:29 DEBUG }, 19:37:29 DEBUG "value": [ 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG [...] 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:37:29 DEBUG }, 19:37:29 DEBUG "value": [ 19:37:29 DEBUG 1713551849, 19:37:29 DEBUG "2333.8" 19:37:29 DEBUG ] 19:37:29 DEBUG } 19:37:29 DEBUG ] 19:37:29 DEBUG } 19:37:29 DEBUG } 19:37:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551849" 19:37:29 INFO [http_cmd]: http status code OK 19:37:29 DEBUG --- status code --- 19:37:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:29 DEBUG --- http response --- 19:37:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:37:29 DEBUG ----- output ----- 19:37:29 DEBUG { 19:37:29 DEBUG "status": "success", 19:37:29 DEBUG "data": { 19:37:29 DEBUG "resultType": "vector", 19:37:29 DEBUG "result": [ 19:37:29 DEBUG { 19:37:29 DEBUG "metric": { 19:37:29 DEBUG "pod": "ds-cts-0" 19:37:29 DEBUG }, 19:37:29 DEBUG "value": [ 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG [...] 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG "pod": "ds-cts-2" 19:37:29 DEBUG }, 19:37:29 DEBUG "value": [ 19:37:29 DEBUG 1713551849, 19:37:29 DEBUG "0.00007309777777777778" 19:37:29 DEBUG ] 19:37:29 DEBUG } 19:37:29 DEBUG ] 19:37:29 DEBUG } 19:37:29 DEBUG } 19:37:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713551849" 19:37:29 INFO [http_cmd]: http status code OK 19:37:29 DEBUG --- status code --- 19:37:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:29 DEBUG --- http response --- 19:37:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:37:29 DEBUG ----- output ----- 19:37:29 DEBUG { 19:37:29 DEBUG "status": "success", 19:37:29 DEBUG "data": { 19:37:29 DEBUG "resultType": "vector", 19:37:29 DEBUG "result": [ 19:37:29 DEBUG { 19:37:29 DEBUG "metric": { 19:37:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:37:29 DEBUG }, 19:37:29 DEBUG "value": [ 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG [...] 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:37:29 DEBUG }, 19:37:29 DEBUG "value": [ 19:37:29 DEBUG 1713551849, 19:37:29 DEBUG "0.01" 19:37:29 DEBUG ] 19:37:29 DEBUG } 19:37:29 DEBUG ] 19:37:29 DEBUG } 19:37:29 DEBUG } 19:37:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551849" 19:37:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551849" 19:37:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713551849" 19:37:29 INFO [http_cmd]: http status code OK 19:37:29 INFO [http_cmd]: http status code OK 19:37:29 DEBUG --- status code --- 19:37:29 DEBUG --- status code --- 19:37:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:29 DEBUG --- http response --- 19:37:29 DEBUG --- http response --- 19:37:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:37:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 19:37:29 DEBUG ----- output ----- 19:37:29 DEBUG ----- output ----- 19:37:29 DEBUG { 19:37:29 DEBUG { 19:37:29 DEBUG "status": "success", 19:37:29 DEBUG "status": "success", 19:37:29 DEBUG "data": { 19:37:29 DEBUG "data": { 19:37:29 DEBUG "resultType": "vector", 19:37:29 DEBUG "resultType": "vector", 19:37:29 DEBUG "result": [ 19:37:29 DEBUG "result": [ 19:37:29 DEBUG { 19:37:29 DEBUG { 19:37:29 DEBUG "metric": { 19:37:29 DEBUG "metric": { 19:37:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:37:29 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:37:29 DEBUG }, 19:37:29 DEBUG }, 19:37:29 DEBUG "value": [ 19:37:29 DEBUG "value": [ 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG [...] 19:37:29 DEBUG [...] 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:37:29 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:37:29 DEBUG }, 19:37:29 DEBUG }, 19:37:29 DEBUG "value": [ 19:37:29 DEBUG "value": [ 19:37:29 DEBUG 1713551849, 19:37:29 DEBUG 1713551849, 19:37:29 DEBUG "2334.1666666666665" 19:37:29 DEBUG "0" 19:37:29 DEBUG ] 19:37:29 DEBUG ] 19:37:29 DEBUG } 19:37:29 DEBUG } 19:37:29 DEBUG ] 19:37:29 DEBUG ] 19:37:29 DEBUG } 19:37:29 DEBUG } 19:37:29 DEBUG } 19:37:29 DEBUG } 19:37:29 INFO [http_cmd]: http status code OK 19:37:29 DEBUG --- status code --- 19:37:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:29 DEBUG --- http response --- 19:37:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:37:29 DEBUG ----- output ----- 19:37:29 DEBUG { 19:37:29 DEBUG "status": "success", 19:37:29 DEBUG "data": { 19:37:29 DEBUG "resultType": "vector", 19:37:29 DEBUG "result": [ 19:37:29 DEBUG { 19:37:29 DEBUG "metric": { 19:37:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:37:29 DEBUG }, 19:37:29 DEBUG "value": [ 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG [...] 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:37:29 DEBUG }, 19:37:29 DEBUG "value": [ 19:37:29 DEBUG 1713551849, 19:37:29 DEBUG "3409444864" 19:37:29 DEBUG ] 19:37:29 DEBUG } 19:37:29 DEBUG ] 19:37:29 DEBUG } 19:37:29 DEBUG } 19:37:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551849" 19:37:29 INFO [http_cmd]: http status code OK 19:37:29 DEBUG --- status code --- 19:37:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:29 DEBUG --- http response --- 19:37:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 19:37:29 DEBUG ----- output ----- 19:37:29 DEBUG { 19:37:29 DEBUG "status": "success", 19:37:29 DEBUG "data": { 19:37:29 DEBUG "resultType": "vector", 19:37:29 DEBUG "result": [ 19:37:29 DEBUG { 19:37:29 DEBUG "metric": { 19:37:29 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:37:29 DEBUG }, 19:37:29 DEBUG "value": [ 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG [...] 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG 19:37:29 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:37:29 DEBUG }, 19:37:29 DEBUG "value": [ 19:37:29 DEBUG 1713551849, 19:37:29 DEBUG "0.00002390889264731639" 19:37:29 DEBUG ] 19:37:29 DEBUG } 19:37:29 DEBUG ] 19:37:29 DEBUG } 19:37:29 DEBUG } 19:37:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551850" 19:37:30 INFO [http_cmd]: http status code OK 19:37:30 DEBUG --- status code --- 19:37:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:30 DEBUG --- http response --- 19:37:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:37:30 DEBUG ----- output ----- 19:37:30 DEBUG { 19:37:30 DEBUG "status": "success", 19:37:30 DEBUG "data": { 19:37:30 DEBUG "resultType": "vector", 19:37:30 DEBUG "result": [ 19:37:30 DEBUG { 19:37:30 DEBUG "metric": { 19:37:30 DEBUG "instance": "10.142.0.107:9100" 19:37:30 DEBUG }, 19:37:30 DEBUG "value": [ 19:37:30 DEBUG 19:37:30 DEBUG 19:37:30 DEBUG 19:37:30 DEBUG [...] 19:37:30 DEBUG 19:37:30 DEBUG 19:37:30 DEBUG 19:37:30 DEBUG "instance": "10.142.0.43:9100" 19:37:30 DEBUG }, 19:37:30 DEBUG "value": [ 19:37:30 DEBUG 1713551850, 19:37:30 DEBUG "70526481.73333333" 19:37:30 DEBUG ] 19:37:30 DEBUG } 19:37:30 DEBUG ] 19:37:30 DEBUG } 19:37:30 DEBUG } 19:37:30 INFO 19:37:30 INFO [loop_until]: kubectl --namespace=xlou top pods 19:37:30 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:37:30 INFO [loop_until]: OK (rc = 0) 19:37:30 DEBUG --- stdout --- 19:37:30 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 14522m 8994Mi am-767f6b67b5-qcqz2 14321m 8998Mi am-767f6b67b5-vbltz 15633m 8981Mi ds-cts-0 15056m 10884Mi ds-cts-1 13921m 11145Mi ds-cts-2 15818m 10889Mi ds-idrepo-0 6386m 3659Mi ds-idrepo-1 6378m 3534Mi ds-idrepo-2 6362m 3493Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 4m 1217Mi lodemon-f8459c597-6tj6f 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7814m 3159Mi 19:37:30 DEBUG --- stderr --- 19:37:30 DEBUG 19:37:30 INFO 19:37:30 INFO [loop_until]: kubectl --namespace=xlou top node 19:37:30 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:37:31 INFO [loop_until]: OK (rc = 0) 19:37:31 DEBUG --- stdout --- 19:37:31 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 71m 0% 6247Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 6487m 21% 7247Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 14314m 47% 11720Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 84m 0% 3332Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 15451m 51% 13913Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 159m 0% 5200Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 14395m 48% 11886Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12317m 41% 2995Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 16418m 54% 11191Mi 10% gke-xlou-cdm-ds-0f601751-j488 6749m 22% 6736Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 13923m 46% 15948Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 6961m 23% 7429Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 15974m 53% 13362Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 7867m 49% 6940Mi 11% 19:37:31 DEBUG --- stderr --- 19:37:31 DEBUG 19:38:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713551909" 19:38:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551909" 19:38:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551909" 19:38:29 ERROR [http_cmd]: ERROR 19:38:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551909" 19:38:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551909" 19:38:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551909" 19:38:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713551909" 19:38:29 INFO [http_cmd]: http status code OK 19:38:29 ERROR --- status code --- 19:38:29 DEBUG --- status code --- 19:38:29 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:38:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:29 ERROR --- http response --- 19:38:29 DEBUG --- http response --- 19:38:29 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:38:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:38:29 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:38:29 DEBUG ----- output ----- 19:38:29 DEBUG { 19:38:29 DEBUG "status": "success", 19:38:29 DEBUG "data": { 19:38:29 DEBUG "resultType": "vector", 19:38:29 DEBUG "result": [ 19:38:29 DEBUG { 19:38:29 DEBUG "metric": { 19:38:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:38:29 DEBUG }, 19:38:29 DEBUG "value": [ 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG [...] 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:38:29 DEBUG }, 19:38:29 DEBUG "value": [ 19:38:29 DEBUG 1713551909, 19:38:29 DEBUG "420489.5" 19:38:29 DEBUG ] 19:38:29 DEBUG } 19:38:29 DEBUG ] 19:38:29 DEBUG } 19:38:29 DEBUG } 19:38:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551909" 19:38:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713551909" 19:38:29 INFO [http_cmd]: http status code OK 19:38:29 DEBUG --- status code --- 19:38:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:29 DEBUG --- http response --- 19:38:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:38:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551909" 19:38:29 DEBUG ----- output ----- 19:38:29 DEBUG { 19:38:29 DEBUG "status": "success", 19:38:29 DEBUG "data": { 19:38:29 DEBUG "resultType": "vector", 19:38:29 DEBUG "result": [ 19:38:29 DEBUG { 19:38:29 DEBUG "metric": { 19:38:29 DEBUG "pod": "ds-idrepo-0" 19:38:29 DEBUG }, 19:38:29 DEBUG "value": [ 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG [...] 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG "pod": "ds-idrepo-2" 19:38:29 DEBUG }, 19:38:29 DEBUG "value": [ 19:38:29 DEBUG 1713551909, 19:38:29 DEBUG "0" 19:38:29 DEBUG ] 19:38:29 DEBUG } 19:38:29 DEBUG ] 19:38:29 DEBUG } 19:38:29 DEBUG } 19:38:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551909" 19:38:29 INFO [http_cmd]: http status code OK 19:38:29 DEBUG --- status code --- 19:38:29 INFO [http_cmd]: http status code OK 19:38:29 INFO [http_cmd]: http status code OK 19:38:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551909" 19:38:29 INFO [http_cmd]: http status code OK 19:38:29 INFO [http_cmd]: http status code OK 19:38:29 DEBUG --- status code --- 19:38:29 INFO [http_cmd]: http status code OK 19:38:29 DEBUG --- status code --- 19:38:29 DEBUG --- http response --- 19:38:29 DEBUG --- status code --- 19:38:29 DEBUG --- status code --- 19:38:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:29 DEBUG --- status code --- 19:38:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:38:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:29 DEBUG --- http response --- 19:38:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:29 DEBUG --- http response --- 19:38:29 DEBUG --- http response --- 19:38:29 DEBUG ----- output ----- 19:38:29 DEBUG --- http response --- 19:38:29 DEBUG --- http response --- 19:38:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:38:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:38:29 DEBUG { 19:38:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:38:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:38:29 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:38:29 DEBUG ----- output ----- 19:38:29 DEBUG ----- output ----- 19:38:29 DEBUG "status": "success", 19:38:29 INFO [http_cmd]: http status code OK 19:38:29 DEBUG ----- output ----- 19:38:29 DEBUG ----- output ----- 19:38:29 DEBUG { 19:38:29 DEBUG { 19:38:29 DEBUG "data": { 19:38:29 DEBUG --- status code --- 19:38:29 DEBUG { 19:38:29 DEBUG { 19:38:29 DEBUG "status": "success", 19:38:29 DEBUG "status": "success", 19:38:29 DEBUG "resultType": "vector", 19:38:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:29 DEBUG "status": "success", 19:38:29 DEBUG "status": "success", 19:38:29 DEBUG "data": { 19:38:29 DEBUG "data": { 19:38:29 DEBUG "result": [ 19:38:29 DEBUG --- http response --- 19:38:29 DEBUG "data": { 19:38:29 DEBUG "data": { 19:38:29 DEBUG "resultType": "vector", 19:38:29 DEBUG "resultType": "vector", 19:38:29 DEBUG { 19:38:29 DEBUG "resultType": "vector", 19:38:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:38:29 DEBUG "resultType": "vector", 19:38:29 DEBUG "result": [ 19:38:29 INFO [http_cmd]: http status code OK 19:38:29 DEBUG "result": [ 19:38:29 DEBUG "metric": { 19:38:29 DEBUG "result": [ 19:38:29 DEBUG ----- output ----- 19:38:29 DEBUG "result": [ 19:38:29 DEBUG { 19:38:29 DEBUG --- status code --- 19:38:29 DEBUG { 19:38:29 DEBUG "pod": "ds-idrepo-0" 19:38:29 DEBUG { 19:38:29 DEBUG { 19:38:29 DEBUG { 19:38:29 DEBUG "metric": { 19:38:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:29 DEBUG "metric": { 19:38:29 DEBUG }, 19:38:29 DEBUG "metric": { 19:38:29 DEBUG "status": "success", 19:38:29 DEBUG "metric": { 19:38:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:38:29 DEBUG --- http response --- 19:38:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:38:29 DEBUG "value": [ 19:38:29 DEBUG "instance": "10.142.0.107:9100" 19:38:29 DEBUG "data": { 19:38:29 DEBUG "pod": "ds-idrepo-0" 19:38:29 DEBUG }, 19:38:29 DEBUG }, 19:38:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:38:29 DEBUG 19:38:29 DEBUG }, 19:38:29 DEBUG "resultType": "vector", 19:38:29 DEBUG }, 19:38:29 DEBUG "value": [ 19:38:29 DEBUG "value": [ 19:38:29 DEBUG ----- output ----- 19:38:29 DEBUG 19:38:29 DEBUG "value": [ 19:38:29 DEBUG "result": [ 19:38:29 DEBUG "value": [ 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG { 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG { 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG "status": "success", 19:38:29 DEBUG [...] 19:38:29 DEBUG 19:38:29 DEBUG "metric": { 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG "data": { 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:38:29 DEBUG 19:38:29 DEBUG [...] 19:38:29 DEBUG [...] 19:38:29 DEBUG "resultType": "vector", 19:38:29 DEBUG 19:38:29 DEBUG [...] 19:38:29 DEBUG }, 19:38:29 DEBUG [...] 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG "result": [ 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG "value": [ 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG { 19:38:29 DEBUG "pod": "ds-idrepo-2" 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG "metric": { 19:38:29 DEBUG }, 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:38:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:38:29 DEBUG "pod": "ds-idrepo-0" 19:38:29 DEBUG "value": [ 19:38:29 DEBUG "instance": "10.142.0.43:9100" 19:38:29 DEBUG 19:38:29 DEBUG "pod": "ds-cts-2" 19:38:29 INFO [http_cmd]: http status code OK 19:38:29 DEBUG }, 19:38:29 DEBUG }, 19:38:29 DEBUG }, 19:38:29 DEBUG 1713551909, 19:38:29 DEBUG }, 19:38:29 DEBUG [...] 19:38:29 DEBUG }, 19:38:29 DEBUG --- status code --- 19:38:29 DEBUG "value": [ 19:38:29 DEBUG "value": [ 19:38:29 DEBUG "value": [ 19:38:29 DEBUG "0" 19:38:29 DEBUG "value": [ 19:38:29 DEBUG 19:38:29 DEBUG "value": [ 19:38:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:29 DEBUG 1713551909, 19:38:29 DEBUG 1713551909, 19:38:29 DEBUG 19:38:29 DEBUG ] 19:38:29 DEBUG 1713551909, 19:38:29 DEBUG 19:38:29 DEBUG 1713551909, 19:38:29 DEBUG --- http response --- 19:38:29 DEBUG "0" 19:38:29 DEBUG "7.677150050405941" 19:38:29 DEBUG 19:38:29 DEBUG } 19:38:29 DEBUG "1925415107584" 19:38:29 DEBUG 19:38:29 DEBUG "0" 19:38:29 DEBUG ] 19:38:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:38:29 DEBUG ] 19:38:29 DEBUG 19:38:29 DEBUG ] 19:38:29 DEBUG ] 19:38:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:38:29 DEBUG ] 19:38:29 DEBUG } 19:38:29 DEBUG ----- output ----- 19:38:29 DEBUG } 19:38:29 DEBUG [...] 19:38:29 DEBUG } 19:38:29 DEBUG } 19:38:29 DEBUG }, 19:38:29 DEBUG } 19:38:29 DEBUG ] 19:38:29 DEBUG { 19:38:29 DEBUG ] 19:38:29 DEBUG 19:38:29 DEBUG } 19:38:29 DEBUG ] 19:38:29 DEBUG "value": [ 19:38:29 DEBUG ] 19:38:29 DEBUG } 19:38:29 DEBUG "status": "success", 19:38:29 DEBUG } 19:38:29 DEBUG 19:38:29 DEBUG } 19:38:29 DEBUG 1713551909, 19:38:29 DEBUG } 19:38:29 DEBUG } 19:38:29 DEBUG "data": { 19:38:29 DEBUG } 19:38:29 DEBUG 19:38:29 DEBUG } 19:38:29 DEBUG "2336.6" 19:38:29 DEBUG } 19:38:29 DEBUG "resultType": "vector", 19:38:29 DEBUG "result": [ 19:38:29 DEBUG ] 19:38:29 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:38:29 DEBUG { 19:38:29 DEBUG } 19:38:29 DEBUG }, 19:38:29 DEBUG "metric": { 19:38:29 DEBUG ] 19:38:29 DEBUG "value": [ 19:38:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:38:29 DEBUG } 19:38:29 DEBUG 1713551909, 19:38:29 DEBUG }, 19:38:29 DEBUG } 19:38:29 DEBUG "0" 19:38:29 DEBUG "value": [ 19:38:29 DEBUG ] 19:38:29 DEBUG 19:38:29 DEBUG } 19:38:29 DEBUG 19:38:29 DEBUG ] 19:38:29 DEBUG 19:38:29 DEBUG } 19:38:29 DEBUG [...] 19:38:29 DEBUG } 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:38:29 DEBUG }, 19:38:29 DEBUG "value": [ 19:38:29 DEBUG 1713551909, 19:38:29 DEBUG "2332.8" 19:38:29 DEBUG ] 19:38:29 DEBUG } 19:38:29 DEBUG ] 19:38:29 DEBUG } 19:38:29 DEBUG } 19:38:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551909" 19:38:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551909" 19:38:29 INFO [http_cmd]: http status code OK 19:38:29 INFO [http_cmd]: http status code OK 19:38:29 DEBUG --- status code --- 19:38:29 DEBUG --- status code --- 19:38:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:29 DEBUG --- http response --- 19:38:29 DEBUG --- http response --- 19:38:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:38:29 DEBUG ----- output ----- 19:38:29 DEBUG { 19:38:29 DEBUG "status": "success", 19:38:29 DEBUG "data": { 19:38:29 DEBUG "resultType": "vector", 19:38:29 DEBUG "result": [ 19:38:29 DEBUG { 19:38:29 DEBUG "metric": { 19:38:29 DEBUG "pod": "ds-cts-0" 19:38:29 DEBUG }, 19:38:29 DEBUG "value": [ 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG [...] 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG "pod": "ds-cts-2" 19:38:29 DEBUG }, 19:38:29 DEBUG "value": [ 19:38:29 DEBUG 1713551909, 19:38:29 DEBUG "0" 19:38:29 DEBUG ] 19:38:29 DEBUG } 19:38:29 DEBUG ] 19:38:29 DEBUG } 19:38:29 DEBUG } 19:38:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:38:29 DEBUG ----- output ----- 19:38:29 DEBUG { 19:38:29 DEBUG "status": "success", 19:38:29 DEBUG "data": { 19:38:29 DEBUG "resultType": "vector", 19:38:29 DEBUG "result": [ 19:38:29 DEBUG { 19:38:29 DEBUG "metric": { 19:38:29 DEBUG "instance": "10.142.0.107:9100" 19:38:29 DEBUG }, 19:38:29 DEBUG "value": [ 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG [...] 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG "instance": "10.142.0.43:9100" 19:38:29 DEBUG }, 19:38:29 DEBUG "value": [ 19:38:29 DEBUG 1713551909, 19:38:29 DEBUG "0.16596666666664436" 19:38:29 DEBUG ] 19:38:29 DEBUG } 19:38:29 DEBUG ] 19:38:29 DEBUG } 19:38:29 DEBUG } 19:38:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551909" 19:38:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551909" 19:38:29 INFO [http_cmd]: http status code OK 19:38:29 DEBUG --- status code --- 19:38:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:29 DEBUG --- http response --- 19:38:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:38:29 DEBUG ----- output ----- 19:38:29 DEBUG { 19:38:29 DEBUG "status": "success", 19:38:29 DEBUG "data": { 19:38:29 DEBUG "resultType": "vector", 19:38:29 DEBUG "result": [ 19:38:29 DEBUG { 19:38:29 DEBUG "metric": { 19:38:29 DEBUG "instance": "10.142.0.107:9100" 19:38:29 DEBUG }, 19:38:29 DEBUG "value": [ 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG [...] 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG "instance": "10.142.0.43:9100" 19:38:29 DEBUG }, 19:38:29 DEBUG "value": [ 19:38:29 DEBUG 1713551909, 19:38:29 DEBUG "70523328.13333333" 19:38:29 DEBUG ] 19:38:29 DEBUG } 19:38:29 DEBUG ] 19:38:29 DEBUG } 19:38:29 DEBUG } 19:38:29 INFO [http_cmd]: http status code OK 19:38:29 DEBUG --- status code --- 19:38:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:29 DEBUG --- http response --- 19:38:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:38:29 DEBUG ----- output ----- 19:38:29 DEBUG { 19:38:29 DEBUG "status": "success", 19:38:29 DEBUG "data": { 19:38:29 DEBUG "resultType": "vector", 19:38:29 DEBUG "result": [ 19:38:29 DEBUG { 19:38:29 DEBUG "metric": { 19:38:29 DEBUG "pod": "ds-idrepo-0" 19:38:29 DEBUG }, 19:38:29 DEBUG "value": [ 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG [...] 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:38:29 DEBUG }, 19:38:29 DEBUG "value": [ 19:38:29 DEBUG 1713551909, 19:38:29 DEBUG "12016.644425237684" 19:38:29 DEBUG ] 19:38:29 DEBUG } 19:38:29 DEBUG ] 19:38:29 DEBUG } 19:38:29 DEBUG } 19:38:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551909" 19:38:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551909" 19:38:29 INFO [http_cmd]: http status code OK 19:38:29 DEBUG --- status code --- 19:38:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:29 DEBUG --- http response --- 19:38:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:38:29 DEBUG ----- output ----- 19:38:29 DEBUG { 19:38:29 DEBUG "status": "success", 19:38:29 DEBUG "data": { 19:38:29 DEBUG "resultType": "vector", 19:38:29 DEBUG "result": [ 19:38:29 DEBUG { 19:38:29 DEBUG "metric": { 19:38:29 DEBUG "pod": "ds-idrepo-0" 19:38:29 DEBUG }, 19:38:29 DEBUG "value": [ 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG [...] 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG "pod": "ds-idrepo-2" 19:38:29 DEBUG }, 19:38:29 DEBUG "value": [ 19:38:29 DEBUG 1713551909, 19:38:29 DEBUG "0" 19:38:29 DEBUG ] 19:38:29 DEBUG } 19:38:29 DEBUG ] 19:38:29 DEBUG } 19:38:29 DEBUG } 19:38:29 INFO [http_cmd]: http status code OK 19:38:29 DEBUG --- status code --- 19:38:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:29 DEBUG --- http response --- 19:38:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:38:29 DEBUG ----- output ----- 19:38:29 DEBUG { 19:38:29 DEBUG "status": "success", 19:38:29 DEBUG "data": { 19:38:29 DEBUG "resultType": "vector", 19:38:29 DEBUG "result": [ 19:38:29 DEBUG { 19:38:29 DEBUG "metric": { 19:38:29 DEBUG "instance": "10.142.0.107:9100" 19:38:29 DEBUG }, 19:38:29 DEBUG "value": [ 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG [...] 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG "instance": "10.142.0.43:9100" 19:38:29 DEBUG }, 19:38:29 DEBUG "value": [ 19:38:29 DEBUG 1713551909, 19:38:29 DEBUG "0.07200000000001032" 19:38:29 DEBUG ] 19:38:29 DEBUG } 19:38:29 DEBUG ] 19:38:29 DEBUG } 19:38:29 DEBUG } 19:38:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551909" 19:38:29 INFO [http_cmd]: http status code OK 19:38:29 DEBUG --- status code --- 19:38:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:29 DEBUG --- http response --- 19:38:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:38:29 DEBUG ----- output ----- 19:38:29 DEBUG { 19:38:29 DEBUG "status": "success", 19:38:29 DEBUG "data": { 19:38:29 DEBUG "resultType": "vector", 19:38:29 DEBUG "result": [ 19:38:29 DEBUG { 19:38:29 DEBUG "metric": { 19:38:29 DEBUG "pod": "ds-cts-0" 19:38:29 DEBUG }, 19:38:29 DEBUG "value": [ 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG [...] 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG "pod": "ds-cts-2" 19:38:29 DEBUG }, 19:38:29 DEBUG "value": [ 19:38:29 DEBUG 1713551909, 19:38:29 DEBUG "29.933333333333334" 19:38:29 DEBUG ] 19:38:29 DEBUG } 19:38:29 DEBUG ] 19:38:29 DEBUG } 19:38:29 DEBUG } 19:38:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551909" 19:38:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551909" 19:38:29 INFO [http_cmd]: http status code OK 19:38:29 DEBUG --- status code --- 19:38:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:29 DEBUG --- http response --- 19:38:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:38:29 DEBUG ----- output ----- 19:38:29 DEBUG { 19:38:29 DEBUG "status": "success", 19:38:29 DEBUG "data": { 19:38:29 DEBUG "resultType": "vector", 19:38:29 DEBUG "result": [ 19:38:29 DEBUG { 19:38:29 DEBUG "metric": { 19:38:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:38:29 DEBUG }, 19:38:29 DEBUG "value": [ 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG [...] 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:38:29 DEBUG }, 19:38:29 DEBUG "value": [ 19:38:29 DEBUG 1713551909, 19:38:29 DEBUG "2332.3333333333335" 19:38:29 DEBUG ] 19:38:29 DEBUG } 19:38:29 DEBUG ] 19:38:29 DEBUG } 19:38:29 DEBUG } 19:38:29 INFO [http_cmd]: http status code OK 19:38:29 DEBUG --- status code --- 19:38:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:29 DEBUG --- http response --- 19:38:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:38:29 DEBUG ----- output ----- 19:38:29 DEBUG { 19:38:29 DEBUG "status": "success", 19:38:29 DEBUG "data": { 19:38:29 DEBUG "resultType": "vector", 19:38:29 DEBUG "result": [ 19:38:29 DEBUG { 19:38:29 DEBUG "metric": { 19:38:29 DEBUG "pod": "ds-cts-0" 19:38:29 DEBUG }, 19:38:29 DEBUG "value": [ 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG [...] 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG "pod": "ds-cts-2" 19:38:29 DEBUG }, 19:38:29 DEBUG "value": [ 19:38:29 DEBUG 1713551909, 19:38:29 DEBUG "0.000033333333333333335" 19:38:29 DEBUG ] 19:38:29 DEBUG } 19:38:29 DEBUG ] 19:38:29 DEBUG } 19:38:29 DEBUG } 19:38:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713551909" 19:38:29 INFO [http_cmd]: http status code OK 19:38:29 DEBUG --- status code --- 19:38:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:29 DEBUG --- http response --- 19:38:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:38:29 DEBUG ----- output ----- 19:38:29 DEBUG { 19:38:29 DEBUG "status": "success", 19:38:29 DEBUG "data": { 19:38:29 DEBUG "resultType": "vector", 19:38:29 DEBUG "result": [ 19:38:29 DEBUG { 19:38:29 DEBUG "metric": { 19:38:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:38:29 DEBUG }, 19:38:29 DEBUG "value": [ 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG [...] 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG 19:38:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:38:29 DEBUG }, 19:38:29 DEBUG "value": [ 19:38:29 DEBUG 1713551909, 19:38:29 DEBUG "0.010499999999999999" 19:38:29 DEBUG ] 19:38:29 DEBUG } 19:38:29 DEBUG ] 19:38:29 DEBUG } 19:38:29 DEBUG } 19:38:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551910" 19:38:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551910" 19:38:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713551910" 19:38:30 INFO [http_cmd]: http status code OK 19:38:30 INFO [http_cmd]: http status code OK 19:38:30 DEBUG --- status code --- 19:38:30 DEBUG --- status code --- 19:38:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:30 DEBUG --- http response --- 19:38:30 DEBUG --- http response --- 19:38:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:38:30 DEBUG ----- output ----- 19:38:30 DEBUG { 19:38:30 DEBUG "status": "success", 19:38:30 DEBUG "data": { 19:38:30 DEBUG "resultType": "vector", 19:38:30 DEBUG "result": [ 19:38:30 DEBUG { 19:38:30 DEBUG "metric": { 19:38:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:38:30 DEBUG }, 19:38:30 DEBUG "value": [ 19:38:30 DEBUG 19:38:30 DEBUG 19:38:30 DEBUG 19:38:30 DEBUG [...] 19:38:30 DEBUG 19:38:30 DEBUG 19:38:30 DEBUG 19:38:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:38:30 DEBUG }, 19:38:30 DEBUG "value": [ 19:38:30 DEBUG 1713551910, 19:38:30 DEBUG "2332.9666666666667" 19:38:30 DEBUG ] 19:38:30 DEBUG } 19:38:30 DEBUG ] 19:38:30 DEBUG } 19:38:30 DEBUG } 19:38:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (107 lines): 19:38:30 INFO [http_cmd]: http status code OK 19:38:30 DEBUG ----- output ----- 19:38:30 DEBUG --- status code --- 19:38:30 DEBUG { 19:38:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:30 DEBUG "status": "success", 19:38:30 DEBUG --- http response --- 19:38:30 DEBUG "data": { 19:38:30 DEBUG "resultType": "vector", 19:38:30 DEBUG "result": [ 19:38:30 DEBUG { 19:38:30 DEBUG "metric": { 19:38:30 DEBUG "pod": "ds-idrepo-0" 19:38:30 DEBUG }, 19:38:30 DEBUG "value": [ 19:38:30 DEBUG 19:38:30 DEBUG 19:38:30 DEBUG 19:38:30 DEBUG [...] 19:38:30 DEBUG 19:38:30 DEBUG 19:38:30 DEBUG 19:38:30 DEBUG "pod": "idm-b85f9d978-txpwv" 19:38:30 DEBUG }, 19:38:30 DEBUG "value": [ 19:38:30 DEBUG 1713551910, 19:38:30 DEBUG "0" 19:38:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:38:30 DEBUG ] 19:38:30 DEBUG ----- output ----- 19:38:30 DEBUG } 19:38:30 DEBUG { 19:38:30 DEBUG ] 19:38:30 DEBUG "status": "success", 19:38:30 DEBUG } 19:38:30 DEBUG "data": { 19:38:30 DEBUG } 19:38:30 DEBUG "resultType": "vector", 19:38:30 DEBUG "result": [ 19:38:30 DEBUG { 19:38:30 DEBUG "metric": { 19:38:30 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:38:30 DEBUG }, 19:38:30 DEBUG "value": [ 19:38:30 DEBUG 19:38:30 DEBUG 19:38:30 DEBUG 19:38:30 DEBUG [...] 19:38:30 DEBUG 19:38:30 DEBUG 19:38:30 DEBUG 19:38:30 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:38:30 DEBUG }, 19:38:30 DEBUG "value": [ 19:38:30 DEBUG 1713551910, 19:38:30 DEBUG "3412377600" 19:38:30 DEBUG ] 19:38:30 DEBUG } 19:38:30 DEBUG ] 19:38:30 DEBUG } 19:38:30 DEBUG } 19:38:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551910" 19:38:30 INFO [http_cmd]: http status code OK 19:38:30 DEBUG --- status code --- 19:38:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:30 DEBUG --- http response --- 19:38:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:38:30 DEBUG ----- output ----- 19:38:30 DEBUG { 19:38:30 DEBUG "status": "success", 19:38:30 DEBUG "data": { 19:38:30 DEBUG "resultType": "vector", 19:38:30 DEBUG "result": [ 19:38:30 DEBUG { 19:38:30 DEBUG "metric": { 19:38:30 DEBUG "pod": "ds-idrepo-0" 19:38:30 DEBUG }, 19:38:30 DEBUG "value": [ 19:38:30 DEBUG 19:38:30 DEBUG 19:38:30 DEBUG 19:38:30 DEBUG [...] 19:38:30 DEBUG 19:38:30 DEBUG 19:38:30 DEBUG 19:38:30 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 19:38:30 DEBUG }, 19:38:30 DEBUG "value": [ 19:38:30 DEBUG 1713551910, 19:38:30 DEBUG "0" 19:38:30 DEBUG ] 19:38:30 DEBUG } 19:38:30 DEBUG ] 19:38:30 DEBUG } 19:38:30 DEBUG } 19:38:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551910" 19:38:30 INFO [http_cmd]: http status code OK 19:38:30 DEBUG --- status code --- 19:38:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:30 DEBUG --- http response --- 19:38:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:38:30 DEBUG ----- output ----- 19:38:30 DEBUG { 19:38:30 DEBUG "status": "success", 19:38:30 DEBUG "data": { 19:38:30 DEBUG "resultType": "vector", 19:38:30 DEBUG "result": [ 19:38:30 DEBUG { 19:38:30 DEBUG "metric": { 19:38:30 DEBUG "instance": "10.142.0.107:9100" 19:38:30 DEBUG }, 19:38:30 DEBUG "value": [ 19:38:30 DEBUG 19:38:30 DEBUG 19:38:30 DEBUG 19:38:30 DEBUG [...] 19:38:30 DEBUG 19:38:30 DEBUG 19:38:30 DEBUG 19:38:30 DEBUG "instance": "10.142.0.43:9100" 19:38:30 DEBUG }, 19:38:30 DEBUG "value": [ 19:38:30 DEBUG 1713551910, 19:38:30 DEBUG "70519831.66666667" 19:38:30 DEBUG ] 19:38:30 DEBUG } 19:38:30 DEBUG ] 19:38:30 DEBUG } 19:38:30 DEBUG } 19:38:30 INFO 19:38:30 INFO [loop_until]: kubectl --namespace=xlou top pods 19:38:30 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:38:30 INFO [loop_until]: OK (rc = 0) 19:38:30 DEBUG --- stdout --- 19:38:30 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 14072m 8988Mi am-767f6b67b5-qcqz2 14330m 9002Mi am-767f6b67b5-vbltz 15659m 8984Mi ds-cts-0 15852m 10889Mi ds-cts-1 14533m 11148Mi ds-cts-2 15785m 10909Mi ds-idrepo-0 6297m 3663Mi ds-idrepo-1 6288m 3534Mi ds-idrepo-2 6376m 3493Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7726m 3161Mi 19:38:30 DEBUG --- stderr --- 19:38:30 DEBUG 19:38:31 INFO 19:38:31 INFO [loop_until]: kubectl --namespace=xlou top node 19:38:31 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:38:31 INFO [loop_until]: OK (rc = 0) 19:38:31 DEBUG --- stdout --- 19:38:31 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 70m 0% 6262Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 6528m 21% 7238Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 14335m 48% 11700Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 87m 0% 3338Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 15557m 52% 13916Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 157m 0% 5184Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 14446m 48% 11885Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12358m 41% 2987Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 16540m 55% 11202Mi 10% gke-xlou-cdm-ds-0f601751-j488 7032m 23% 6741Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 14024m 46% 15947Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 7030m 23% 7420Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 15721m 52% 13378Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 7898m 49% 6938Mi 11% 19:38:31 DEBUG --- stderr --- 19:38:31 DEBUG 19:39:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551969" 19:39:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551969" 19:39:29 INFO [http_cmd]: http status code OK 19:39:29 DEBUG --- status code --- 19:39:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:29 DEBUG --- http response --- 19:39:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:39:29 DEBUG ----- output ----- 19:39:29 DEBUG { 19:39:29 DEBUG "status": "success", 19:39:29 DEBUG "data": { 19:39:29 DEBUG "resultType": "vector", 19:39:29 DEBUG "result": [ 19:39:29 DEBUG { 19:39:29 DEBUG "metric": { 19:39:29 DEBUG "pod": "ds-idrepo-0" 19:39:29 DEBUG }, 19:39:29 DEBUG "value": [ 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG [...] 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG "pod": "ds-idrepo-2" 19:39:29 DEBUG }, 19:39:29 DEBUG "value": [ 19:39:29 DEBUG 1713551969, 19:39:29 DEBUG "0" 19:39:29 DEBUG ] 19:39:29 DEBUG } 19:39:29 DEBUG ] 19:39:29 DEBUG } 19:39:29 DEBUG } 19:39:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551969" 19:39:29 INFO [http_cmd]: http status code OK 19:39:29 DEBUG --- status code --- 19:39:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:29 DEBUG --- http response --- 19:39:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:39:29 DEBUG ----- output ----- 19:39:29 DEBUG { 19:39:29 DEBUG "status": "success", 19:39:29 DEBUG "data": { 19:39:29 DEBUG "resultType": "vector", 19:39:29 DEBUG "result": [ 19:39:29 DEBUG { 19:39:29 DEBUG "metric": { 19:39:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:39:29 DEBUG }, 19:39:29 DEBUG "value": [ 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG [...] 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:39:29 DEBUG }, 19:39:29 DEBUG "value": [ 19:39:29 DEBUG 1713551969, 19:39:29 DEBUG "0" 19:39:29 DEBUG ] 19:39:29 DEBUG } 19:39:29 DEBUG ] 19:39:29 DEBUG } 19:39:29 DEBUG } 19:39:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713551969" 19:39:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551969" 19:39:29 INFO [http_cmd]: http status code OK 19:39:29 DEBUG --- status code --- 19:39:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:29 DEBUG --- http response --- 19:39:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:39:29 DEBUG ----- output ----- 19:39:29 DEBUG { 19:39:29 DEBUG "status": "success", 19:39:29 DEBUG "data": { 19:39:29 DEBUG "resultType": "vector", 19:39:29 DEBUG "result": [ 19:39:29 DEBUG { 19:39:29 DEBUG "metric": { 19:39:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:39:29 DEBUG }, 19:39:29 DEBUG "value": [ 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG [...] 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:39:29 DEBUG }, 19:39:29 DEBUG "value": [ 19:39:29 DEBUG 1713551969, 19:39:29 DEBUG "2345.4" 19:39:29 DEBUG ] 19:39:29 DEBUG } 19:39:29 DEBUG ] 19:39:29 DEBUG } 19:39:29 DEBUG } 19:39:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551969" 19:39:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551969" 19:39:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713551969" 19:39:29 ERROR [http_cmd]: ERROR 19:39:29 ERROR --- status code --- 19:39:29 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:39:29 ERROR --- http response --- 19:39:29 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:39:29 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:39:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551969" 19:39:29 INFO [http_cmd]: http status code OK 19:39:29 DEBUG --- status code --- 19:39:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:29 DEBUG --- http response --- 19:39:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:39:29 DEBUG ----- output ----- 19:39:29 DEBUG { 19:39:29 DEBUG "status": "success", 19:39:29 DEBUG "data": { 19:39:29 DEBUG "resultType": "vector", 19:39:29 DEBUG "result": [ 19:39:29 DEBUG { 19:39:29 DEBUG "metric": { 19:39:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:39:29 DEBUG }, 19:39:29 DEBUG "value": [ 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG [...] 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:39:29 DEBUG }, 19:39:29 DEBUG "value": [ 19:39:29 DEBUG 1713551969, 19:39:29 DEBUG "420253" 19:39:29 DEBUG ] 19:39:29 DEBUG } 19:39:29 DEBUG ] 19:39:29 DEBUG } 19:39:29 DEBUG } 19:39:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551969" 19:39:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551969" 19:39:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713551969" 19:39:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551969" 19:39:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551969" 19:39:29 INFO [http_cmd]: http status code OK 19:39:29 DEBUG --- status code --- 19:39:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:29 DEBUG --- http response --- 19:39:29 INFO [http_cmd]: http status code OK 19:39:29 DEBUG --- status code --- 19:39:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:29 DEBUG --- http response --- 19:39:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:39:29 DEBUG ----- output ----- 19:39:29 DEBUG { 19:39:29 DEBUG "status": "success", 19:39:29 DEBUG "data": { 19:39:29 DEBUG "resultType": "vector", 19:39:29 DEBUG "result": [ 19:39:29 DEBUG { 19:39:29 DEBUG "metric": { 19:39:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:39:29 DEBUG }, 19:39:29 DEBUG "value": [ 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG [...] 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:39:29 DEBUG }, 19:39:29 DEBUG "value": [ 19:39:29 DEBUG 1713551969, 19:39:29 DEBUG "2327.4" 19:39:29 DEBUG ] 19:39:29 DEBUG } 19:39:29 DEBUG ] 19:39:29 DEBUG } 19:39:29 DEBUG } 19:39:29 INFO [http_cmd]: http status code OK 19:39:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:39:29 INFO [http_cmd]: http status code OK 19:39:29 INFO [http_cmd]: http status code OK 19:39:29 DEBUG --- status code --- 19:39:29 DEBUG ----- output ----- 19:39:29 DEBUG --- status code --- 19:39:29 DEBUG --- status code --- 19:39:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:29 DEBUG { 19:39:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:29 DEBUG --- http response --- 19:39:29 DEBUG "status": "success", 19:39:29 INFO [http_cmd]: http status code OK 19:39:29 DEBUG --- http response --- 19:39:29 DEBUG --- http response --- 19:39:29 DEBUG "data": { 19:39:29 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:39:29 INFO [http_cmd]: http status code OK 19:39:29 DEBUG --- status code --- 19:39:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:39:29 DEBUG "resultType": "vector", 19:39:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:39:29 INFO [http_cmd]: http status code OK 19:39:29 DEBUG --- status code --- 19:39:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:29 DEBUG ----- output ----- 19:39:29 DEBUG "result": [ 19:39:29 DEBUG ----- output ----- 19:39:29 DEBUG --- status code --- 19:39:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:29 DEBUG --- http response --- 19:39:29 DEBUG { 19:39:29 DEBUG { 19:39:29 DEBUG { 19:39:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:29 DEBUG --- http response --- 19:39:29 DEBUG "status": "success", 19:39:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:39:29 DEBUG "metric": { 19:39:29 DEBUG "status": "success", 19:39:29 DEBUG --- http response --- 19:39:29 DEBUG "data": { 19:39:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:39:29 DEBUG ----- output ----- 19:39:29 DEBUG "instance": "10.142.0.107:9100" 19:39:29 DEBUG "data": { 19:39:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551969" 19:39:29 DEBUG "resultType": "vector", 19:39:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:39:29 DEBUG ----- output ----- 19:39:29 DEBUG { 19:39:29 DEBUG }, 19:39:29 DEBUG "resultType": "vector", 19:39:29 DEBUG "result": [ 19:39:29 DEBUG ----- output ----- 19:39:29 DEBUG { 19:39:29 INFO [http_cmd]: http status code OK 19:39:29 DEBUG "status": "success", 19:39:29 DEBUG "value": [ 19:39:29 DEBUG "result": [ 19:39:29 DEBUG { 19:39:29 DEBUG { 19:39:29 DEBUG "status": "success", 19:39:29 DEBUG --- status code --- 19:39:29 DEBUG "data": { 19:39:29 DEBUG 19:39:29 DEBUG { 19:39:29 DEBUG "metric": { 19:39:29 DEBUG "status": "success", 19:39:29 DEBUG "data": { 19:39:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:29 DEBUG "resultType": "vector", 19:39:29 DEBUG 19:39:29 DEBUG "metric": { 19:39:29 DEBUG "pod": "ds-idrepo-0" 19:39:29 DEBUG "data": { 19:39:29 DEBUG "resultType": "vector", 19:39:29 DEBUG --- http response --- 19:39:29 DEBUG "result": [ 19:39:29 DEBUG 19:39:29 DEBUG "instance": "10.142.0.107:9100" 19:39:29 DEBUG }, 19:39:29 DEBUG "resultType": "vector", 19:39:29 DEBUG "result": [ 19:39:29 DEBUG { 19:39:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:39:29 DEBUG [...] 19:39:29 DEBUG }, 19:39:29 DEBUG "value": [ 19:39:29 DEBUG "result": [ 19:39:29 DEBUG { 19:39:29 DEBUG "metric": { 19:39:29 DEBUG ----- output ----- 19:39:29 DEBUG { 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG { 19:39:29 DEBUG "metric": { 19:39:29 DEBUG "pod": "ds-idrepo-0" 19:39:29 DEBUG "value": [ 19:39:29 DEBUG "status": "success", 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG "metric": { 19:39:29 DEBUG "instance": "10.142.0.107:9100" 19:39:29 DEBUG }, 19:39:29 DEBUG 19:39:29 DEBUG "data": { 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:39:29 DEBUG }, 19:39:29 DEBUG "value": [ 19:39:29 DEBUG 19:39:29 DEBUG "resultType": "vector", 19:39:29 DEBUG "instance": "10.142.0.43:9100" 19:39:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551969" 19:39:29 DEBUG [...] 19:39:29 DEBUG }, 19:39:29 DEBUG "value": [ 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG "result": [ 19:39:29 DEBUG }, 19:39:29 DEBUG 19:39:29 DEBUG "value": [ 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG [...] 19:39:29 DEBUG { 19:39:29 DEBUG "value": [ 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG "metric": { 19:39:29 DEBUG 1713551969, 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG [...] 19:39:29 DEBUG 19:39:29 DEBUG "pod": "ds-idrepo-1" 19:39:29 DEBUG "0.16716666666671454" 19:39:29 DEBUG "pod": "ds-cts-2" 19:39:29 DEBUG 19:39:29 DEBUG [...] 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG }, 19:39:29 DEBUG ] 19:39:29 DEBUG }, 19:39:29 DEBUG [...] 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG "instance": "10.142.0.43:9100" 19:39:29 DEBUG "value": [ 19:39:29 DEBUG } 19:39:29 DEBUG "value": [ 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG }, 19:39:29 DEBUG 19:39:29 DEBUG ] 19:39:29 DEBUG 1713551969, 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG "instance": "10.142.0.43:9100" 19:39:29 DEBUG "value": [ 19:39:29 DEBUG 19:39:29 DEBUG } 19:39:29 DEBUG "0" 19:39:29 DEBUG 19:39:29 DEBUG "pod": "ds-idrepo-2" 19:39:29 DEBUG }, 19:39:29 DEBUG 1713551969, 19:39:29 DEBUG 19:39:29 INFO [http_cmd]: http status code OK 19:39:29 DEBUG } 19:39:29 DEBUG ] 19:39:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:39:29 DEBUG }, 19:39:29 DEBUG "value": [ 19:39:29 DEBUG "70638148.56666666" 19:39:29 DEBUG [...] 19:39:29 DEBUG --- status code --- 19:39:29 DEBUG } 19:39:29 DEBUG }, 19:39:29 DEBUG "value": [ 19:39:29 DEBUG 1713551969, 19:39:29 DEBUG ] 19:39:29 DEBUG 19:39:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:29 DEBUG ] 19:39:29 DEBUG "value": [ 19:39:29 DEBUG 1713551969, 19:39:29 DEBUG "1928722250752" 19:39:29 DEBUG } 19:39:29 DEBUG 19:39:29 DEBUG --- http response --- 19:39:29 DEBUG } 19:39:29 DEBUG 1713551969, 19:39:29 DEBUG "0" 19:39:29 DEBUG ] 19:39:29 DEBUG ] 19:39:29 DEBUG 19:39:29 DEBUG } 19:39:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:39:29 DEBUG "7.7012521423560125" 19:39:29 DEBUG ] 19:39:29 DEBUG } 19:39:29 DEBUG } 19:39:29 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:39:29 DEBUG ----- output ----- 19:39:29 DEBUG ] 19:39:29 DEBUG } 19:39:29 DEBUG ] 19:39:29 DEBUG } 19:39:29 DEBUG }, 19:39:29 DEBUG { 19:39:29 DEBUG } 19:39:29 DEBUG ] 19:39:29 DEBUG } 19:39:29 DEBUG "value": [ 19:39:29 DEBUG "status": "success", 19:39:29 DEBUG ] 19:39:29 DEBUG } 19:39:29 DEBUG } 19:39:29 DEBUG 1713551969, 19:39:29 DEBUG "data": { 19:39:29 DEBUG } 19:39:29 DEBUG } 19:39:29 DEBUG "0" 19:39:29 DEBUG "resultType": "vector", 19:39:29 DEBUG } 19:39:29 DEBUG ] 19:39:29 DEBUG "result": [ 19:39:29 DEBUG } 19:39:29 DEBUG { 19:39:29 DEBUG ] 19:39:29 DEBUG "metric": { 19:39:29 DEBUG } 19:39:29 DEBUG "pod": "ds-idrepo-1" 19:39:29 DEBUG } 19:39:29 DEBUG }, 19:39:29 INFO [http_cmd]: http status code OK 19:39:29 DEBUG "value": [ 19:39:29 DEBUG --- status code --- 19:39:29 DEBUG 19:39:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:29 DEBUG 19:39:29 DEBUG --- http response --- 19:39:29 DEBUG 19:39:29 DEBUG [...] 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:39:29 DEBUG }, 19:39:29 DEBUG "value": [ 19:39:29 DEBUG 1713551969, 19:39:29 DEBUG "0" 19:39:29 DEBUG ] 19:39:29 DEBUG } 19:39:29 DEBUG ] 19:39:29 DEBUG } 19:39:29 DEBUG } 19:39:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:39:29 DEBUG ----- output ----- 19:39:29 DEBUG { 19:39:29 DEBUG "status": "success", 19:39:29 DEBUG "data": { 19:39:29 DEBUG "resultType": "vector", 19:39:29 DEBUG "result": [ 19:39:29 DEBUG { 19:39:29 DEBUG "metric": { 19:39:29 DEBUG "pod": "ds-cts-0" 19:39:29 DEBUG }, 19:39:29 DEBUG "value": [ 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG [...] 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG "pod": "ds-cts-2" 19:39:29 DEBUG }, 19:39:29 DEBUG "value": [ 19:39:29 DEBUG 1713551969, 19:39:29 DEBUG "0" 19:39:29 DEBUG ] 19:39:29 DEBUG } 19:39:29 DEBUG ] 19:39:29 DEBUG } 19:39:29 DEBUG } 19:39:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551969" 19:39:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551969" 19:39:29 INFO [http_cmd]: http status code OK 19:39:29 DEBUG --- status code --- 19:39:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:29 DEBUG --- http response --- 19:39:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:39:29 DEBUG ----- output ----- 19:39:29 DEBUG { 19:39:29 DEBUG "status": "success", 19:39:29 DEBUG "data": { 19:39:29 DEBUG "resultType": "vector", 19:39:29 DEBUG "result": [ 19:39:29 DEBUG { 19:39:29 DEBUG "metric": { 19:39:29 DEBUG "pod": "ds-idrepo-0" 19:39:29 DEBUG }, 19:39:29 DEBUG "value": [ 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG [...] 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG "pod": "ds-idrepo-2" 19:39:29 DEBUG }, 19:39:29 DEBUG "value": [ 19:39:29 DEBUG 1713551969, 19:39:29 DEBUG "0" 19:39:29 DEBUG ] 19:39:29 DEBUG } 19:39:29 DEBUG ] 19:39:29 DEBUG } 19:39:29 DEBUG } 19:39:29 INFO [http_cmd]: http status code OK 19:39:29 DEBUG --- status code --- 19:39:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:29 DEBUG --- http response --- 19:39:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:39:29 DEBUG ----- output ----- 19:39:29 DEBUG { 19:39:29 DEBUG "status": "success", 19:39:29 DEBUG "data": { 19:39:29 DEBUG "resultType": "vector", 19:39:29 DEBUG "result": [ 19:39:29 DEBUG { 19:39:29 DEBUG "metric": { 19:39:29 DEBUG "instance": "10.142.0.107:9100" 19:39:29 DEBUG }, 19:39:29 DEBUG "value": [ 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG [...] 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG "instance": "10.142.0.43:9100" 19:39:29 DEBUG }, 19:39:29 DEBUG "value": [ 19:39:29 DEBUG 1713551969, 19:39:29 DEBUG "0.08266666666666157" 19:39:29 DEBUG ] 19:39:29 DEBUG } 19:39:29 DEBUG ] 19:39:29 DEBUG } 19:39:29 DEBUG } 19:39:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551969" 19:39:29 INFO [http_cmd]: http status code OK 19:39:29 DEBUG --- status code --- 19:39:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:29 DEBUG --- http response --- 19:39:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:39:29 DEBUG ----- output ----- 19:39:29 DEBUG { 19:39:29 DEBUG "status": "success", 19:39:29 DEBUG "data": { 19:39:29 DEBUG "resultType": "vector", 19:39:29 DEBUG "result": [ 19:39:29 DEBUG { 19:39:29 DEBUG "metric": { 19:39:29 DEBUG "pod": "ds-cts-0" 19:39:29 DEBUG }, 19:39:29 DEBUG "value": [ 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG [...] 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG "pod": "ds-cts-2" 19:39:29 DEBUG }, 19:39:29 DEBUG "value": [ 19:39:29 DEBUG 1713551969, 19:39:29 DEBUG "29.933333333333334" 19:39:29 DEBUG ] 19:39:29 DEBUG } 19:39:29 DEBUG ] 19:39:29 DEBUG } 19:39:29 DEBUG } 19:39:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551969" 19:39:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551969" 19:39:29 INFO [http_cmd]: http status code OK 19:39:29 DEBUG --- status code --- 19:39:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:29 DEBUG --- http response --- 19:39:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:39:29 DEBUG ----- output ----- 19:39:29 DEBUG { 19:39:29 DEBUG "status": "success", 19:39:29 DEBUG "data": { 19:39:29 DEBUG "resultType": "vector", 19:39:29 DEBUG "result": [ 19:39:29 DEBUG { 19:39:29 DEBUG "metric": { 19:39:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:39:29 DEBUG }, 19:39:29 DEBUG "value": [ 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG [...] 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:39:29 DEBUG }, 19:39:29 DEBUG "value": [ 19:39:29 DEBUG 1713551969, 19:39:29 DEBUG "2327.2" 19:39:29 DEBUG ] 19:39:29 DEBUG } 19:39:29 DEBUG ] 19:39:29 DEBUG } 19:39:29 DEBUG } 19:39:29 INFO [http_cmd]: http status code OK 19:39:29 DEBUG --- status code --- 19:39:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:29 DEBUG --- http response --- 19:39:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:39:29 DEBUG ----- output ----- 19:39:29 DEBUG { 19:39:29 DEBUG "status": "success", 19:39:29 DEBUG "data": { 19:39:29 DEBUG "resultType": "vector", 19:39:29 DEBUG "result": [ 19:39:29 DEBUG { 19:39:29 DEBUG "metric": { 19:39:29 DEBUG "pod": "ds-cts-0" 19:39:29 DEBUG }, 19:39:29 DEBUG "value": [ 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG [...] 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG 19:39:29 DEBUG "pod": "ds-cts-2" 19:39:29 DEBUG }, 19:39:29 DEBUG "value": [ 19:39:29 DEBUG 1713551969, 19:39:29 DEBUG "0.000033333333333333335" 19:39:29 DEBUG ] 19:39:29 DEBUG } 19:39:29 DEBUG ] 19:39:29 DEBUG } 19:39:29 DEBUG } 19:39:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713551970" 19:39:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551970" 19:39:30 INFO [http_cmd]: http status code OK 19:39:30 DEBUG --- status code --- 19:39:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:30 DEBUG --- http response --- 19:39:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:39:30 DEBUG ----- output ----- 19:39:30 DEBUG { 19:39:30 DEBUG "status": "success", 19:39:30 DEBUG "data": { 19:39:30 DEBUG "resultType": "vector", 19:39:30 DEBUG "result": [ 19:39:30 DEBUG { 19:39:30 DEBUG "metric": { 19:39:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:39:30 DEBUG }, 19:39:30 DEBUG "value": [ 19:39:30 DEBUG 19:39:30 DEBUG 19:39:30 DEBUG 19:39:30 DEBUG [...] 19:39:30 DEBUG 19:39:30 DEBUG 19:39:30 DEBUG 19:39:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:39:30 DEBUG }, 19:39:30 DEBUG "value": [ 19:39:30 DEBUG 1713551970, 19:39:30 DEBUG "0.011" 19:39:30 DEBUG ] 19:39:30 DEBUG } 19:39:30 DEBUG ] 19:39:30 DEBUG } 19:39:30 DEBUG } 19:39:30 INFO [http_cmd]: http status code OK 19:39:30 DEBUG --- status code --- 19:39:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:30 DEBUG --- http response --- 19:39:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:39:30 DEBUG ----- output ----- 19:39:30 DEBUG { 19:39:30 DEBUG "status": "success", 19:39:30 DEBUG "data": { 19:39:30 DEBUG "resultType": "vector", 19:39:30 DEBUG "result": [ 19:39:30 DEBUG { 19:39:30 DEBUG "metric": { 19:39:30 DEBUG "pod": "ds-idrepo-1" 19:39:30 DEBUG }, 19:39:30 DEBUG "value": [ 19:39:30 DEBUG 19:39:30 DEBUG 19:39:30 DEBUG 19:39:30 DEBUG [...] 19:39:30 DEBUG 19:39:30 DEBUG 19:39:30 DEBUG 19:39:30 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:39:30 DEBUG }, 19:39:30 DEBUG "value": [ 19:39:30 DEBUG 1713551970, 19:39:30 DEBUG "0" 19:39:30 DEBUG ] 19:39:30 DEBUG } 19:39:30 DEBUG ] 19:39:30 DEBUG } 19:39:30 DEBUG } 19:39:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551970" 19:39:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713551970" 19:39:30 INFO [http_cmd]: http status code OK 19:39:30 DEBUG --- status code --- 19:39:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:30 DEBUG --- http response --- 19:39:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:39:30 DEBUG ----- output ----- 19:39:30 DEBUG { 19:39:30 DEBUG "status": "success", 19:39:30 DEBUG "data": { 19:39:30 DEBUG "resultType": "vector", 19:39:30 DEBUG "result": [ 19:39:30 DEBUG { 19:39:30 DEBUG "metric": { 19:39:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:39:30 DEBUG }, 19:39:30 DEBUG "value": [ 19:39:30 DEBUG 19:39:30 DEBUG 19:39:30 DEBUG 19:39:30 DEBUG [...] 19:39:30 DEBUG 19:39:30 DEBUG 19:39:30 DEBUG 19:39:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:39:30 DEBUG }, 19:39:30 DEBUG "value": [ 19:39:30 DEBUG 1713551970, 19:39:30 DEBUG "2327.4666666666667" 19:39:30 DEBUG ] 19:39:30 DEBUG } 19:39:30 DEBUG ] 19:39:30 DEBUG } 19:39:30 DEBUG } 19:39:30 INFO [http_cmd]: http status code OK 19:39:30 DEBUG --- status code --- 19:39:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:30 DEBUG --- http response --- 19:39:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:39:30 DEBUG ----- output ----- 19:39:30 DEBUG { 19:39:30 DEBUG "status": "success", 19:39:30 DEBUG "data": { 19:39:30 DEBUG "resultType": "vector", 19:39:30 DEBUG "result": [ 19:39:30 DEBUG { 19:39:30 DEBUG "metric": { 19:39:30 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:39:30 DEBUG }, 19:39:30 DEBUG "value": [ 19:39:30 DEBUG 19:39:30 DEBUG 19:39:30 DEBUG 19:39:30 DEBUG [...] 19:39:30 DEBUG 19:39:30 DEBUG 19:39:30 DEBUG 19:39:30 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:39:30 DEBUG }, 19:39:30 DEBUG "value": [ 19:39:30 DEBUG 1713551970, 19:39:30 DEBUG "3410182144" 19:39:30 DEBUG ] 19:39:30 DEBUG } 19:39:30 DEBUG ] 19:39:30 DEBUG } 19:39:30 DEBUG } 19:39:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713551970" 19:39:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713551970" 19:39:30 INFO [http_cmd]: http status code OK 19:39:30 DEBUG --- status code --- 19:39:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:30 DEBUG --- http response --- 19:39:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:39:30 DEBUG ----- output ----- 19:39:30 DEBUG { 19:39:30 DEBUG "status": "success", 19:39:30 DEBUG "data": { 19:39:30 DEBUG "resultType": "vector", 19:39:30 DEBUG "result": [ 19:39:30 DEBUG { 19:39:30 DEBUG "metric": { 19:39:30 DEBUG "pod": "ds-idrepo-1" 19:39:30 DEBUG }, 19:39:30 DEBUG "value": [ 19:39:30 DEBUG 19:39:30 DEBUG 19:39:30 DEBUG 19:39:30 DEBUG [...] 19:39:30 DEBUG 19:39:30 DEBUG 19:39:30 DEBUG 19:39:30 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:39:30 DEBUG }, 19:39:30 DEBUG "value": [ 19:39:30 DEBUG 1713551970, 19:39:30 DEBUG "0.000009357834870453788" 19:39:30 DEBUG ] 19:39:30 DEBUG } 19:39:30 DEBUG ] 19:39:30 DEBUG } 19:39:30 DEBUG } 19:39:30 INFO [http_cmd]: http status code OK 19:39:30 DEBUG --- status code --- 19:39:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:30 DEBUG --- http response --- 19:39:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:39:30 DEBUG ----- output ----- 19:39:30 DEBUG { 19:39:30 DEBUG "status": "success", 19:39:30 DEBUG "data": { 19:39:30 DEBUG "resultType": "vector", 19:39:30 DEBUG "result": [ 19:39:30 DEBUG { 19:39:30 DEBUG "metric": { 19:39:30 DEBUG "instance": "10.142.0.107:9100" 19:39:30 DEBUG }, 19:39:30 DEBUG "value": [ 19:39:30 DEBUG 19:39:30 DEBUG 19:39:30 DEBUG 19:39:30 DEBUG [...] 19:39:30 DEBUG 19:39:30 DEBUG 19:39:30 DEBUG 19:39:30 DEBUG "instance": "10.142.0.43:9100" 19:39:30 DEBUG }, 19:39:30 DEBUG "value": [ 19:39:30 DEBUG 1713551970, 19:39:30 DEBUG "70635254.46666667" 19:39:30 DEBUG ] 19:39:30 DEBUG } 19:39:30 DEBUG ] 19:39:30 DEBUG } 19:39:30 DEBUG } 19:39:30 INFO 19:39:30 INFO [loop_until]: kubectl --namespace=xlou top pods 19:39:30 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:39:30 INFO [loop_until]: OK (rc = 0) 19:39:30 DEBUG --- stdout --- 19:39:30 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 14459m 8993Mi am-767f6b67b5-qcqz2 14344m 9005Mi am-767f6b67b5-vbltz 15528m 8981Mi ds-cts-0 14752m 10905Mi ds-cts-1 13945m 11160Mi ds-cts-2 16059m 10900Mi ds-idrepo-0 6287m 3663Mi ds-idrepo-1 6236m 3533Mi ds-idrepo-2 6389m 3492Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7819m 3162Mi 19:39:30 DEBUG --- stderr --- 19:39:30 DEBUG 19:39:31 INFO 19:39:31 INFO [loop_until]: kubectl --namespace=xlou top node 19:39:31 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:39:31 INFO [loop_until]: OK (rc = 0) 19:39:31 DEBUG --- stdout --- 19:39:31 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 68m 0% 6251Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 6520m 21% 7242Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 14205m 47% 11707Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 93m 0% 3343Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 15255m 51% 13909Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 158m 0% 5167Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 14326m 47% 11890Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12313m 41% 2991Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 16366m 54% 11195Mi 10% gke-xlou-cdm-ds-0f601751-j488 6987m 23% 6737Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 13907m 46% 15939Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 6980m 23% 7433Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 16015m 53% 13387Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 7679m 48% 6941Mi 11% 19:39:31 DEBUG --- stderr --- 19:39:31 DEBUG 19:40:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552029" 19:40:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552029" 19:40:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552029" 19:40:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552029" 19:40:29 INFO [http_cmd]: http status code OK 19:40:29 INFO [http_cmd]: http status code OK 19:40:29 INFO [http_cmd]: http status code OK 19:40:29 DEBUG --- status code --- 19:40:29 DEBUG --- status code --- 19:40:29 DEBUG --- status code --- 19:40:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:29 DEBUG --- http response --- 19:40:29 DEBUG --- http response --- 19:40:29 DEBUG --- http response --- 19:40:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:40:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:40:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:40:29 DEBUG ----- output ----- 19:40:29 DEBUG ----- output ----- 19:40:29 DEBUG ----- output ----- 19:40:29 DEBUG { 19:40:29 DEBUG { 19:40:29 DEBUG { 19:40:29 DEBUG "status": "success", 19:40:29 DEBUG "status": "success", 19:40:29 DEBUG "status": "success", 19:40:29 DEBUG "data": { 19:40:29 DEBUG "data": { 19:40:29 DEBUG "data": { 19:40:29 DEBUG "resultType": "vector", 19:40:29 DEBUG "resultType": "vector", 19:40:29 DEBUG "resultType": "vector", 19:40:29 DEBUG "result": [ 19:40:29 DEBUG "result": [ 19:40:29 DEBUG "result": [ 19:40:29 DEBUG { 19:40:29 DEBUG { 19:40:29 DEBUG { 19:40:29 DEBUG "metric": { 19:40:29 DEBUG "metric": { 19:40:29 DEBUG "metric": { 19:40:29 DEBUG "pod": "ds-idrepo-0" 19:40:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:40:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:40:29 DEBUG }, 19:40:29 DEBUG }, 19:40:29 DEBUG }, 19:40:29 DEBUG "value": [ 19:40:29 DEBUG "value": [ 19:40:29 DEBUG "value": [ 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552029" 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG [...] 19:40:29 DEBUG [...] 19:40:29 DEBUG [...] 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:40:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:40:29 DEBUG "pod": "ds-idrepo-2" 19:40:29 DEBUG }, 19:40:29 DEBUG }, 19:40:29 DEBUG }, 19:40:29 DEBUG "value": [ 19:40:29 DEBUG "value": [ 19:40:29 DEBUG "value": [ 19:40:29 DEBUG 1713552029, 19:40:29 DEBUG 1713552029, 19:40:29 DEBUG 1713552029, 19:40:29 DEBUG "419625.5" 19:40:29 DEBUG "2333.9" 19:40:29 DEBUG "0" 19:40:29 DEBUG ] 19:40:29 DEBUG ] 19:40:29 DEBUG ] 19:40:29 DEBUG } 19:40:29 DEBUG } 19:40:29 DEBUG } 19:40:29 DEBUG ] 19:40:29 DEBUG ] 19:40:29 DEBUG ] 19:40:29 DEBUG } 19:40:29 DEBUG } 19:40:29 DEBUG } 19:40:29 DEBUG } 19:40:29 DEBUG } 19:40:29 DEBUG } 19:40:29 INFO [http_cmd]: http status code OK 19:40:29 DEBUG --- status code --- 19:40:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:29 DEBUG --- http response --- 19:40:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:40:29 DEBUG ----- output ----- 19:40:29 DEBUG { 19:40:29 DEBUG "status": "success", 19:40:29 DEBUG "data": { 19:40:29 DEBUG "resultType": "vector", 19:40:29 DEBUG "result": [ 19:40:29 DEBUG { 19:40:29 DEBUG "metric": { 19:40:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:40:29 DEBUG }, 19:40:29 DEBUG "value": [ 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG [...] 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:40:29 DEBUG }, 19:40:29 DEBUG "value": [ 19:40:29 DEBUG 1713552029, 19:40:29 DEBUG "0" 19:40:29 DEBUG ] 19:40:29 INFO [http_cmd]: http status code OK 19:40:29 DEBUG } 19:40:29 DEBUG --- status code --- 19:40:29 DEBUG ] 19:40:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:29 DEBUG } 19:40:29 DEBUG --- http response --- 19:40:29 DEBUG } 19:40:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:40:29 DEBUG ----- output ----- 19:40:29 DEBUG { 19:40:29 DEBUG "status": "success", 19:40:29 DEBUG "data": { 19:40:29 DEBUG "resultType": "vector", 19:40:29 DEBUG "result": [ 19:40:29 DEBUG { 19:40:29 DEBUG "metric": { 19:40:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:40:29 DEBUG }, 19:40:29 DEBUG "value": [ 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG [...] 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:40:29 DEBUG }, 19:40:29 DEBUG "value": [ 19:40:29 DEBUG 1713552029, 19:40:29 DEBUG "2330.6666666666665" 19:40:29 DEBUG ] 19:40:29 DEBUG } 19:40:29 DEBUG ] 19:40:29 DEBUG } 19:40:29 DEBUG } 19:40:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713552029" 19:40:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552029" 19:40:29 ERROR [http_cmd]: ERROR 19:40:29 ERROR --- status code --- 19:40:29 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:40:29 ERROR --- http response --- 19:40:29 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:40:29 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:40:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552029" 19:40:29 INFO [http_cmd]: http status code OK 19:40:29 DEBUG --- status code --- 19:40:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:29 DEBUG --- http response --- 19:40:29 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:40:29 INFO [http_cmd]: http status code OK 19:40:29 DEBUG --- status code --- 19:40:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:29 DEBUG --- http response --- 19:40:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:40:29 DEBUG ----- output ----- 19:40:29 DEBUG { 19:40:29 DEBUG "status": "success", 19:40:29 DEBUG "data": { 19:40:29 DEBUG "resultType": "vector", 19:40:29 DEBUG "result": [ 19:40:29 DEBUG { 19:40:29 DEBUG "metric": { 19:40:29 DEBUG "pod": "ds-cts-0" 19:40:29 DEBUG }, 19:40:29 DEBUG "value": [ 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG [...] 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG "pod": "ds-cts-2" 19:40:29 DEBUG }, 19:40:29 DEBUG "value": [ 19:40:29 DEBUG 1713552029, 19:40:29 DEBUG "0" 19:40:29 DEBUG ] 19:40:29 DEBUG } 19:40:29 DEBUG ] 19:40:29 DEBUG } 19:40:29 DEBUG } 19:40:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552029" 19:40:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713552029" 19:40:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552029" 19:40:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552029" 19:40:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552029" 19:40:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713552029" 19:40:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552029" 19:40:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552029" 19:40:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552029" 19:40:29 INFO [http_cmd]: http status code OK 19:40:29 DEBUG --- status code --- 19:40:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:29 DEBUG --- http response --- 19:40:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:40:29 DEBUG ----- output ----- 19:40:29 DEBUG { 19:40:29 DEBUG "status": "success", 19:40:29 DEBUG "data": { 19:40:29 DEBUG "resultType": "vector", 19:40:29 DEBUG "result": [ 19:40:29 DEBUG { 19:40:29 DEBUG "metric": { 19:40:29 DEBUG "instance": "10.142.0.107:9100" 19:40:29 DEBUG }, 19:40:29 DEBUG "value": [ 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG [...] 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG "instance": "10.142.0.43:9100" 19:40:29 DEBUG }, 19:40:29 DEBUG "value": [ 19:40:29 DEBUG 1713552029, 19:40:29 DEBUG "0.05676666666663645" 19:40:29 DEBUG ] 19:40:29 DEBUG } 19:40:29 INFO [http_cmd]: http status code OK 19:40:29 DEBUG ] 19:40:29 INFO [http_cmd]: http status code OK 19:40:29 INFO [http_cmd]: http status code OK 19:40:29 INFO [http_cmd]: http status code OK 19:40:29 DEBUG --- status code --- 19:40:29 INFO [http_cmd]: http status code OK 19:40:29 DEBUG } 19:40:29 DEBUG --- status code --- 19:40:29 DEBUG --- status code --- 19:40:29 DEBUG --- status code --- 19:40:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:29 DEBUG --- status code --- 19:40:29 DEBUG } 19:40:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:29 DEBUG --- http response --- 19:40:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:29 INFO [http_cmd]: http status code OK 19:40:29 DEBUG --- http response --- 19:40:29 INFO [http_cmd]: http status code OK 19:40:29 INFO [http_cmd]: http status code OK 19:40:29 DEBUG --- http response --- 19:40:29 DEBUG --- http response --- 19:40:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:40:29 DEBUG --- http response --- 19:40:29 DEBUG --- status code --- 19:40:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:40:29 DEBUG --- status code --- 19:40:29 DEBUG --- status code --- 19:40:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:40:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:40:29 DEBUG ----- output ----- 19:40:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:40:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:29 DEBUG ----- output ----- 19:40:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:29 DEBUG ----- output ----- 19:40:29 DEBUG ----- output ----- 19:40:29 DEBUG { 19:40:29 DEBUG ----- output ----- 19:40:29 DEBUG --- http response --- 19:40:29 DEBUG { 19:40:29 DEBUG --- http response --- 19:40:29 DEBUG --- http response --- 19:40:29 DEBUG { 19:40:29 DEBUG { 19:40:29 DEBUG "status": "success", 19:40:29 DEBUG { 19:40:29 DEBUG "status": "success", 19:40:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:40:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:40:29 DEBUG "status": "success", 19:40:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:40:29 DEBUG "status": "success", 19:40:29 DEBUG "data": { 19:40:29 DEBUG "status": "success", 19:40:29 DEBUG "data": { 19:40:29 DEBUG ----- output ----- 19:40:29 DEBUG ----- output ----- 19:40:29 DEBUG "data": { 19:40:29 DEBUG ----- output ----- 19:40:29 DEBUG "data": { 19:40:29 DEBUG "resultType": "vector", 19:40:29 DEBUG "data": { 19:40:29 DEBUG "resultType": "vector", 19:40:29 DEBUG { 19:40:29 DEBUG { 19:40:29 DEBUG "resultType": "vector", 19:40:29 DEBUG { 19:40:29 DEBUG "resultType": "vector", 19:40:29 DEBUG "result": [ 19:40:29 DEBUG "resultType": "vector", 19:40:29 DEBUG "result": [ 19:40:29 DEBUG "status": "success", 19:40:29 DEBUG "status": "success", 19:40:29 DEBUG "result": [ 19:40:29 DEBUG "status": "success", 19:40:29 DEBUG "result": [ 19:40:29 DEBUG { 19:40:29 DEBUG "result": [ 19:40:29 DEBUG { 19:40:29 DEBUG "data": { 19:40:29 DEBUG "data": { 19:40:29 DEBUG { 19:40:29 DEBUG "data": { 19:40:29 DEBUG { 19:40:29 DEBUG "metric": { 19:40:29 DEBUG { 19:40:29 DEBUG "metric": { 19:40:29 DEBUG "resultType": "vector", 19:40:29 DEBUG "resultType": "vector", 19:40:29 DEBUG "metric": { 19:40:29 DEBUG "resultType": "vector", 19:40:29 DEBUG "metric": { 19:40:29 DEBUG "pod": "ds-idrepo-0" 19:40:29 DEBUG "metric": { 19:40:29 DEBUG "pod": "ds-idrepo-0" 19:40:29 DEBUG "result": [ 19:40:29 DEBUG "result": [ 19:40:29 DEBUG "instance": "10.142.0.107:9100" 19:40:29 DEBUG "result": [ 19:40:29 DEBUG "instance": "10.142.0.107:9100" 19:40:29 DEBUG }, 19:40:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:40:29 DEBUG }, 19:40:29 DEBUG { 19:40:29 DEBUG { 19:40:29 DEBUG }, 19:40:29 DEBUG { 19:40:29 DEBUG }, 19:40:29 DEBUG "value": [ 19:40:29 DEBUG }, 19:40:29 DEBUG "value": [ 19:40:29 DEBUG "metric": { 19:40:29 DEBUG "metric": { 19:40:29 DEBUG "value": [ 19:40:29 DEBUG "metric": { 19:40:29 DEBUG "value": [ 19:40:29 DEBUG 19:40:29 DEBUG "value": [ 19:40:29 DEBUG 19:40:29 DEBUG "instance": "10.142.0.107:9100" 19:40:29 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:40:29 DEBUG 19:40:29 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG }, 19:40:29 DEBUG }, 19:40:29 DEBUG 19:40:29 DEBUG }, 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG "value": [ 19:40:29 DEBUG "value": [ 19:40:29 DEBUG 19:40:29 DEBUG "value": [ 19:40:29 DEBUG 19:40:29 DEBUG [...] 19:40:29 DEBUG 19:40:29 DEBUG [...] 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG [...] 19:40:29 DEBUG 19:40:29 DEBUG [...] 19:40:29 DEBUG 19:40:29 DEBUG [...] 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG [...] 19:40:29 DEBUG [...] 19:40:29 DEBUG 19:40:29 DEBUG [...] 19:40:29 DEBUG 19:40:29 DEBUG "pod": "ds-idrepo-2" 19:40:29 DEBUG 19:40:29 DEBUG "pod": "ds-cts-2" 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG "instance": "10.142.0.43:9100" 19:40:29 DEBUG 19:40:29 DEBUG "instance": "10.142.0.43:9100" 19:40:29 DEBUG }, 19:40:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:40:29 DEBUG }, 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG }, 19:40:29 DEBUG 19:40:29 DEBUG }, 19:40:29 DEBUG "value": [ 19:40:29 DEBUG }, 19:40:29 DEBUG "value": [ 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG "value": [ 19:40:29 DEBUG 19:40:29 DEBUG "value": [ 19:40:29 DEBUG 1713552029, 19:40:29 DEBUG "value": [ 19:40:29 DEBUG 1713552029, 19:40:29 DEBUG "instance": "10.142.0.43:9100" 19:40:29 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 19:40:29 DEBUG 1713552029, 19:40:29 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 19:40:29 DEBUG 1713552029, 19:40:29 DEBUG "0" 19:40:29 DEBUG 1713552029, 19:40:29 DEBUG "0" 19:40:29 DEBUG }, 19:40:29 DEBUG }, 19:40:29 DEBUG "70622270.83333333" 19:40:29 DEBUG }, 19:40:29 DEBUG "1931703340032" 19:40:29 DEBUG ] 19:40:29 DEBUG "7.67124369238158" 19:40:29 DEBUG ] 19:40:29 DEBUG "value": [ 19:40:29 DEBUG "value": [ 19:40:29 DEBUG ] 19:40:29 DEBUG "value": [ 19:40:29 DEBUG ] 19:40:29 DEBUG } 19:40:29 DEBUG ] 19:40:29 DEBUG } 19:40:29 DEBUG 1713552029, 19:40:29 DEBUG 1713552029, 19:40:29 DEBUG } 19:40:29 DEBUG 1713552029, 19:40:29 DEBUG } 19:40:29 DEBUG ] 19:40:29 DEBUG } 19:40:29 DEBUG ] 19:40:29 DEBUG "0.03366666666666826" 19:40:29 DEBUG "0" 19:40:29 DEBUG ] 19:40:29 DEBUG "0" 19:40:29 DEBUG ] 19:40:29 DEBUG } 19:40:29 DEBUG ] 19:40:29 DEBUG } 19:40:29 DEBUG ] 19:40:29 DEBUG ] 19:40:29 DEBUG } 19:40:29 DEBUG ] 19:40:29 DEBUG } 19:40:29 DEBUG } 19:40:29 DEBUG } 19:40:29 DEBUG } 19:40:29 DEBUG } 19:40:29 DEBUG } 19:40:29 DEBUG } 19:40:29 DEBUG } 19:40:29 DEBUG } 19:40:29 DEBUG } 19:40:29 DEBUG ] 19:40:29 DEBUG ] 19:40:29 DEBUG ] 19:40:29 DEBUG } 19:40:29 DEBUG } 19:40:29 DEBUG } 19:40:29 DEBUG } 19:40:29 DEBUG } 19:40:29 DEBUG } 19:40:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552029" 19:40:29 INFO [http_cmd]: http status code OK 19:40:29 DEBUG --- status code --- 19:40:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:29 DEBUG --- http response --- 19:40:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:40:29 DEBUG ----- output ----- 19:40:29 DEBUG { 19:40:29 DEBUG "status": "success", 19:40:29 DEBUG "data": { 19:40:29 DEBUG "resultType": "vector", 19:40:29 DEBUG "result": [ 19:40:29 DEBUG { 19:40:29 DEBUG "metric": { 19:40:29 DEBUG "pod": "ds-idrepo-0" 19:40:29 DEBUG }, 19:40:29 DEBUG "value": [ 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG [...] 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG 19:40:29 DEBUG "pod": "ds-idrepo-2" 19:40:29 DEBUG }, 19:40:29 DEBUG "value": [ 19:40:29 DEBUG 1713552029, 19:40:29 DEBUG "0" 19:40:29 DEBUG ] 19:40:29 DEBUG } 19:40:29 DEBUG ] 19:40:29 DEBUG } 19:40:29 DEBUG } 19:40:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552030" 19:40:30 INFO [http_cmd]: http status code OK 19:40:30 DEBUG --- status code --- 19:40:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:30 DEBUG --- http response --- 19:40:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:40:30 DEBUG ----- output ----- 19:40:30 DEBUG { 19:40:30 DEBUG "status": "success", 19:40:30 DEBUG "data": { 19:40:30 DEBUG "resultType": "vector", 19:40:30 DEBUG "result": [ 19:40:30 DEBUG { 19:40:30 DEBUG "metric": { 19:40:30 DEBUG "pod": "ds-cts-0" 19:40:30 DEBUG }, 19:40:30 DEBUG "value": [ 19:40:30 DEBUG 19:40:30 DEBUG 19:40:30 DEBUG 19:40:30 DEBUG [...] 19:40:30 DEBUG 19:40:30 DEBUG 19:40:30 DEBUG 19:40:30 DEBUG "pod": "ds-cts-2" 19:40:30 DEBUG }, 19:40:30 DEBUG "value": [ 19:40:30 DEBUG 1713552030, 19:40:30 DEBUG "29.733333333333334" 19:40:30 DEBUG ] 19:40:30 DEBUG } 19:40:30 DEBUG ] 19:40:30 DEBUG } 19:40:30 DEBUG } 19:40:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552030" 19:40:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552030" 19:40:30 INFO [http_cmd]: http status code OK 19:40:30 DEBUG --- status code --- 19:40:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:30 DEBUG --- http response --- 19:40:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:40:30 DEBUG ----- output ----- 19:40:30 DEBUG { 19:40:30 DEBUG "status": "success", 19:40:30 DEBUG "data": { 19:40:30 DEBUG "resultType": "vector", 19:40:30 DEBUG "result": [ 19:40:30 DEBUG { 19:40:30 DEBUG "metric": { 19:40:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:40:30 DEBUG }, 19:40:30 DEBUG "value": [ 19:40:30 DEBUG 19:40:30 DEBUG 19:40:30 DEBUG 19:40:30 DEBUG [...] 19:40:30 DEBUG 19:40:30 DEBUG 19:40:30 DEBUG 19:40:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:40:30 DEBUG }, 19:40:30 DEBUG "value": [ 19:40:30 DEBUG 1713552030, 19:40:30 DEBUG "2330.366666666667" 19:40:30 DEBUG ] 19:40:30 DEBUG } 19:40:30 DEBUG ] 19:40:30 DEBUG } 19:40:30 DEBUG } 19:40:30 INFO [http_cmd]: http status code OK 19:40:30 DEBUG --- status code --- 19:40:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:30 DEBUG --- http response --- 19:40:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:40:30 DEBUG ----- output ----- 19:40:30 DEBUG { 19:40:30 DEBUG "status": "success", 19:40:30 DEBUG "data": { 19:40:30 DEBUG "resultType": "vector", 19:40:30 DEBUG "result": [ 19:40:30 DEBUG { 19:40:30 DEBUG "metric": { 19:40:30 DEBUG "pod": "ds-cts-0" 19:40:30 DEBUG }, 19:40:30 DEBUG "value": [ 19:40:30 DEBUG 19:40:30 DEBUG 19:40:30 DEBUG 19:40:30 DEBUG [...] 19:40:30 DEBUG 19:40:30 DEBUG 19:40:30 DEBUG 19:40:30 DEBUG "pod": "ds-cts-2" 19:40:30 DEBUG }, 19:40:30 DEBUG "value": [ 19:40:30 DEBUG 1713552030, 19:40:30 DEBUG "0.0003575422222222222" 19:40:30 DEBUG ] 19:40:30 DEBUG } 19:40:30 DEBUG ] 19:40:30 DEBUG } 19:40:30 DEBUG } 19:40:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713552030" 19:40:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552030" 19:40:30 INFO [http_cmd]: http status code OK 19:40:30 DEBUG --- status code --- 19:40:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:30 DEBUG --- http response --- 19:40:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:40:30 DEBUG ----- output ----- 19:40:30 DEBUG { 19:40:30 DEBUG "status": "success", 19:40:30 DEBUG "data": { 19:40:30 DEBUG "resultType": "vector", 19:40:30 DEBUG "result": [ 19:40:30 DEBUG { 19:40:30 DEBUG "metric": { 19:40:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:40:30 DEBUG }, 19:40:30 DEBUG "value": [ 19:40:30 DEBUG 19:40:30 DEBUG 19:40:30 DEBUG 19:40:30 DEBUG [...] 19:40:30 DEBUG 19:40:30 DEBUG 19:40:30 DEBUG 19:40:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:40:30 DEBUG }, 19:40:30 DEBUG "value": [ 19:40:30 DEBUG 1713552030, 19:40:30 DEBUG "0.0115" 19:40:30 DEBUG ] 19:40:30 DEBUG } 19:40:30 DEBUG ] 19:40:30 DEBUG } 19:40:30 DEBUG } 19:40:30 INFO [http_cmd]: http status code OK 19:40:30 DEBUG --- status code --- 19:40:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:30 DEBUG --- http response --- 19:40:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:40:30 DEBUG ----- output ----- 19:40:30 DEBUG { 19:40:30 DEBUG "status": "success", 19:40:30 DEBUG "data": { 19:40:30 DEBUG "resultType": "vector", 19:40:30 DEBUG "result": [ 19:40:30 DEBUG { 19:40:30 DEBUG "metric": { 19:40:30 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:40:30 DEBUG }, 19:40:30 DEBUG "value": [ 19:40:30 DEBUG 19:40:30 DEBUG 19:40:30 DEBUG 19:40:30 DEBUG [...] 19:40:30 DEBUG 19:40:30 DEBUG 19:40:30 DEBUG 19:40:30 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 19:40:30 DEBUG }, 19:40:30 DEBUG "value": [ 19:40:30 DEBUG 1713552030, 19:40:30 DEBUG "0" 19:40:30 DEBUG ] 19:40:30 DEBUG } 19:40:30 DEBUG ] 19:40:30 DEBUG } 19:40:30 DEBUG } 19:40:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552030" 19:40:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713552030" 19:40:30 INFO [http_cmd]: http status code OK 19:40:30 DEBUG --- status code --- 19:40:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:30 DEBUG --- http response --- 19:40:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:40:30 DEBUG ----- output ----- 19:40:30 DEBUG { 19:40:30 DEBUG "status": "success", 19:40:30 DEBUG "data": { 19:40:30 DEBUG "resultType": "vector", 19:40:30 DEBUG "result": [ 19:40:30 DEBUG { 19:40:30 DEBUG "metric": { 19:40:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:40:30 DEBUG }, 19:40:30 DEBUG "value": [ 19:40:30 DEBUG 19:40:30 DEBUG 19:40:30 DEBUG 19:40:30 DEBUG [...] 19:40:30 DEBUG 19:40:30 DEBUG 19:40:30 DEBUG 19:40:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:40:30 DEBUG }, 19:40:30 DEBUG "value": [ 19:40:30 DEBUG 1713552030, 19:40:30 DEBUG "2330.4666666666667" 19:40:30 DEBUG ] 19:40:30 DEBUG } 19:40:30 DEBUG ] 19:40:30 DEBUG } 19:40:30 DEBUG } 19:40:30 INFO [http_cmd]: http status code OK 19:40:30 DEBUG --- status code --- 19:40:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:30 DEBUG --- http response --- 19:40:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:40:30 DEBUG ----- output ----- 19:40:30 DEBUG { 19:40:30 DEBUG "status": "success", 19:40:30 DEBUG "data": { 19:40:30 DEBUG "resultType": "vector", 19:40:30 DEBUG "result": [ 19:40:30 DEBUG { 19:40:30 DEBUG "metric": { 19:40:30 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:40:30 DEBUG }, 19:40:30 DEBUG "value": [ 19:40:30 DEBUG 19:40:30 DEBUG 19:40:30 DEBUG 19:40:30 DEBUG [...] 19:40:30 DEBUG 19:40:30 DEBUG 19:40:30 DEBUG 19:40:30 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:40:30 DEBUG }, 19:40:30 DEBUG "value": [ 19:40:30 DEBUG 1713552030, 19:40:30 DEBUG "3413282816" 19:40:30 DEBUG ] 19:40:30 DEBUG } 19:40:30 DEBUG ] 19:40:30 DEBUG } 19:40:30 DEBUG } 19:40:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552030" 19:40:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552030" 19:40:30 INFO [http_cmd]: http status code OK 19:40:30 DEBUG --- status code --- 19:40:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:30 DEBUG --- http response --- 19:40:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:40:30 DEBUG ----- output ----- 19:40:30 DEBUG { 19:40:30 DEBUG "status": "success", 19:40:30 DEBUG "data": { 19:40:30 DEBUG "resultType": "vector", 19:40:30 DEBUG "result": [ 19:40:30 DEBUG { 19:40:30 DEBUG "metric": { 19:40:30 DEBUG "instance": "10.142.0.107:9100" 19:40:30 DEBUG }, 19:40:30 DEBUG "value": [ 19:40:30 DEBUG 19:40:30 DEBUG 19:40:30 DEBUG 19:40:30 DEBUG [...] 19:40:30 DEBUG 19:40:30 DEBUG 19:40:30 DEBUG 19:40:30 DEBUG "instance": "10.142.0.43:9100" 19:40:30 DEBUG }, 19:40:30 DEBUG "value": [ 19:40:30 DEBUG 1713552030, 19:40:30 DEBUG "70620268.3" 19:40:30 DEBUG ] 19:40:30 DEBUG } 19:40:30 DEBUG ] 19:40:30 DEBUG } 19:40:30 DEBUG } 19:40:30 INFO [http_cmd]: http status code OK 19:40:30 DEBUG --- status code --- 19:40:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:30 DEBUG --- http response --- 19:40:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:40:30 DEBUG ----- output ----- 19:40:30 DEBUG { 19:40:30 DEBUG "status": "success", 19:40:30 DEBUG "data": { 19:40:30 DEBUG "resultType": "vector", 19:40:30 DEBUG "result": [ 19:40:30 DEBUG { 19:40:30 DEBUG "metric": { 19:40:30 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:40:30 DEBUG }, 19:40:30 DEBUG "value": [ 19:40:30 DEBUG 19:40:30 DEBUG 19:40:30 DEBUG 19:40:30 DEBUG [...] 19:40:30 DEBUG 19:40:30 DEBUG 19:40:30 DEBUG 19:40:30 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 19:40:30 DEBUG }, 19:40:30 DEBUG "value": [ 19:40:30 DEBUG 1713552030, 19:40:30 DEBUG "0.000029024754365882134" 19:40:30 DEBUG ] 19:40:30 DEBUG } 19:40:30 DEBUG ] 19:40:30 DEBUG } 19:40:30 DEBUG } 19:40:30 INFO 19:40:30 INFO [loop_until]: kubectl --namespace=xlou top pods 19:40:30 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:40:30 INFO [loop_until]: OK (rc = 0) 19:40:30 DEBUG --- stdout --- 19:40:30 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 14402m 8994Mi am-767f6b67b5-qcqz2 14343m 9004Mi am-767f6b67b5-vbltz 15453m 8981Mi ds-cts-0 15264m 10912Mi ds-cts-1 13935m 11158Mi ds-cts-2 15755m 10909Mi ds-idrepo-0 6520m 3663Mi ds-idrepo-1 6223m 3533Mi ds-idrepo-2 6514m 3492Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7822m 3164Mi 19:40:30 DEBUG --- stderr --- 19:40:30 DEBUG 19:40:31 INFO 19:40:31 INFO [loop_until]: kubectl --namespace=xlou top node 19:40:31 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:40:31 INFO [loop_until]: OK (rc = 0) 19:40:31 DEBUG --- stdout --- 19:40:31 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 69m 0% 6249Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 6508m 21% 7249Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 14266m 47% 11708Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 3345Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 15249m 51% 13904Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 142m 0% 5219Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 14282m 47% 11884Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12303m 41% 2978Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 16313m 54% 11193Mi 10% gke-xlou-cdm-ds-0f601751-j488 7000m 23% 6733Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 13755m 46% 15947Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 6681m 22% 7431Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 15687m 52% 13384Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 7875m 49% 6938Mi 11% 19:40:31 DEBUG --- stderr --- 19:40:31 DEBUG 19:41:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552089" 19:41:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552089" 19:41:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552089" 19:41:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552089" 19:41:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713552089" 19:41:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552089" 19:41:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552089" 19:41:29 INFO [http_cmd]: http status code OK 19:41:29 ERROR [http_cmd]: ERROR 19:41:29 INFO [http_cmd]: http status code OK 19:41:29 DEBUG --- status code --- 19:41:29 INFO [http_cmd]: http status code OK 19:41:29 INFO [http_cmd]: http status code OK 19:41:29 ERROR --- status code --- 19:41:29 DEBUG --- status code --- 19:41:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:29 DEBUG --- status code --- 19:41:29 DEBUG --- status code --- 19:41:29 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:41:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:29 DEBUG --- http response --- 19:41:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:29 ERROR --- http response --- 19:41:29 DEBUG --- http response --- 19:41:29 DEBUG --- http response --- 19:41:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:41:29 DEBUG --- http response --- 19:41:29 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:41:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:41:29 DEBUG ----- output ----- 19:41:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:41:29 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:41:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:41:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552089" 19:41:29 DEBUG ----- output ----- 19:41:29 DEBUG { 19:41:29 DEBUG ----- output ----- 19:41:29 DEBUG ----- output ----- 19:41:29 DEBUG { 19:41:29 DEBUG "status": "success", 19:41:29 DEBUG { 19:41:29 DEBUG { 19:41:29 DEBUG "status": "success", 19:41:29 DEBUG "data": { 19:41:29 DEBUG "status": "success", 19:41:29 DEBUG "status": "success", 19:41:29 DEBUG "data": { 19:41:29 DEBUG "resultType": "vector", 19:41:29 DEBUG "data": { 19:41:29 DEBUG "data": { 19:41:29 DEBUG "resultType": "vector", 19:41:29 DEBUG "result": [ 19:41:29 DEBUG "resultType": "vector", 19:41:29 DEBUG "resultType": "vector", 19:41:29 DEBUG "result": [ 19:41:29 DEBUG { 19:41:29 DEBUG "result": [ 19:41:29 DEBUG "result": [ 19:41:29 DEBUG { 19:41:29 DEBUG "metric": { 19:41:29 DEBUG { 19:41:29 DEBUG { 19:41:29 DEBUG "metric": { 19:41:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:41:29 DEBUG "metric": { 19:41:29 DEBUG "metric": { 19:41:29 DEBUG "pod": "ds-cts-0" 19:41:29 DEBUG }, 19:41:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:41:29 DEBUG "pod": "ds-idrepo-0" 19:41:29 DEBUG }, 19:41:29 DEBUG "value": [ 19:41:29 DEBUG }, 19:41:29 DEBUG }, 19:41:29 DEBUG "value": [ 19:41:29 DEBUG 19:41:29 DEBUG "value": [ 19:41:29 DEBUG "value": [ 19:41:29 DEBUG 19:41:29 DEBUG 19:41:29 DEBUG 19:41:29 DEBUG 19:41:29 DEBUG 19:41:29 DEBUG 19:41:29 DEBUG 19:41:29 DEBUG 19:41:29 DEBUG 19:41:29 DEBUG [...] 19:41:29 DEBUG 19:41:29 DEBUG 19:41:29 DEBUG [...] 19:41:29 DEBUG 19:41:29 DEBUG [...] 19:41:29 DEBUG [...] 19:41:29 DEBUG 19:41:29 DEBUG 19:41:29 DEBUG 19:41:29 DEBUG 19:41:29 DEBUG 19:41:29 DEBUG 19:41:29 DEBUG 19:41:29 DEBUG 19:41:29 DEBUG 19:41:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:41:29 DEBUG 19:41:29 DEBUG 19:41:29 DEBUG "pod": "ds-cts-2" 19:41:29 DEBUG }, 19:41:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:41:29 DEBUG "pod": "ds-idrepo-2" 19:41:29 DEBUG }, 19:41:29 DEBUG "value": [ 19:41:29 DEBUG }, 19:41:29 DEBUG }, 19:41:29 DEBUG "value": [ 19:41:29 DEBUG 1713552089, 19:41:29 DEBUG "value": [ 19:41:29 DEBUG "value": [ 19:41:29 DEBUG 1713552089, 19:41:29 DEBUG "419725" 19:41:29 DEBUG 1713552089, 19:41:29 DEBUG 1713552089, 19:41:29 DEBUG "0" 19:41:29 DEBUG ] 19:41:29 DEBUG "2333.2" 19:41:29 DEBUG "0" 19:41:29 DEBUG ] 19:41:29 DEBUG } 19:41:29 DEBUG ] 19:41:29 DEBUG ] 19:41:29 DEBUG } 19:41:29 DEBUG ] 19:41:29 DEBUG } 19:41:29 DEBUG } 19:41:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552089" 19:41:29 DEBUG ] 19:41:29 DEBUG } 19:41:29 DEBUG ] 19:41:29 DEBUG ] 19:41:29 DEBUG } 19:41:29 DEBUG } 19:41:29 DEBUG } 19:41:29 DEBUG } 19:41:29 INFO [http_cmd]: http status code OK 19:41:29 DEBUG } 19:41:29 INFO [http_cmd]: http status code OK 19:41:29 DEBUG } 19:41:29 DEBUG } 19:41:29 DEBUG --- status code --- 19:41:29 DEBUG --- status code --- 19:41:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:29 DEBUG --- http response --- 19:41:29 DEBUG --- http response --- 19:41:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:41:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:41:29 INFO [http_cmd]: http status code OK 19:41:29 DEBUG ----- output ----- 19:41:29 DEBUG ----- output ----- 19:41:29 DEBUG --- status code --- 19:41:29 DEBUG { 19:41:29 DEBUG { 19:41:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:29 DEBUG "status": "success", 19:41:29 DEBUG "status": "success", 19:41:29 DEBUG --- http response --- 19:41:29 DEBUG "data": { 19:41:29 DEBUG "data": { 19:41:29 DEBUG "resultType": "vector", 19:41:29 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:41:29 DEBUG "resultType": "vector", 19:41:29 DEBUG "result": [ 19:41:29 DEBUG "result": [ 19:41:29 DEBUG { 19:41:29 DEBUG { 19:41:29 DEBUG "metric": { 19:41:29 DEBUG "metric": { 19:41:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:41:29 DEBUG "pod": "am-767f6b67b5-9zw22" 19:41:29 DEBUG }, 19:41:29 DEBUG }, 19:41:29 DEBUG "value": [ 19:41:29 DEBUG "value": [ 19:41:29 DEBUG 19:41:29 DEBUG 19:41:29 DEBUG 19:41:29 DEBUG 19:41:29 DEBUG 19:41:29 DEBUG 19:41:29 DEBUG [...] 19:41:29 DEBUG [...] 19:41:29 DEBUG 19:41:29 DEBUG 19:41:29 DEBUG 19:41:29 DEBUG 19:41:29 DEBUG 19:41:29 DEBUG 19:41:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:41:29 DEBUG "pod": "am-767f6b67b5-vbltz" 19:41:29 DEBUG }, 19:41:29 DEBUG }, 19:41:29 DEBUG "value": [ 19:41:29 DEBUG "value": [ 19:41:29 DEBUG 1713552089, 19:41:29 DEBUG 1713552089, 19:41:29 DEBUG "0" 19:41:29 DEBUG "2329.7" 19:41:29 DEBUG ] 19:41:29 DEBUG ] 19:41:29 DEBUG } 19:41:29 DEBUG } 19:41:29 DEBUG ] 19:41:29 DEBUG ] 19:41:29 DEBUG } 19:41:29 DEBUG } 19:41:29 DEBUG } 19:41:29 DEBUG } 19:41:29 INFO [http_cmd]: http status code OK 19:41:29 DEBUG --- status code --- 19:41:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:29 DEBUG --- http response --- 19:41:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:41:29 DEBUG ----- output ----- 19:41:29 DEBUG { 19:41:29 DEBUG "status": "success", 19:41:29 DEBUG "data": { 19:41:29 DEBUG "resultType": "vector", 19:41:29 DEBUG "result": [ 19:41:29 DEBUG { 19:41:29 DEBUG "metric": { 19:41:29 DEBUG "instance": "10.142.0.107:9100" 19:41:29 DEBUG }, 19:41:29 DEBUG "value": [ 19:41:29 DEBUG 19:41:29 DEBUG 19:41:29 DEBUG 19:41:29 DEBUG [...] 19:41:29 DEBUG 19:41:29 DEBUG 19:41:29 DEBUG 19:41:29 DEBUG "instance": "10.142.0.43:9100" 19:41:29 DEBUG }, 19:41:29 DEBUG "value": [ 19:41:29 DEBUG 1713552089, 19:41:29 DEBUG "0.27169999999998706" 19:41:29 DEBUG ] 19:41:29 DEBUG } 19:41:29 DEBUG ] 19:41:29 DEBUG } 19:41:29 DEBUG } 19:41:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552090" 19:41:30 INFO [http_cmd]: http status code OK 19:41:30 DEBUG --- status code --- 19:41:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:30 DEBUG --- http response --- 19:41:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:41:30 DEBUG ----- output ----- 19:41:30 DEBUG { 19:41:30 DEBUG "status": "success", 19:41:30 DEBUG "data": { 19:41:30 DEBUG "resultType": "vector", 19:41:30 DEBUG "result": [ 19:41:30 DEBUG { 19:41:30 DEBUG "metric": { 19:41:30 DEBUG "instance": "10.142.0.107:9100" 19:41:30 DEBUG }, 19:41:30 DEBUG "value": [ 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG [...] 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG "instance": "10.142.0.43:9100" 19:41:30 DEBUG }, 19:41:30 DEBUG "value": [ 19:41:30 DEBUG 1713552090, 19:41:30 DEBUG "70324298.96666667" 19:41:30 DEBUG ] 19:41:30 DEBUG } 19:41:30 DEBUG ] 19:41:30 DEBUG } 19:41:30 DEBUG } 19:41:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552090" 19:41:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713552090" 19:41:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713552090" 19:41:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552090" 19:41:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552090" 19:41:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552090" 19:41:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552090" 19:41:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552090" 19:41:30 INFO [http_cmd]: http status code OK 19:41:30 INFO [http_cmd]: http status code OK 19:41:30 DEBUG --- status code --- 19:41:30 DEBUG --- status code --- 19:41:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:30 DEBUG --- http response --- 19:41:30 DEBUG --- http response --- 19:41:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:41:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:41:30 DEBUG ----- output ----- 19:41:30 DEBUG ----- output ----- 19:41:30 DEBUG { 19:41:30 DEBUG { 19:41:30 INFO [http_cmd]: http status code OK 19:41:30 DEBUG "status": "success", 19:41:30 DEBUG "status": "success", 19:41:30 DEBUG --- status code --- 19:41:30 DEBUG "data": { 19:41:30 INFO [http_cmd]: http status code OK 19:41:30 DEBUG "data": { 19:41:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:30 DEBUG "resultType": "vector", 19:41:30 DEBUG --- status code --- 19:41:30 DEBUG "resultType": "vector", 19:41:30 DEBUG --- http response --- 19:41:30 DEBUG "result": [ 19:41:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:30 DEBUG "result": [ 19:41:30 DEBUG { 19:41:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:41:30 DEBUG --- http response --- 19:41:30 DEBUG { 19:41:30 DEBUG "metric": { 19:41:30 DEBUG ----- output ----- 19:41:30 DEBUG "metric": { 19:41:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:41:30 DEBUG "pod": "ds-idrepo-0" 19:41:30 DEBUG { 19:41:30 DEBUG "pod": "ds-idrepo-0" 19:41:30 DEBUG ----- output ----- 19:41:30 DEBUG }, 19:41:30 DEBUG "status": "success", 19:41:30 DEBUG }, 19:41:30 DEBUG { 19:41:30 DEBUG "value": [ 19:41:30 DEBUG "data": { 19:41:30 DEBUG "value": [ 19:41:30 DEBUG "status": "success", 19:41:30 DEBUG 19:41:30 DEBUG "resultType": "vector", 19:41:30 DEBUG 19:41:30 DEBUG "data": { 19:41:30 DEBUG "resultType": "vector", 19:41:30 DEBUG "result": [ 19:41:30 DEBUG "result": [ 19:41:30 DEBUG 19:41:30 INFO [http_cmd]: http status code OK 19:41:30 DEBUG 19:41:30 DEBUG { 19:41:30 INFO [http_cmd]: http status code OK 19:41:30 INFO [http_cmd]: http status code OK 19:41:30 DEBUG { 19:41:30 DEBUG 19:41:30 DEBUG --- status code --- 19:41:30 DEBUG 19:41:30 DEBUG "metric": { 19:41:30 DEBUG --- status code --- 19:41:30 DEBUG --- status code --- 19:41:30 DEBUG "metric": { 19:41:30 DEBUG [...] 19:41:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:30 DEBUG [...] 19:41:30 DEBUG "pod": "ds-idrepo-0" 19:41:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:30 DEBUG "instance": "10.142.0.107:9100" 19:41:30 DEBUG 19:41:30 DEBUG --- http response --- 19:41:30 DEBUG 19:41:30 DEBUG }, 19:41:30 DEBUG --- http response --- 19:41:30 DEBUG --- http response --- 19:41:30 DEBUG }, 19:41:30 INFO [http_cmd]: http status code OK 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:41:30 DEBUG "value": [ 19:41:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:41:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:41:30 DEBUG "value": [ 19:41:30 DEBUG --- status code --- 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG ----- output ----- 19:41:30 DEBUG 19:41:30 DEBUG ----- output ----- 19:41:30 DEBUG ----- output ----- 19:41:30 DEBUG 19:41:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:30 DEBUG "pod": "ds-cts-2" 19:41:30 DEBUG "pod": "ds-idrepo-2" 19:41:30 DEBUG { 19:41:30 DEBUG 19:41:30 DEBUG { 19:41:30 DEBUG { 19:41:30 DEBUG 19:41:30 DEBUG --- http response --- 19:41:30 DEBUG }, 19:41:30 DEBUG }, 19:41:30 DEBUG "status": "success", 19:41:30 DEBUG 19:41:30 DEBUG "status": "success", 19:41:30 DEBUG "status": "success", 19:41:30 DEBUG 19:41:30 DEBUG "value": [ 19:41:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:41:30 DEBUG "value": [ 19:41:30 DEBUG "data": { 19:41:30 DEBUG [...] 19:41:30 DEBUG "data": { 19:41:30 DEBUG "data": { 19:41:30 DEBUG [...] 19:41:30 DEBUG 1713552090, 19:41:30 DEBUG ----- output ----- 19:41:30 DEBUG 1713552090, 19:41:30 DEBUG "resultType": "vector", 19:41:30 DEBUG 19:41:30 DEBUG "resultType": "vector", 19:41:30 DEBUG "resultType": "vector", 19:41:30 DEBUG 19:41:30 DEBUG "0" 19:41:30 DEBUG { 19:41:30 DEBUG "0" 19:41:30 DEBUG "result": [ 19:41:30 DEBUG 19:41:30 DEBUG "result": [ 19:41:30 DEBUG "result": [ 19:41:30 DEBUG 19:41:30 DEBUG ] 19:41:30 DEBUG "status": "success", 19:41:30 DEBUG ] 19:41:30 DEBUG { 19:41:30 DEBUG 19:41:30 DEBUG { 19:41:30 DEBUG { 19:41:30 DEBUG 19:41:30 DEBUG } 19:41:30 DEBUG "data": { 19:41:30 DEBUG } 19:41:30 DEBUG "metric": { 19:41:30 DEBUG "pod": "ds-idrepo-2" 19:41:30 DEBUG "metric": { 19:41:30 DEBUG "metric": { 19:41:30 DEBUG "instance": "10.142.0.43:9100" 19:41:30 DEBUG ] 19:41:30 DEBUG "resultType": "vector", 19:41:30 DEBUG ] 19:41:30 DEBUG "instance": "10.142.0.107:9100" 19:41:30 DEBUG }, 19:41:30 DEBUG "pod": "ds-idrepo-0" 19:41:30 DEBUG "pod": "ds-idrepo-0" 19:41:30 DEBUG }, 19:41:30 DEBUG } 19:41:30 DEBUG "result": [ 19:41:30 DEBUG } 19:41:30 DEBUG }, 19:41:30 DEBUG "value": [ 19:41:30 DEBUG }, 19:41:30 DEBUG }, 19:41:30 DEBUG "value": [ 19:41:30 DEBUG } 19:41:30 DEBUG { 19:41:30 DEBUG } 19:41:30 DEBUG "value": [ 19:41:30 DEBUG 1713552090, 19:41:30 DEBUG "value": [ 19:41:30 DEBUG "value": [ 19:41:30 DEBUG 1713552090, 19:41:30 DEBUG "metric": { 19:41:30 DEBUG 19:41:30 DEBUG "4337.068737315803" 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG "1935123932160" 19:41:30 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:41:30 DEBUG 19:41:30 DEBUG ] 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG ] 19:41:30 DEBUG }, 19:41:30 DEBUG 19:41:30 DEBUG } 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG } 19:41:30 DEBUG "value": [ 19:41:30 DEBUG [...] 19:41:30 DEBUG ] 19:41:30 DEBUG [...] 19:41:30 DEBUG [...] 19:41:30 DEBUG ] 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG } 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG } 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG } 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG } 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG [...] 19:41:30 DEBUG "instance": "10.142.0.43:9100" 19:41:30 DEBUG "pod": "ds-idrepo-2" 19:41:30 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:41:30 DEBUG 19:41:30 DEBUG }, 19:41:30 DEBUG }, 19:41:30 DEBUG }, 19:41:30 DEBUG 19:41:30 DEBUG "value": [ 19:41:30 DEBUG "value": [ 19:41:30 DEBUG "value": [ 19:41:30 DEBUG 19:41:30 DEBUG 1713552090, 19:41:30 DEBUG 1713552090, 19:41:30 DEBUG 1713552090, 19:41:30 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:41:30 DEBUG "0.11333333333333637" 19:41:30 DEBUG "0" 19:41:30 DEBUG "0" 19:41:30 DEBUG }, 19:41:30 DEBUG ] 19:41:30 DEBUG ] 19:41:30 DEBUG ] 19:41:30 DEBUG "value": [ 19:41:30 DEBUG } 19:41:30 DEBUG } 19:41:30 DEBUG } 19:41:30 DEBUG 1713552090, 19:41:30 DEBUG ] 19:41:30 DEBUG ] 19:41:30 DEBUG ] 19:41:30 DEBUG "7.683273966808756" 19:41:30 DEBUG } 19:41:30 DEBUG } 19:41:30 DEBUG } 19:41:30 DEBUG ] 19:41:30 DEBUG } 19:41:30 DEBUG } 19:41:30 DEBUG } 19:41:30 DEBUG } 19:41:30 DEBUG ] 19:41:30 DEBUG } 19:41:30 DEBUG } 19:41:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552090" 19:41:30 INFO [http_cmd]: http status code OK 19:41:30 DEBUG --- status code --- 19:41:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:30 DEBUG --- http response --- 19:41:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:41:30 DEBUG ----- output ----- 19:41:30 DEBUG { 19:41:30 DEBUG "status": "success", 19:41:30 DEBUG "data": { 19:41:30 DEBUG "resultType": "vector", 19:41:30 DEBUG "result": [ 19:41:30 DEBUG { 19:41:30 DEBUG "metric": { 19:41:30 DEBUG "pod": "ds-cts-0" 19:41:30 DEBUG }, 19:41:30 DEBUG "value": [ 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG [...] 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG "pod": "ds-cts-2" 19:41:30 DEBUG }, 19:41:30 DEBUG "value": [ 19:41:30 DEBUG 1713552090, 19:41:30 DEBUG "29.966666666666665" 19:41:30 DEBUG ] 19:41:30 DEBUG } 19:41:30 DEBUG ] 19:41:30 DEBUG } 19:41:30 DEBUG } 19:41:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552090" 19:41:30 INFO [http_cmd]: http status code OK 19:41:30 DEBUG --- status code --- 19:41:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:30 DEBUG --- http response --- 19:41:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:41:30 DEBUG ----- output ----- 19:41:30 DEBUG { 19:41:30 DEBUG "status": "success", 19:41:30 DEBUG "data": { 19:41:30 DEBUG "resultType": "vector", 19:41:30 DEBUG "result": [ 19:41:30 DEBUG { 19:41:30 DEBUG "metric": { 19:41:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:41:30 DEBUG }, 19:41:30 DEBUG "value": [ 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG [...] 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:41:30 DEBUG }, 19:41:30 DEBUG "value": [ 19:41:30 DEBUG 1713552090, 19:41:30 DEBUG "2329.4333333333334" 19:41:30 DEBUG ] 19:41:30 DEBUG } 19:41:30 DEBUG ] 19:41:30 DEBUG } 19:41:30 DEBUG } 19:41:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552090" 19:41:30 INFO [http_cmd]: http status code OK 19:41:30 DEBUG --- status code --- 19:41:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:30 DEBUG --- http response --- 19:41:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:41:30 DEBUG ----- output ----- 19:41:30 DEBUG { 19:41:30 DEBUG "status": "success", 19:41:30 DEBUG "data": { 19:41:30 DEBUG "resultType": "vector", 19:41:30 DEBUG "result": [ 19:41:30 DEBUG { 19:41:30 DEBUG "metric": { 19:41:30 DEBUG "pod": "ds-cts-0" 19:41:30 DEBUG }, 19:41:30 DEBUG "value": [ 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG [...] 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG "pod": "ds-cts-2" 19:41:30 DEBUG }, 19:41:30 DEBUG "value": [ 19:41:30 DEBUG 1713552090, 19:41:30 DEBUG "0.0006000000000000001" 19:41:30 DEBUG ] 19:41:30 DEBUG } 19:41:30 DEBUG ] 19:41:30 DEBUG } 19:41:30 DEBUG } 19:41:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713552090" 19:41:30 INFO [http_cmd]: http status code OK 19:41:30 DEBUG --- status code --- 19:41:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:30 DEBUG --- http response --- 19:41:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:41:30 DEBUG ----- output ----- 19:41:30 DEBUG { 19:41:30 DEBUG "status": "success", 19:41:30 DEBUG "data": { 19:41:30 DEBUG "resultType": "vector", 19:41:30 DEBUG "result": [ 19:41:30 DEBUG { 19:41:30 DEBUG "metric": { 19:41:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:41:30 DEBUG }, 19:41:30 DEBUG "value": [ 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG [...] 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:41:30 DEBUG }, 19:41:30 DEBUG "value": [ 19:41:30 DEBUG 1713552090, 19:41:30 DEBUG "0.012" 19:41:30 DEBUG ] 19:41:30 DEBUG } 19:41:30 DEBUG ] 19:41:30 DEBUG } 19:41:30 DEBUG } 19:41:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552090" 19:41:30 INFO [http_cmd]: http status code OK 19:41:30 DEBUG --- status code --- 19:41:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:30 DEBUG --- http response --- 19:41:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:41:30 DEBUG ----- output ----- 19:41:30 DEBUG { 19:41:30 DEBUG "status": "success", 19:41:30 DEBUG "data": { 19:41:30 DEBUG "resultType": "vector", 19:41:30 DEBUG "result": [ 19:41:30 DEBUG { 19:41:30 DEBUG "metric": { 19:41:30 DEBUG "pod": "ds-idrepo-0" 19:41:30 DEBUG }, 19:41:30 DEBUG "value": [ 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG [...] 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG "pod": "ds-idrepo-2" 19:41:30 DEBUG }, 19:41:30 DEBUG "value": [ 19:41:30 DEBUG 1713552090, 19:41:30 DEBUG "0" 19:41:30 DEBUG ] 19:41:30 DEBUG } 19:41:30 DEBUG ] 19:41:30 DEBUG } 19:41:30 DEBUG } 19:41:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552090" 19:41:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713552090" 19:41:30 INFO [http_cmd]: http status code OK 19:41:30 DEBUG --- status code --- 19:41:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:30 DEBUG --- http response --- 19:41:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:41:30 DEBUG ----- output ----- 19:41:30 DEBUG { 19:41:30 DEBUG "status": "success", 19:41:30 DEBUG "data": { 19:41:30 DEBUG "resultType": "vector", 19:41:30 DEBUG "result": [ 19:41:30 DEBUG { 19:41:30 DEBUG "metric": { 19:41:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:41:30 DEBUG }, 19:41:30 DEBUG "value": [ 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG [...] 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:41:30 DEBUG }, 19:41:30 DEBUG "value": [ 19:41:30 DEBUG 1713552090, 19:41:30 DEBUG "2330.1666666666665" 19:41:30 DEBUG ] 19:41:30 DEBUG } 19:41:30 DEBUG ] 19:41:30 DEBUG } 19:41:30 DEBUG } 19:41:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552090" 19:41:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552090" 19:41:30 INFO [http_cmd]: http status code OK 19:41:30 DEBUG --- status code --- 19:41:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:30 DEBUG --- http response --- 19:41:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:41:30 DEBUG ----- output ----- 19:41:30 DEBUG { 19:41:30 DEBUG "status": "success", 19:41:30 DEBUG "data": { 19:41:30 DEBUG "resultType": "vector", 19:41:30 DEBUG "result": [ 19:41:30 DEBUG { 19:41:30 DEBUG "metric": { 19:41:30 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:41:30 DEBUG }, 19:41:30 DEBUG "value": [ 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG [...] 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:41:30 DEBUG }, 19:41:30 DEBUG "value": [ 19:41:30 DEBUG 1713552090, 19:41:30 DEBUG "3416961024" 19:41:30 DEBUG ] 19:41:30 DEBUG } 19:41:30 DEBUG ] 19:41:30 DEBUG } 19:41:30 DEBUG } 19:41:30 INFO [http_cmd]: http status code OK 19:41:30 DEBUG --- status code --- 19:41:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:30 DEBUG --- http response --- 19:41:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:41:30 DEBUG ----- output ----- 19:41:30 DEBUG { 19:41:30 DEBUG "status": "success", 19:41:30 DEBUG "data": { 19:41:30 DEBUG "resultType": "vector", 19:41:30 DEBUG "result": [ 19:41:30 DEBUG { 19:41:30 DEBUG "metric": { 19:41:30 DEBUG "instance": "10.142.0.107:9100" 19:41:30 DEBUG }, 19:41:30 DEBUG "value": [ 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG [...] 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG "instance": "10.142.0.43:9100" 19:41:30 DEBUG }, 19:41:30 DEBUG "value": [ 19:41:30 DEBUG 1713552090, 19:41:30 DEBUG "70321651.73333333" 19:41:30 DEBUG ] 19:41:30 DEBUG } 19:41:30 DEBUG ] 19:41:30 DEBUG } 19:41:30 DEBUG } 19:41:30 INFO [http_cmd]: http status code OK 19:41:30 DEBUG --- status code --- 19:41:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:30 DEBUG --- http response --- 19:41:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:41:30 DEBUG ----- output ----- 19:41:30 DEBUG { 19:41:30 DEBUG "status": "success", 19:41:30 DEBUG "data": { 19:41:30 DEBUG "resultType": "vector", 19:41:30 DEBUG "result": [ 19:41:30 DEBUG { 19:41:30 DEBUG "metric": { 19:41:30 DEBUG "pod": "ds-idrepo-0" 19:41:30 DEBUG }, 19:41:30 DEBUG "value": [ 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG [...] 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG 19:41:30 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:41:30 DEBUG }, 19:41:30 DEBUG "value": [ 19:41:30 DEBUG 1713552090, 19:41:30 DEBUG "0" 19:41:30 DEBUG ] 19:41:30 DEBUG } 19:41:30 DEBUG ] 19:41:30 DEBUG } 19:41:30 DEBUG } 19:41:30 INFO 19:41:30 INFO [loop_until]: kubectl --namespace=xlou top pods 19:41:30 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:41:30 INFO [loop_until]: OK (rc = 0) 19:41:30 DEBUG --- stdout --- 19:41:30 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 14201m 8993Mi am-767f6b67b5-qcqz2 14298m 9004Mi am-767f6b67b5-vbltz 15414m 8981Mi ds-cts-0 15595m 10904Mi ds-cts-1 13722m 11154Mi ds-cts-2 15501m 10911Mi ds-idrepo-0 6495m 3663Mi ds-idrepo-1 6187m 3533Mi ds-idrepo-2 6138m 3493Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 4m 1218Mi lodemon-f8459c597-6tj6f 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7696m 3166Mi 19:41:30 DEBUG --- stderr --- 19:41:30 DEBUG 19:41:31 INFO 19:41:31 INFO [loop_until]: kubectl --namespace=xlou top node 19:41:31 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:41:31 INFO [loop_until]: OK (rc = 0) 19:41:31 DEBUG --- stdout --- 19:41:31 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 70m 0% 6271Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 6475m 21% 7259Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 14296m 47% 11700Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 86m 0% 3347Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 15779m 52% 13918Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 155m 0% 5207Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 14424m 48% 11878Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12314m 41% 2997Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 15801m 52% 11205Mi 10% gke-xlou-cdm-ds-0f601751-j488 7002m 23% 6738Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 13860m 46% 15946Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 6945m 23% 7441Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 15848m 53% 13380Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 7868m 49% 6939Mi 11% 19:41:31 DEBUG --- stderr --- 19:41:31 DEBUG 19:42:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713552150" 19:42:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552150" 19:42:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552150" 19:42:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552150" 19:42:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552150" 19:42:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552150" 19:42:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552150" 19:42:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552150" 19:42:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552150" 19:42:30 ERROR [http_cmd]: ERROR 19:42:30 ERROR --- status code --- 19:42:30 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:42:30 ERROR --- http response --- 19:42:30 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:42:30 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:42:30 INFO [http_cmd]: http status code OK 19:42:30 INFO [http_cmd]: http status code OK 19:42:30 INFO [http_cmd]: http status code OK 19:42:30 INFO [http_cmd]: http status code OK 19:42:30 DEBUG --- status code --- 19:42:30 INFO [http_cmd]: http status code OK 19:42:30 DEBUG --- status code --- 19:42:30 DEBUG --- status code --- 19:42:30 DEBUG --- status code --- 19:42:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:30 DEBUG --- status code --- 19:42:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:30 DEBUG --- http response --- 19:42:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:30 DEBUG --- http response --- 19:42:30 DEBUG --- http response --- 19:42:30 DEBUG --- http response --- 19:42:30 DEBUG --- http response --- 19:42:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:42:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:42:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:42:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:42:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:42:30 DEBUG ----- output ----- 19:42:30 DEBUG ----- output ----- 19:42:30 DEBUG ----- output ----- 19:42:30 DEBUG ----- output ----- 19:42:30 DEBUG ----- output ----- 19:42:30 DEBUG { 19:42:30 DEBUG { 19:42:30 DEBUG { 19:42:30 DEBUG { 19:42:30 DEBUG { 19:42:30 DEBUG "status": "success", 19:42:30 DEBUG "status": "success", 19:42:30 DEBUG "status": "success", 19:42:30 DEBUG "status": "success", 19:42:30 DEBUG "status": "success", 19:42:30 DEBUG "data": { 19:42:30 DEBUG "data": { 19:42:30 DEBUG "data": { 19:42:30 DEBUG "data": { 19:42:30 DEBUG "data": { 19:42:30 DEBUG "resultType": "vector", 19:42:30 DEBUG "resultType": "vector", 19:42:30 DEBUG "resultType": "vector", 19:42:30 DEBUG "resultType": "vector", 19:42:30 DEBUG "resultType": "vector", 19:42:30 DEBUG "result": [ 19:42:30 DEBUG "result": [ 19:42:30 DEBUG "result": [ 19:42:30 DEBUG { 19:42:30 DEBUG "metric": { 19:42:30 DEBUG { 19:42:30 DEBUG { 19:42:30 DEBUG "result": [ 19:42:30 DEBUG "result": [ 19:42:30 DEBUG "pod": "ds-cts-0" 19:42:30 DEBUG "metric": { 19:42:30 DEBUG "metric": { 19:42:30 DEBUG { 19:42:30 DEBUG { 19:42:30 DEBUG }, 19:42:30 DEBUG "pod": "ds-idrepo-0" 19:42:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:42:30 DEBUG "metric": { 19:42:30 DEBUG "metric": { 19:42:30 DEBUG "value": [ 19:42:30 DEBUG }, 19:42:30 DEBUG }, 19:42:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:42:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:42:30 DEBUG 19:42:30 DEBUG "value": [ 19:42:30 DEBUG "value": [ 19:42:30 DEBUG }, 19:42:30 DEBUG }, 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG "value": [ 19:42:30 INFO [http_cmd]: http status code OK 19:42:30 DEBUG "value": [ 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG --- status code --- 19:42:30 DEBUG 19:42:30 DEBUG [...] 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG [...] 19:42:30 DEBUG [...] 19:42:30 DEBUG 19:42:30 DEBUG --- http response --- 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG [...] 19:42:30 DEBUG [...] 19:42:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG ----- output ----- 19:42:30 DEBUG "pod": "ds-cts-2" 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG { 19:42:30 DEBUG }, 19:42:30 DEBUG "pod": "ds-idrepo-2" 19:42:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG "status": "success", 19:42:30 DEBUG "value": [ 19:42:30 DEBUG }, 19:42:30 DEBUG }, 19:42:30 INFO [http_cmd]: http status code OK 19:42:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:42:30 INFO [http_cmd]: http status code OK 19:42:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:42:30 DEBUG "data": { 19:42:30 DEBUG 1713552150, 19:42:30 DEBUG "value": [ 19:42:30 DEBUG "value": [ 19:42:30 DEBUG --- status code --- 19:42:30 DEBUG }, 19:42:30 DEBUG --- status code --- 19:42:30 DEBUG }, 19:42:30 DEBUG "resultType": "vector", 19:42:30 DEBUG "0" 19:42:30 DEBUG 1713552150, 19:42:30 DEBUG 1713552150, 19:42:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:30 DEBUG "value": [ 19:42:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:30 DEBUG "value": [ 19:42:30 DEBUG "result": [ 19:42:30 DEBUG ] 19:42:30 DEBUG "0" 19:42:30 DEBUG "2327.366666666667" 19:42:30 DEBUG --- http response --- 19:42:30 DEBUG 1713552150, 19:42:30 DEBUG --- http response --- 19:42:30 DEBUG 1713552150, 19:42:30 DEBUG { 19:42:30 DEBUG } 19:42:30 DEBUG ] 19:42:30 DEBUG ] 19:42:30 DEBUG "419624" 19:42:30 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:42:30 DEBUG "2331.0666666666666" 19:42:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:42:30 DEBUG "metric": { 19:42:30 DEBUG ] 19:42:30 DEBUG } 19:42:30 DEBUG } 19:42:30 DEBUG ] 19:42:30 DEBUG ] 19:42:30 DEBUG ----- output ----- 19:42:30 DEBUG "instance": "10.142.0.107:9100" 19:42:30 DEBUG } 19:42:30 DEBUG ] 19:42:30 DEBUG ] 19:42:30 DEBUG } 19:42:30 DEBUG } 19:42:30 DEBUG { 19:42:30 DEBUG }, 19:42:30 DEBUG } 19:42:30 DEBUG } 19:42:30 DEBUG } 19:42:30 DEBUG ] 19:42:30 DEBUG ] 19:42:30 DEBUG "status": "success", 19:42:30 DEBUG "value": [ 19:42:30 DEBUG } 19:42:30 DEBUG } 19:42:30 DEBUG } 19:42:30 DEBUG } 19:42:30 DEBUG "data": { 19:42:30 DEBUG 19:42:30 DEBUG } 19:42:30 DEBUG } 19:42:30 DEBUG "resultType": "vector", 19:42:30 DEBUG 19:42:30 DEBUG "result": [ 19:42:30 DEBUG 19:42:30 DEBUG { 19:42:30 DEBUG [...] 19:42:30 DEBUG "metric": { 19:42:30 DEBUG 19:42:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:42:30 DEBUG 19:42:30 DEBUG }, 19:42:30 DEBUG 19:42:30 DEBUG "value": [ 19:42:30 DEBUG "instance": "10.142.0.43:9100" 19:42:30 DEBUG 19:42:30 DEBUG }, 19:42:30 DEBUG 19:42:30 DEBUG "value": [ 19:42:30 DEBUG 19:42:30 DEBUG 1713552150, 19:42:30 DEBUG [...] 19:42:30 DEBUG "0.15959999999996152" 19:42:30 DEBUG 19:42:30 DEBUG ] 19:42:30 DEBUG 19:42:30 DEBUG } 19:42:30 DEBUG 19:42:30 DEBUG ] 19:42:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:42:30 DEBUG } 19:42:30 DEBUG }, 19:42:30 DEBUG } 19:42:30 DEBUG "value": [ 19:42:30 DEBUG 1713552150, 19:42:30 DEBUG "0" 19:42:30 DEBUG ] 19:42:30 DEBUG } 19:42:30 DEBUG ] 19:42:30 DEBUG } 19:42:30 DEBUG } 19:42:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713552150" 19:42:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552150" 19:42:30 INFO [http_cmd]: http status code OK 19:42:30 DEBUG --- status code --- 19:42:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:30 DEBUG --- http response --- 19:42:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:42:30 DEBUG ----- output ----- 19:42:30 DEBUG { 19:42:30 DEBUG "status": "success", 19:42:30 DEBUG "data": { 19:42:30 DEBUG "resultType": "vector", 19:42:30 DEBUG "result": [ 19:42:30 DEBUG { 19:42:30 DEBUG "metric": { 19:42:30 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:42:30 DEBUG }, 19:42:30 DEBUG "value": [ 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG [...] 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:42:30 DEBUG }, 19:42:30 DEBUG "value": [ 19:42:30 DEBUG 1713552150, 19:42:30 DEBUG "7.694019145304178" 19:42:30 DEBUG ] 19:42:30 DEBUG } 19:42:30 DEBUG ] 19:42:30 DEBUG } 19:42:30 DEBUG } 19:42:30 INFO [http_cmd]: http status code OK 19:42:30 DEBUG --- status code --- 19:42:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:30 DEBUG --- http response --- 19:42:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:42:30 DEBUG ----- output ----- 19:42:30 DEBUG { 19:42:30 DEBUG "status": "success", 19:42:30 DEBUG "data": { 19:42:30 DEBUG "resultType": "vector", 19:42:30 DEBUG "result": [ 19:42:30 DEBUG { 19:42:30 DEBUG "metric": { 19:42:30 DEBUG "instance": "10.142.0.107:9100" 19:42:30 DEBUG }, 19:42:30 DEBUG "value": [ 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG [...] 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG "instance": "10.142.0.43:9100" 19:42:30 DEBUG }, 19:42:30 DEBUG "value": [ 19:42:30 DEBUG 1713552150, 19:42:30 DEBUG "70755911.8" 19:42:30 DEBUG ] 19:42:30 DEBUG } 19:42:30 DEBUG ] 19:42:30 DEBUG } 19:42:30 DEBUG } 19:42:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552150" 19:42:30 INFO [http_cmd]: http status code OK 19:42:30 DEBUG --- status code --- 19:42:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:30 DEBUG --- http response --- 19:42:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:42:30 DEBUG ----- output ----- 19:42:30 DEBUG { 19:42:30 DEBUG "status": "success", 19:42:30 DEBUG "data": { 19:42:30 DEBUG "resultType": "vector", 19:42:30 DEBUG "result": [ 19:42:30 DEBUG { 19:42:30 DEBUG "metric": { 19:42:30 DEBUG "instance": "10.142.0.107:9100" 19:42:30 DEBUG }, 19:42:30 DEBUG "value": [ 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG [...] 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG "instance": "10.142.0.43:9100" 19:42:30 DEBUG }, 19:42:30 DEBUG "value": [ 19:42:30 DEBUG 1713552150, 19:42:30 DEBUG "1938639768576" 19:42:30 DEBUG ] 19:42:30 DEBUG } 19:42:30 DEBUG ] 19:42:30 DEBUG } 19:42:30 DEBUG } 19:42:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552150" 19:42:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713552150" 19:42:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552150" 19:42:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552150" 19:42:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552150" 19:42:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552150" 19:42:30 INFO [http_cmd]: http status code OK 19:42:30 DEBUG --- status code --- 19:42:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:30 DEBUG --- http response --- 19:42:30 INFO [http_cmd]: http status code OK 19:42:30 INFO [http_cmd]: http status code OK 19:42:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:42:30 DEBUG --- status code --- 19:42:30 DEBUG --- status code --- 19:42:30 DEBUG ----- output ----- 19:42:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:30 DEBUG { 19:42:30 DEBUG --- http response --- 19:42:30 DEBUG --- http response --- 19:42:30 DEBUG "status": "success", 19:42:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:42:30 DEBUG "data": { 19:42:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:42:30 DEBUG ----- output ----- 19:42:30 DEBUG "resultType": "vector", 19:42:30 DEBUG ----- output ----- 19:42:30 DEBUG { 19:42:30 DEBUG "result": [ 19:42:30 DEBUG { 19:42:30 DEBUG "status": "success", 19:42:30 DEBUG { 19:42:30 DEBUG "status": "success", 19:42:30 DEBUG "data": { 19:42:30 DEBUG "metric": { 19:42:30 DEBUG "data": { 19:42:30 DEBUG "resultType": "vector", 19:42:30 DEBUG "pod": "ds-idrepo-0" 19:42:30 DEBUG "resultType": "vector", 19:42:30 DEBUG "result": [ 19:42:30 DEBUG }, 19:42:30 DEBUG "result": [ 19:42:30 DEBUG { 19:42:30 DEBUG "value": [ 19:42:30 DEBUG { 19:42:30 DEBUG "metric": { 19:42:30 DEBUG 19:42:30 DEBUG "metric": { 19:42:30 DEBUG "pod": "ds-idrepo-1" 19:42:30 DEBUG 19:42:30 DEBUG "pod": "ds-idrepo-0" 19:42:30 DEBUG }, 19:42:30 DEBUG 19:42:30 DEBUG }, 19:42:30 DEBUG "value": [ 19:42:30 DEBUG [...] 19:42:30 DEBUG "value": [ 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG [...] 19:42:30 DEBUG "pod": "ds-idrepo-2" 19:42:30 DEBUG [...] 19:42:30 DEBUG 19:42:30 DEBUG }, 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG "value": [ 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 1713552150, 19:42:30 DEBUG 19:42:30 DEBUG "pod": "am-767f6b67b5-qcqz2" 19:42:30 DEBUG "0" 19:42:30 DEBUG "pod": "ds-cts-2" 19:42:30 DEBUG }, 19:42:30 DEBUG ] 19:42:30 DEBUG }, 19:42:30 DEBUG "value": [ 19:42:30 DEBUG } 19:42:30 DEBUG "value": [ 19:42:30 DEBUG 1713552150, 19:42:30 DEBUG ] 19:42:30 DEBUG 1713552150, 19:42:30 DEBUG "515.7066414856783" 19:42:30 DEBUG } 19:42:30 DEBUG "0" 19:42:30 DEBUG ] 19:42:30 DEBUG } 19:42:30 DEBUG ] 19:42:30 DEBUG } 19:42:30 DEBUG } 19:42:30 DEBUG ] 19:42:30 DEBUG ] 19:42:30 DEBUG } 19:42:30 DEBUG } 19:42:30 INFO [http_cmd]: http status code OK 19:42:30 DEBUG } 19:42:30 DEBUG } 19:42:30 DEBUG --- status code --- 19:42:30 INFO [http_cmd]: http status code OK 19:42:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:30 DEBUG --- status code --- 19:42:30 DEBUG --- http response --- 19:42:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:30 DEBUG --- http response --- 19:42:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:42:30 DEBUG ----- output ----- 19:42:30 DEBUG { 19:42:30 DEBUG "status": "success", 19:42:30 DEBUG "data": { 19:42:30 DEBUG "resultType": "vector", 19:42:30 DEBUG "result": [ 19:42:30 DEBUG { 19:42:30 DEBUG "metric": { 19:42:30 DEBUG "pod": "ds-idrepo-0" 19:42:30 DEBUG }, 19:42:30 DEBUG "value": [ 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG [...] 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG "pod": "ds-idrepo-2" 19:42:30 DEBUG }, 19:42:30 DEBUG "value": [ 19:42:30 DEBUG 1713552150, 19:42:30 DEBUG "0" 19:42:30 DEBUG ] 19:42:30 DEBUG } 19:42:30 DEBUG ] 19:42:30 DEBUG } 19:42:30 DEBUG } 19:42:30 INFO [http_cmd]: http status code OK 19:42:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:42:30 DEBUG --- status code --- 19:42:30 DEBUG ----- output ----- 19:42:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:30 DEBUG { 19:42:30 DEBUG --- http response --- 19:42:30 DEBUG "status": "success", 19:42:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:42:30 DEBUG "data": { 19:42:30 DEBUG ----- output ----- 19:42:30 DEBUG "resultType": "vector", 19:42:30 DEBUG { 19:42:30 DEBUG "result": [ 19:42:30 DEBUG "status": "success", 19:42:30 DEBUG { 19:42:30 DEBUG "data": { 19:42:30 DEBUG "metric": { 19:42:30 DEBUG "resultType": "vector", 19:42:30 DEBUG "pod": "ds-idrepo-1" 19:42:30 DEBUG "result": [ 19:42:30 DEBUG }, 19:42:30 DEBUG { 19:42:30 DEBUG "value": [ 19:42:30 DEBUG "metric": { 19:42:30 DEBUG 19:42:30 DEBUG "instance": "10.142.0.107:9100" 19:42:30 DEBUG 19:42:30 DEBUG }, 19:42:30 DEBUG 19:42:30 DEBUG "value": [ 19:42:30 DEBUG [...] 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG [...] 19:42:30 DEBUG "pod": "am-767f6b67b5-qcqz2" 19:42:30 DEBUG 19:42:30 DEBUG }, 19:42:30 DEBUG 19:42:30 DEBUG "value": [ 19:42:30 DEBUG 19:42:30 DEBUG 1713552150, 19:42:30 DEBUG "instance": "10.142.0.43:9100" 19:42:30 DEBUG "297150166.82404786" 19:42:30 DEBUG }, 19:42:30 DEBUG ] 19:42:30 DEBUG "value": [ 19:42:30 DEBUG } 19:42:30 DEBUG 1713552150, 19:42:30 DEBUG ] 19:42:30 DEBUG "0.07299999999999802" 19:42:30 DEBUG } 19:42:30 DEBUG ] 19:42:30 DEBUG } 19:42:30 DEBUG } 19:42:30 DEBUG ] 19:42:30 DEBUG } 19:42:30 DEBUG } 19:42:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552150" 19:42:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552150" 19:42:30 INFO [http_cmd]: http status code OK 19:42:30 INFO [http_cmd]: http status code OK 19:42:30 DEBUG --- status code --- 19:42:30 DEBUG --- status code --- 19:42:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:30 DEBUG --- http response --- 19:42:30 DEBUG --- http response --- 19:42:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:42:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:42:30 DEBUG ----- output ----- 19:42:30 DEBUG ----- output ----- 19:42:30 DEBUG { 19:42:30 DEBUG { 19:42:30 DEBUG "status": "success", 19:42:30 DEBUG "status": "success", 19:42:30 DEBUG "data": { 19:42:30 DEBUG "data": { 19:42:30 DEBUG "resultType": "vector", 19:42:30 DEBUG "resultType": "vector", 19:42:30 DEBUG "result": [ 19:42:30 DEBUG "result": [ 19:42:30 DEBUG { 19:42:30 DEBUG { 19:42:30 DEBUG "metric": { 19:42:30 DEBUG "metric": { 19:42:30 DEBUG "pod": "ds-cts-0" 19:42:30 DEBUG "pod": "ds-cts-0" 19:42:30 DEBUG }, 19:42:30 DEBUG }, 19:42:30 DEBUG "value": [ 19:42:30 DEBUG "value": [ 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG [...] 19:42:30 DEBUG [...] 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG "pod": "ds-cts-2" 19:42:30 DEBUG "pod": "ds-cts-2" 19:42:30 DEBUG }, 19:42:30 DEBUG }, 19:42:30 DEBUG "value": [ 19:42:30 DEBUG "value": [ 19:42:30 DEBUG 1713552150, 19:42:30 DEBUG 1713552150, 19:42:30 DEBUG "29.833333333333332" 19:42:30 DEBUG "0" 19:42:30 DEBUG ] 19:42:30 DEBUG ] 19:42:30 DEBUG } 19:42:30 DEBUG } 19:42:30 DEBUG ] 19:42:30 DEBUG ] 19:42:30 DEBUG } 19:42:30 DEBUG } 19:42:30 DEBUG } 19:42:30 DEBUG } 19:42:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552150" 19:42:30 INFO [http_cmd]: http status code OK 19:42:30 DEBUG --- status code --- 19:42:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:30 DEBUG --- http response --- 19:42:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:42:30 DEBUG ----- output ----- 19:42:30 DEBUG { 19:42:30 DEBUG "status": "success", 19:42:30 DEBUG "data": { 19:42:30 DEBUG "resultType": "vector", 19:42:30 DEBUG "result": [ 19:42:30 DEBUG { 19:42:30 DEBUG "metric": { 19:42:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:42:30 DEBUG }, 19:42:30 DEBUG "value": [ 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG [...] 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:42:30 DEBUG }, 19:42:30 DEBUG "value": [ 19:42:30 DEBUG 1713552150, 19:42:30 DEBUG "2330.6666666666665" 19:42:30 DEBUG ] 19:42:30 DEBUG } 19:42:30 DEBUG ] 19:42:30 DEBUG } 19:42:30 DEBUG } 19:42:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713552150" 19:42:30 INFO [http_cmd]: http status code OK 19:42:30 DEBUG --- status code --- 19:42:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:30 DEBUG --- http response --- 19:42:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:42:30 DEBUG ----- output ----- 19:42:30 DEBUG { 19:42:30 DEBUG "status": "success", 19:42:30 DEBUG "data": { 19:42:30 DEBUG "resultType": "vector", 19:42:30 DEBUG "result": [ 19:42:30 DEBUG { 19:42:30 DEBUG "metric": { 19:42:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:42:30 DEBUG }, 19:42:30 DEBUG "value": [ 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG [...] 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:42:30 DEBUG }, 19:42:30 DEBUG "value": [ 19:42:30 DEBUG 1713552150, 19:42:30 DEBUG "0.012" 19:42:30 DEBUG ] 19:42:30 DEBUG } 19:42:30 DEBUG ] 19:42:30 DEBUG } 19:42:30 DEBUG } 19:42:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552150" 19:42:30 INFO [http_cmd]: http status code OK 19:42:30 DEBUG --- status code --- 19:42:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:30 DEBUG --- http response --- 19:42:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:42:30 DEBUG ----- output ----- 19:42:30 DEBUG { 19:42:30 DEBUG "status": "success", 19:42:30 DEBUG "data": { 19:42:30 DEBUG "resultType": "vector", 19:42:30 DEBUG "result": [ 19:42:30 DEBUG { 19:42:30 DEBUG "metric": { 19:42:30 DEBUG "pod": "ds-idrepo-1" 19:42:30 DEBUG }, 19:42:30 DEBUG "value": [ 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG [...] 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG "pod": "am-767f6b67b5-qcqz2" 19:42:30 DEBUG }, 19:42:30 DEBUG "value": [ 19:42:30 DEBUG 1713552150, 19:42:30 DEBUG "0" 19:42:30 DEBUG ] 19:42:30 DEBUG } 19:42:30 DEBUG ] 19:42:30 DEBUG } 19:42:30 DEBUG } 19:42:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552150" 19:42:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552150" 19:42:30 INFO [http_cmd]: http status code OK 19:42:30 DEBUG --- status code --- 19:42:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:30 DEBUG --- http response --- 19:42:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:42:30 DEBUG ----- output ----- 19:42:30 DEBUG { 19:42:30 DEBUG "status": "success", 19:42:30 DEBUG "data": { 19:42:30 DEBUG "resultType": "vector", 19:42:30 DEBUG "result": [ 19:42:30 DEBUG { 19:42:30 DEBUG "metric": { 19:42:30 DEBUG "instance": "10.142.0.107:9100" 19:42:30 DEBUG }, 19:42:30 DEBUG "value": [ 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG [...] 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG "instance": "10.142.0.43:9100" 19:42:30 DEBUG }, 19:42:30 DEBUG "value": [ 19:42:30 DEBUG 1713552150, 19:42:30 DEBUG "70752012.23333333" 19:42:30 DEBUG ] 19:42:30 DEBUG } 19:42:30 DEBUG ] 19:42:30 DEBUG } 19:42:30 DEBUG } 19:42:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713552150" 19:42:30 INFO [http_cmd]: http status code OK 19:42:30 DEBUG --- status code --- 19:42:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:30 DEBUG --- http response --- 19:42:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:42:30 DEBUG ----- output ----- 19:42:30 DEBUG { 19:42:30 DEBUG "status": "success", 19:42:30 DEBUG "data": { 19:42:30 DEBUG "resultType": "vector", 19:42:30 DEBUG "result": [ 19:42:30 DEBUG { 19:42:30 DEBUG "metric": { 19:42:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:42:30 DEBUG }, 19:42:30 DEBUG "value": [ 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG [...] 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:42:30 DEBUG }, 19:42:30 DEBUG "value": [ 19:42:30 DEBUG 1713552150, 19:42:30 DEBUG "2331" 19:42:30 DEBUG ] 19:42:30 DEBUG } 19:42:30 DEBUG ] 19:42:30 DEBUG } 19:42:30 DEBUG } 19:42:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552150" 19:42:30 INFO [http_cmd]: http status code OK 19:42:30 DEBUG --- status code --- 19:42:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:30 DEBUG --- http response --- 19:42:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:42:30 DEBUG ----- output ----- 19:42:30 DEBUG { 19:42:30 DEBUG "status": "success", 19:42:30 DEBUG "data": { 19:42:30 DEBUG "resultType": "vector", 19:42:30 DEBUG "result": [ 19:42:30 DEBUG { 19:42:30 DEBUG "metric": { 19:42:30 DEBUG "pod": "ds-idrepo-1" 19:42:30 DEBUG }, 19:42:30 DEBUG "value": [ 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG [...] 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG "pod": "am-767f6b67b5-qcqz2" 19:42:30 DEBUG }, 19:42:30 DEBUG "value": [ 19:42:30 DEBUG 1713552150, 19:42:30 DEBUG "14.432346742209619" 19:42:30 DEBUG ] 19:42:30 DEBUG } 19:42:30 DEBUG ] 19:42:30 DEBUG } 19:42:30 DEBUG } 19:42:30 INFO [http_cmd]: http status code OK 19:42:30 DEBUG --- status code --- 19:42:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:30 DEBUG --- http response --- 19:42:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:42:30 DEBUG ----- output ----- 19:42:30 DEBUG { 19:42:30 DEBUG "status": "success", 19:42:30 DEBUG "data": { 19:42:30 DEBUG "resultType": "vector", 19:42:30 DEBUG "result": [ 19:42:30 DEBUG { 19:42:30 DEBUG "metric": { 19:42:30 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:42:30 DEBUG }, 19:42:30 DEBUG "value": [ 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG [...] 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG 19:42:30 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:42:30 DEBUG }, 19:42:30 DEBUG "value": [ 19:42:30 DEBUG 1713552150, 19:42:30 DEBUG "3417964544" 19:42:30 DEBUG ] 19:42:30 DEBUG } 19:42:30 DEBUG ] 19:42:30 DEBUG } 19:42:30 DEBUG } 19:42:30 INFO 19:42:30 INFO [loop_until]: kubectl --namespace=xlou top pods 19:42:30 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:42:31 INFO [loop_until]: OK (rc = 0) 19:42:31 DEBUG --- stdout --- 19:42:31 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 14422m 8993Mi am-767f6b67b5-qcqz2 14313m 9002Mi am-767f6b67b5-vbltz 15415m 8982Mi ds-cts-0 15373m 10921Mi ds-cts-1 13673m 11140Mi ds-cts-2 16052m 10924Mi ds-idrepo-0 6164m 3664Mi ds-idrepo-1 6241m 3533Mi ds-idrepo-2 6432m 3492Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 1m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7491m 3167Mi 19:42:31 DEBUG --- stderr --- 19:42:31 DEBUG 19:42:31 INFO 19:42:31 INFO [loop_until]: kubectl --namespace=xlou top node 19:42:31 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:42:31 INFO [loop_until]: OK (rc = 0) 19:42:31 DEBUG --- stdout --- 19:42:31 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 68m 0% 6256Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 6465m 21% 7250Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13717m 45% 11701Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 3333Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 15304m 51% 13917Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 134m 0% 5211Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 14392m 48% 11882Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12300m 41% 2995Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 16373m 54% 11197Mi 10% gke-xlou-cdm-ds-0f601751-j488 7044m 23% 6733Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 13464m 45% 15913Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 6855m 22% 7417Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 15775m 52% 13396Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 7855m 49% 6939Mi 11% 19:42:31 DEBUG --- stderr --- 19:42:31 DEBUG 19:43:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713552210" 19:43:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552210" 19:43:30 ERROR [http_cmd]: ERROR 19:43:30 ERROR --- status code --- 19:43:30 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:43:30 ERROR --- http response --- 19:43:30 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:43:30 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:43:30 INFO [http_cmd]: http status code OK 19:43:30 DEBUG --- status code --- 19:43:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:30 DEBUG --- http response --- 19:43:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:43:30 DEBUG ----- output ----- 19:43:30 DEBUG { 19:43:30 DEBUG "status": "success", 19:43:30 DEBUG "data": { 19:43:30 DEBUG "resultType": "vector", 19:43:30 DEBUG "result": [ 19:43:30 DEBUG { 19:43:30 DEBUG "metric": { 19:43:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:43:30 DEBUG }, 19:43:30 DEBUG "value": [ 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG [...] 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:43:30 DEBUG }, 19:43:30 DEBUG "value": [ 19:43:30 DEBUG 1713552210, 19:43:30 DEBUG "420093.5" 19:43:30 DEBUG ] 19:43:30 DEBUG } 19:43:30 DEBUG ] 19:43:30 DEBUG } 19:43:30 DEBUG } 19:43:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552210" 19:43:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552210" 19:43:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552210" 19:43:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552210" 19:43:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552210" 19:43:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552210" 19:43:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552210" 19:43:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552210" 19:43:30 INFO [http_cmd]: http status code OK 19:43:30 INFO [http_cmd]: http status code OK 19:43:30 DEBUG --- status code --- 19:43:30 DEBUG --- status code --- 19:43:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:30 DEBUG --- http response --- 19:43:30 DEBUG --- http response --- 19:43:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:43:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:43:30 DEBUG ----- output ----- 19:43:30 DEBUG ----- output ----- 19:43:30 DEBUG { 19:43:30 DEBUG { 19:43:30 DEBUG "status": "success", 19:43:30 DEBUG "status": "success", 19:43:30 DEBUG "data": { 19:43:30 DEBUG "data": { 19:43:30 DEBUG "resultType": "vector", 19:43:30 DEBUG "resultType": "vector", 19:43:30 DEBUG "result": [ 19:43:30 DEBUG "result": [ 19:43:30 DEBUG { 19:43:30 DEBUG { 19:43:30 DEBUG "metric": { 19:43:30 DEBUG "metric": { 19:43:30 DEBUG "pod": "ds-cts-0" 19:43:30 DEBUG "instance": "10.142.0.107:9100" 19:43:30 DEBUG }, 19:43:30 INFO [http_cmd]: http status code OK 19:43:30 DEBUG }, 19:43:30 DEBUG "value": [ 19:43:30 DEBUG --- status code --- 19:43:30 DEBUG "value": [ 19:43:30 DEBUG 19:43:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG --- http response --- 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG [...] 19:43:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:43:30 DEBUG [...] 19:43:30 DEBUG 19:43:30 DEBUG ----- output ----- 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG { 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG "status": "success", 19:43:30 DEBUG 19:43:30 DEBUG "pod": "ds-cts-2" 19:43:30 DEBUG "data": { 19:43:30 DEBUG "instance": "10.142.0.43:9100" 19:43:30 DEBUG }, 19:43:30 DEBUG "resultType": "vector", 19:43:30 DEBUG }, 19:43:30 DEBUG "value": [ 19:43:30 DEBUG "result": [ 19:43:30 DEBUG "value": [ 19:43:30 DEBUG 1713552210, 19:43:30 DEBUG { 19:43:30 DEBUG 1713552210, 19:43:30 DEBUG "0" 19:43:30 DEBUG "metric": { 19:43:30 DEBUG "1941707056128" 19:43:30 DEBUG ] 19:43:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:43:30 DEBUG ] 19:43:30 DEBUG } 19:43:30 DEBUG }, 19:43:30 DEBUG } 19:43:30 DEBUG ] 19:43:30 DEBUG "value": [ 19:43:30 DEBUG ] 19:43:30 DEBUG } 19:43:30 DEBUG 19:43:30 DEBUG } 19:43:30 DEBUG } 19:43:30 DEBUG 19:43:30 DEBUG } 19:43:30 DEBUG 19:43:30 DEBUG [...] 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:43:30 DEBUG }, 19:43:30 DEBUG "value": [ 19:43:30 DEBUG 1713552210, 19:43:30 DEBUG "2331.3" 19:43:30 DEBUG ] 19:43:30 DEBUG } 19:43:30 DEBUG ] 19:43:30 DEBUG } 19:43:30 DEBUG } 19:43:30 INFO [http_cmd]: http status code OK 19:43:30 INFO [http_cmd]: http status code OK 19:43:30 INFO [http_cmd]: http status code OK 19:43:30 DEBUG --- status code --- 19:43:30 DEBUG --- status code --- 19:43:30 DEBUG --- status code --- 19:43:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:30 DEBUG --- http response --- 19:43:30 DEBUG --- http response --- 19:43:30 DEBUG --- http response --- 19:43:30 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:43:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:43:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:43:30 INFO [http_cmd]: http status code OK 19:43:30 INFO [http_cmd]: http status code OK 19:43:30 DEBUG ----- output ----- 19:43:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713552210" 19:43:30 DEBUG ----- output ----- 19:43:30 DEBUG --- status code --- 19:43:30 DEBUG --- status code --- 19:43:30 DEBUG { 19:43:30 DEBUG { 19:43:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552210" 19:43:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:30 DEBUG "status": "success", 19:43:30 DEBUG "status": "success", 19:43:30 DEBUG --- http response --- 19:43:30 DEBUG --- http response --- 19:43:30 DEBUG "data": { 19:43:30 DEBUG "data": { 19:43:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:43:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:43:30 DEBUG "resultType": "vector", 19:43:30 DEBUG "resultType": "vector", 19:43:30 DEBUG ----- output ----- 19:43:30 DEBUG ----- output ----- 19:43:30 DEBUG "result": [ 19:43:30 DEBUG "result": [ 19:43:30 DEBUG { 19:43:30 DEBUG { 19:43:30 DEBUG { 19:43:30 DEBUG { 19:43:30 DEBUG "status": "success", 19:43:30 DEBUG "status": "success", 19:43:30 DEBUG "metric": { 19:43:30 DEBUG "metric": { 19:43:30 DEBUG "data": { 19:43:30 DEBUG "data": { 19:43:30 DEBUG "pod": "ds-idrepo-0" 19:43:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:43:30 DEBUG "resultType": "vector", 19:43:30 DEBUG "resultType": "vector", 19:43:30 DEBUG }, 19:43:30 DEBUG }, 19:43:30 DEBUG "result": [ 19:43:30 DEBUG "result": [ 19:43:30 DEBUG "value": [ 19:43:30 DEBUG "value": [ 19:43:30 DEBUG { 19:43:30 DEBUG { 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG "metric": { 19:43:30 DEBUG "metric": { 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG "instance": "10.142.0.107:9100" 19:43:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG }, 19:43:30 DEBUG }, 19:43:30 DEBUG [...] 19:43:30 DEBUG [...] 19:43:30 DEBUG "value": [ 19:43:30 DEBUG "value": [ 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG "pod": "ds-idrepo-2" 19:43:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:43:30 DEBUG [...] 19:43:30 DEBUG [...] 19:43:30 DEBUG }, 19:43:30 DEBUG }, 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG "value": [ 19:43:30 DEBUG "value": [ 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 1713552210, 19:43:30 DEBUG 1713552210, 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG "0" 19:43:30 DEBUG "2336.5666666666666" 19:43:30 DEBUG "instance": "10.142.0.43:9100" 19:43:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:43:30 DEBUG ] 19:43:30 DEBUG ] 19:43:30 DEBUG }, 19:43:30 DEBUG }, 19:43:30 DEBUG } 19:43:30 DEBUG } 19:43:30 DEBUG "value": [ 19:43:30 DEBUG "value": [ 19:43:30 DEBUG ] 19:43:30 DEBUG ] 19:43:30 DEBUG 1713552210, 19:43:30 DEBUG 1713552210, 19:43:30 DEBUG } 19:43:30 DEBUG } 19:43:30 DEBUG "0.16629999999997835" 19:43:30 DEBUG "0" 19:43:30 DEBUG } 19:43:30 DEBUG } 19:43:30 DEBUG ] 19:43:30 DEBUG ] 19:43:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552210" 19:43:30 DEBUG } 19:43:30 DEBUG } 19:43:30 DEBUG ] 19:43:30 DEBUG ] 19:43:30 DEBUG } 19:43:30 DEBUG } 19:43:30 DEBUG } 19:43:30 DEBUG } 19:43:30 INFO [http_cmd]: http status code OK 19:43:30 DEBUG --- status code --- 19:43:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:30 DEBUG --- http response --- 19:43:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:43:30 DEBUG ----- output ----- 19:43:30 DEBUG { 19:43:30 DEBUG "status": "success", 19:43:30 DEBUG "data": { 19:43:30 DEBUG "resultType": "vector", 19:43:30 DEBUG "result": [ 19:43:30 DEBUG { 19:43:30 DEBUG "metric": { 19:43:30 DEBUG "pod": "ds-idrepo-1" 19:43:30 DEBUG }, 19:43:30 DEBUG "value": [ 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG [...] 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:43:30 DEBUG }, 19:43:30 DEBUG "value": [ 19:43:30 DEBUG 1713552210, 19:43:30 DEBUG "0" 19:43:30 DEBUG ] 19:43:30 DEBUG } 19:43:30 DEBUG ] 19:43:30 DEBUG } 19:43:30 DEBUG } 19:43:30 INFO [http_cmd]: http status code OK 19:43:30 DEBUG --- status code --- 19:43:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:30 DEBUG --- http response --- 19:43:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:43:30 DEBUG ----- output ----- 19:43:30 DEBUG { 19:43:30 DEBUG "status": "success", 19:43:30 DEBUG "data": { 19:43:30 DEBUG "resultType": "vector", 19:43:30 DEBUG "result": [ 19:43:30 DEBUG { 19:43:30 DEBUG "metric": { 19:43:30 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:43:30 DEBUG }, 19:43:30 DEBUG "value": [ 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG [...] 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:43:30 DEBUG }, 19:43:30 DEBUG "value": [ 19:43:30 DEBUG 1713552210, 19:43:30 DEBUG "7.726875317276962" 19:43:30 DEBUG ] 19:43:30 DEBUG } 19:43:30 DEBUG ] 19:43:30 DEBUG } 19:43:30 DEBUG } 19:43:30 INFO [http_cmd]: http status code OK 19:43:30 DEBUG --- status code --- 19:43:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:30 DEBUG --- http response --- 19:43:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:43:30 DEBUG ----- output ----- 19:43:30 DEBUG { 19:43:30 DEBUG "status": "success", 19:43:30 DEBUG "data": { 19:43:30 DEBUG "resultType": "vector", 19:43:30 DEBUG "result": [ 19:43:30 DEBUG { 19:43:30 DEBUG "metric": { 19:43:30 DEBUG "instance": "10.142.0.107:9100" 19:43:30 DEBUG }, 19:43:30 DEBUG "value": [ 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG [...] 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG "instance": "10.142.0.43:9100" 19:43:30 DEBUG }, 19:43:30 DEBUG "value": [ 19:43:30 DEBUG 1713552210, 19:43:30 DEBUG "70820296.06666666" 19:43:30 DEBUG ] 19:43:30 DEBUG } 19:43:30 DEBUG ] 19:43:30 DEBUG } 19:43:30 DEBUG } 19:43:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552210" 19:43:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552210" 19:43:30 INFO [http_cmd]: http status code OK 19:43:30 DEBUG --- status code --- 19:43:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:30 DEBUG --- http response --- 19:43:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:43:30 DEBUG ----- output ----- 19:43:30 DEBUG { 19:43:30 DEBUG "status": "success", 19:43:30 DEBUG "data": { 19:43:30 DEBUG "resultType": "vector", 19:43:30 DEBUG "result": [ 19:43:30 DEBUG { 19:43:30 DEBUG "metric": { 19:43:30 DEBUG "pod": "ds-idrepo-0" 19:43:30 DEBUG }, 19:43:30 DEBUG "value": [ 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG [...] 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG "pod": "ds-idrepo-2" 19:43:30 DEBUG }, 19:43:30 DEBUG "value": [ 19:43:30 DEBUG 1713552210, 19:43:30 DEBUG "0" 19:43:30 DEBUG ] 19:43:30 DEBUG } 19:43:30 DEBUG ] 19:43:30 DEBUG } 19:43:30 DEBUG } 19:43:30 INFO [http_cmd]: http status code OK 19:43:30 DEBUG --- status code --- 19:43:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:30 DEBUG --- http response --- 19:43:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:43:30 DEBUG ----- output ----- 19:43:30 DEBUG { 19:43:30 DEBUG "status": "success", 19:43:30 DEBUG "data": { 19:43:30 DEBUG "resultType": "vector", 19:43:30 DEBUG "result": [ 19:43:30 DEBUG { 19:43:30 DEBUG "metric": { 19:43:30 DEBUG "pod": "ds-idrepo-1" 19:43:30 DEBUG }, 19:43:30 DEBUG "value": [ 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG [...] 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:43:30 DEBUG }, 19:43:30 DEBUG "value": [ 19:43:30 DEBUG 1713552210, 19:43:30 DEBUG "0" 19:43:30 DEBUG ] 19:43:30 DEBUG } 19:43:30 DEBUG ] 19:43:30 DEBUG } 19:43:30 DEBUG } 19:43:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552210" 19:43:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713552210" 19:43:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552210" 19:43:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552210" 19:43:30 INFO [http_cmd]: http status code OK 19:43:30 DEBUG --- status code --- 19:43:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:30 DEBUG --- http response --- 19:43:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:43:30 DEBUG ----- output ----- 19:43:30 DEBUG { 19:43:30 DEBUG "status": "success", 19:43:30 DEBUG "data": { 19:43:30 DEBUG "resultType": "vector", 19:43:30 DEBUG "result": [ 19:43:30 DEBUG { 19:43:30 DEBUG "metric": { 19:43:30 DEBUG "pod": "ds-idrepo-0" 19:43:30 DEBUG }, 19:43:30 DEBUG "value": [ 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG [...] 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG "pod": "ds-idrepo-2" 19:43:30 DEBUG }, 19:43:30 DEBUG "value": [ 19:43:30 DEBUG 1713552210, 19:43:30 DEBUG "0" 19:43:30 DEBUG ] 19:43:30 DEBUG } 19:43:30 DEBUG ] 19:43:30 DEBUG } 19:43:30 DEBUG } 19:43:30 INFO [http_cmd]: http status code OK 19:43:30 DEBUG --- status code --- 19:43:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:30 DEBUG --- http response --- 19:43:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:43:30 DEBUG ----- output ----- 19:43:30 DEBUG { 19:43:30 DEBUG "status": "success", 19:43:30 DEBUG "data": { 19:43:30 DEBUG "resultType": "vector", 19:43:30 DEBUG "result": [ 19:43:30 DEBUG { 19:43:30 DEBUG "metric": { 19:43:30 DEBUG "pod": "ds-idrepo-0" 19:43:30 DEBUG }, 19:43:30 DEBUG "value": [ 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG [...] 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG "pod": "ds-cts-2" 19:43:30 DEBUG }, 19:43:30 DEBUG "value": [ 19:43:30 DEBUG 1713552210, 19:43:30 DEBUG "0" 19:43:30 DEBUG ] 19:43:30 DEBUG } 19:43:30 DEBUG ] 19:43:30 DEBUG } 19:43:30 DEBUG } 19:43:30 INFO [http_cmd]: http status code OK 19:43:30 DEBUG --- status code --- 19:43:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:30 DEBUG --- http response --- 19:43:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:43:30 DEBUG ----- output ----- 19:43:30 DEBUG { 19:43:30 DEBUG "status": "success", 19:43:30 DEBUG "data": { 19:43:30 DEBUG "resultType": "vector", 19:43:30 DEBUG "result": [ 19:43:30 DEBUG { 19:43:30 DEBUG "metric": { 19:43:30 DEBUG "pod": "ds-cts-0" 19:43:30 DEBUG }, 19:43:30 DEBUG "value": [ 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG [...] 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG "pod": "ds-cts-2" 19:43:30 DEBUG }, 19:43:30 DEBUG "value": [ 19:43:30 DEBUG 1713552210, 19:43:30 DEBUG "29.8" 19:43:30 DEBUG ] 19:43:30 DEBUG } 19:43:30 DEBUG ] 19:43:30 DEBUG } 19:43:30 DEBUG } 19:43:30 INFO [http_cmd]: http status code OK 19:43:30 DEBUG --- status code --- 19:43:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:30 DEBUG --- http response --- 19:43:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:43:30 DEBUG ----- output ----- 19:43:30 DEBUG { 19:43:30 DEBUG "status": "success", 19:43:30 DEBUG "data": { 19:43:30 DEBUG "resultType": "vector", 19:43:30 DEBUG "result": [ 19:43:30 DEBUG { 19:43:30 DEBUG "metric": { 19:43:30 DEBUG "instance": "10.142.0.107:9100" 19:43:30 DEBUG }, 19:43:30 DEBUG "value": [ 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG [...] 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG "instance": "10.142.0.43:9100" 19:43:30 DEBUG }, 19:43:30 DEBUG "value": [ 19:43:30 DEBUG 1713552210, 19:43:30 DEBUG "0.07766666666666802" 19:43:30 DEBUG ] 19:43:30 DEBUG } 19:43:30 DEBUG ] 19:43:30 DEBUG } 19:43:30 DEBUG } 19:43:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552210" 19:43:30 INFO [http_cmd]: http status code OK 19:43:30 DEBUG --- status code --- 19:43:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:30 DEBUG --- http response --- 19:43:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:43:30 DEBUG ----- output ----- 19:43:30 DEBUG { 19:43:30 DEBUG "status": "success", 19:43:30 DEBUG "data": { 19:43:30 DEBUG "resultType": "vector", 19:43:30 DEBUG "result": [ 19:43:30 DEBUG { 19:43:30 DEBUG "metric": { 19:43:30 DEBUG "pod": "ds-cts-0" 19:43:30 DEBUG }, 19:43:30 DEBUG "value": [ 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG [...] 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG "pod": "ds-cts-2" 19:43:30 DEBUG }, 19:43:30 DEBUG "value": [ 19:43:30 DEBUG 1713552210, 19:43:30 DEBUG "0.0009605755555555555" 19:43:30 DEBUG ] 19:43:30 DEBUG } 19:43:30 DEBUG ] 19:43:30 DEBUG } 19:43:30 DEBUG } 19:43:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552210" 19:43:30 INFO [http_cmd]: http status code OK 19:43:30 DEBUG --- status code --- 19:43:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:30 DEBUG --- http response --- 19:43:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:43:30 DEBUG ----- output ----- 19:43:30 DEBUG { 19:43:30 DEBUG "status": "success", 19:43:30 DEBUG "data": { 19:43:30 DEBUG "resultType": "vector", 19:43:30 DEBUG "result": [ 19:43:30 DEBUG { 19:43:30 DEBUG "metric": { 19:43:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:43:30 DEBUG }, 19:43:30 DEBUG "value": [ 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG [...] 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:43:30 DEBUG }, 19:43:30 DEBUG "value": [ 19:43:30 DEBUG 1713552210, 19:43:30 DEBUG "2336.3333333333335" 19:43:30 DEBUG ] 19:43:30 DEBUG } 19:43:30 DEBUG ] 19:43:30 DEBUG } 19:43:30 DEBUG } 19:43:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713552210" 19:43:30 INFO [http_cmd]: http status code OK 19:43:30 DEBUG --- status code --- 19:43:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:30 DEBUG --- http response --- 19:43:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:43:30 DEBUG ----- output ----- 19:43:30 DEBUG { 19:43:30 DEBUG "status": "success", 19:43:30 DEBUG "data": { 19:43:30 DEBUG "resultType": "vector", 19:43:30 DEBUG "result": [ 19:43:30 DEBUG { 19:43:30 DEBUG "metric": { 19:43:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:43:30 DEBUG }, 19:43:30 DEBUG "value": [ 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG [...] 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:43:30 DEBUG }, 19:43:30 DEBUG "value": [ 19:43:30 DEBUG 1713552210, 19:43:30 DEBUG "0.012" 19:43:30 DEBUG ] 19:43:30 DEBUG } 19:43:30 DEBUG ] 19:43:30 DEBUG } 19:43:30 DEBUG } 19:43:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552210" 19:43:30 INFO [http_cmd]: http status code OK 19:43:30 DEBUG --- status code --- 19:43:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:30 DEBUG --- http response --- 19:43:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:43:30 DEBUG ----- output ----- 19:43:30 DEBUG { 19:43:30 DEBUG "status": "success", 19:43:30 DEBUG "data": { 19:43:30 DEBUG "resultType": "vector", 19:43:30 DEBUG "result": [ 19:43:30 DEBUG { 19:43:30 DEBUG "metric": { 19:43:30 DEBUG "pod": "ds-idrepo-1" 19:43:30 DEBUG }, 19:43:30 DEBUG "value": [ 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG [...] 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:43:30 DEBUG }, 19:43:30 DEBUG "value": [ 19:43:30 DEBUG 1713552210, 19:43:30 DEBUG "0" 19:43:30 DEBUG ] 19:43:30 DEBUG } 19:43:30 DEBUG ] 19:43:30 DEBUG } 19:43:30 DEBUG } 19:43:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552210" 19:43:30 INFO [http_cmd]: http status code OK 19:43:30 DEBUG --- status code --- 19:43:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:30 DEBUG --- http response --- 19:43:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:43:30 DEBUG ----- output ----- 19:43:30 DEBUG { 19:43:30 DEBUG "status": "success", 19:43:30 DEBUG "data": { 19:43:30 DEBUG "resultType": "vector", 19:43:30 DEBUG "result": [ 19:43:30 DEBUG { 19:43:30 DEBUG "metric": { 19:43:30 DEBUG "pod": "ds-idrepo-1" 19:43:30 DEBUG }, 19:43:30 DEBUG "value": [ 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG [...] 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:43:30 DEBUG }, 19:43:30 DEBUG "value": [ 19:43:30 DEBUG 1713552210, 19:43:30 DEBUG "0.000009223825788872741" 19:43:30 DEBUG ] 19:43:30 DEBUG } 19:43:30 DEBUG ] 19:43:30 DEBUG } 19:43:30 DEBUG } 19:43:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713552210" 19:43:30 INFO [http_cmd]: http status code OK 19:43:30 DEBUG --- status code --- 19:43:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:30 DEBUG --- http response --- 19:43:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:43:30 DEBUG ----- output ----- 19:43:30 DEBUG { 19:43:30 DEBUG "status": "success", 19:43:30 DEBUG "data": { 19:43:30 DEBUG "resultType": "vector", 19:43:30 DEBUG "result": [ 19:43:30 DEBUG { 19:43:30 DEBUG "metric": { 19:43:30 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:43:30 DEBUG }, 19:43:30 DEBUG "value": [ 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG [...] 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:43:30 DEBUG }, 19:43:30 DEBUG "value": [ 19:43:30 DEBUG 1713552210, 19:43:30 DEBUG "3421315072" 19:43:30 DEBUG ] 19:43:30 DEBUG } 19:43:30 DEBUG ] 19:43:30 DEBUG } 19:43:30 DEBUG } 19:43:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552210" 19:43:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552210" 19:43:30 INFO [http_cmd]: http status code OK 19:43:30 DEBUG --- status code --- 19:43:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:30 DEBUG --- http response --- 19:43:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:43:30 DEBUG ----- output ----- 19:43:30 DEBUG { 19:43:30 DEBUG "status": "success", 19:43:30 DEBUG "data": { 19:43:30 DEBUG "resultType": "vector", 19:43:30 DEBUG "result": [ 19:43:30 DEBUG { 19:43:30 DEBUG "metric": { 19:43:30 DEBUG "instance": "10.142.0.107:9100" 19:43:30 DEBUG }, 19:43:30 DEBUG "value": [ 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG [...] 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG "instance": "10.142.0.43:9100" 19:43:30 DEBUG }, 19:43:30 DEBUG "value": [ 19:43:30 DEBUG 1713552210, 19:43:30 DEBUG "70816469.66666666" 19:43:30 DEBUG ] 19:43:30 DEBUG } 19:43:30 DEBUG ] 19:43:30 DEBUG } 19:43:30 DEBUG } 19:43:30 INFO [http_cmd]: http status code OK 19:43:30 DEBUG --- status code --- 19:43:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:30 DEBUG --- http response --- 19:43:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:43:30 DEBUG ----- output ----- 19:43:30 DEBUG { 19:43:30 DEBUG "status": "success", 19:43:30 DEBUG "data": { 19:43:30 DEBUG "resultType": "vector", 19:43:30 DEBUG "result": [ 19:43:30 DEBUG { 19:43:30 DEBUG "metric": { 19:43:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:43:30 DEBUG }, 19:43:30 DEBUG "value": [ 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG [...] 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG 19:43:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:43:30 DEBUG }, 19:43:30 DEBUG "value": [ 19:43:30 DEBUG 1713552210, 19:43:30 DEBUG "2336.266666666667" 19:43:30 DEBUG ] 19:43:30 DEBUG } 19:43:30 DEBUG ] 19:43:30 DEBUG } 19:43:30 DEBUG } 19:43:31 INFO 19:43:31 INFO [loop_until]: kubectl --namespace=xlou top pods 19:43:31 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:43:31 INFO [loop_until]: OK (rc = 0) 19:43:31 DEBUG --- stdout --- 19:43:31 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 14686m 8993Mi am-767f6b67b5-qcqz2 14175m 8999Mi am-767f6b67b5-vbltz 15294m 8982Mi ds-cts-0 15574m 10915Mi ds-cts-1 14040m 11149Mi ds-cts-2 15995m 10920Mi ds-idrepo-0 6422m 3663Mi ds-idrepo-1 6358m 3533Mi ds-idrepo-2 6464m 3492Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 1m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7583m 3170Mi 19:43:31 DEBUG --- stderr --- 19:43:31 DEBUG 19:43:31 INFO 19:43:31 INFO [loop_until]: kubectl --namespace=xlou top node 19:43:31 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:43:31 INFO [loop_until]: OK (rc = 0) 19:43:31 DEBUG --- stdout --- 19:43:31 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 67m 0% 6252Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 6354m 21% 7250Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 14154m 47% 11700Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 78m 0% 3340Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 15044m 50% 13925Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 157m 0% 5220Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 14318m 47% 11883Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12351m 41% 2988Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 16305m 54% 11192Mi 10% gke-xlou-cdm-ds-0f601751-j488 7051m 23% 6736Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 13934m 46% 15919Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 6822m 22% 7428Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 15573m 52% 13386Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 7500m 47% 6941Mi 11% 19:43:31 DEBUG --- stderr --- 19:43:31 DEBUG 19:44:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713552270" 19:44:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552270" 19:44:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552270" 19:44:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552270" 19:44:30 ERROR [http_cmd]: ERROR 19:44:30 INFO [http_cmd]: http status code OK 19:44:30 ERROR --- status code --- 19:44:30 DEBUG --- status code --- 19:44:30 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:44:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:30 ERROR --- http response --- 19:44:30 DEBUG --- http response --- 19:44:30 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:44:30 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:44:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552270" 19:44:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:44:30 DEBUG ----- output ----- 19:44:30 DEBUG { 19:44:30 DEBUG "status": "success", 19:44:30 DEBUG "data": { 19:44:30 DEBUG "resultType": "vector", 19:44:30 DEBUG "result": [ 19:44:30 DEBUG { 19:44:30 DEBUG "metric": { 19:44:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:44:30 DEBUG }, 19:44:30 DEBUG "value": [ 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG [...] 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:44:30 DEBUG }, 19:44:30 DEBUG "value": [ 19:44:30 DEBUG 1713552270, 19:44:30 DEBUG "418826" 19:44:30 DEBUG ] 19:44:30 DEBUG } 19:44:30 DEBUG ] 19:44:30 DEBUG } 19:44:30 DEBUG } 19:44:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552270" 19:44:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552270" 19:44:30 INFO [http_cmd]: http status code OK 19:44:30 DEBUG --- status code --- 19:44:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:30 DEBUG --- http response --- 19:44:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:44:30 DEBUG ----- output ----- 19:44:30 DEBUG { 19:44:30 DEBUG "status": "success", 19:44:30 DEBUG "data": { 19:44:30 DEBUG "resultType": "vector", 19:44:30 DEBUG "result": [ 19:44:30 DEBUG { 19:44:30 DEBUG "metric": { 19:44:30 DEBUG "instance": "10.142.0.107:9100" 19:44:30 DEBUG }, 19:44:30 DEBUG "value": [ 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG [...] 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG "instance": "10.142.0.43:9100" 19:44:30 DEBUG }, 19:44:30 DEBUG "value": [ 19:44:30 DEBUG 1713552270, 19:44:30 DEBUG "1944840545280" 19:44:30 DEBUG ] 19:44:30 DEBUG } 19:44:30 DEBUG ] 19:44:30 DEBUG } 19:44:30 DEBUG } 19:44:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552270" 19:44:30 INFO [http_cmd]: http status code OK 19:44:30 INFO [http_cmd]: http status code OK 19:44:30 DEBUG --- status code --- 19:44:30 DEBUG --- status code --- 19:44:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:30 DEBUG --- http response --- 19:44:30 DEBUG --- http response --- 19:44:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:44:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:44:30 DEBUG ----- output ----- 19:44:30 DEBUG ----- output ----- 19:44:30 DEBUG { 19:44:30 DEBUG { 19:44:30 DEBUG "status": "success", 19:44:30 DEBUG "status": "success", 19:44:30 DEBUG "data": { 19:44:30 DEBUG "data": { 19:44:30 DEBUG "resultType": "vector", 19:44:30 DEBUG "resultType": "vector", 19:44:30 DEBUG "result": [ 19:44:30 DEBUG "result": [ 19:44:30 DEBUG { 19:44:30 DEBUG { 19:44:30 DEBUG "metric": { 19:44:30 DEBUG "metric": { 19:44:30 DEBUG "pod": "ds-idrepo-0" 19:44:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:44:30 DEBUG }, 19:44:30 DEBUG }, 19:44:30 DEBUG "value": [ 19:44:30 DEBUG "value": [ 19:44:30 INFO [http_cmd]: http status code OK 19:44:30 INFO [http_cmd]: http status code OK 19:44:30 DEBUG 19:44:30 INFO [http_cmd]: http status code OK 19:44:30 DEBUG 19:44:30 DEBUG --- status code --- 19:44:30 DEBUG --- status code --- 19:44:30 DEBUG 19:44:30 DEBUG --- status code --- 19:44:30 DEBUG 19:44:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:30 DEBUG 19:44:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:30 DEBUG 19:44:30 DEBUG --- http response --- 19:44:30 DEBUG --- http response --- 19:44:30 DEBUG [...] 19:44:30 DEBUG --- http response --- 19:44:30 DEBUG [...] 19:44:30 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:44:30 DEBUG 19:44:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:44:30 DEBUG 19:44:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:44:30 DEBUG 19:44:30 DEBUG ----- output ----- 19:44:30 DEBUG 19:44:30 DEBUG ----- output ----- 19:44:30 DEBUG 19:44:30 DEBUG { 19:44:30 DEBUG 19:44:30 DEBUG { 19:44:30 DEBUG "pod": "ds-idrepo-2" 19:44:30 DEBUG "status": "success", 19:44:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:44:30 DEBUG "status": "success", 19:44:30 DEBUG }, 19:44:30 DEBUG "data": { 19:44:30 DEBUG }, 19:44:30 DEBUG "data": { 19:44:30 DEBUG "value": [ 19:44:30 DEBUG "resultType": "vector", 19:44:30 DEBUG "value": [ 19:44:30 DEBUG "resultType": "vector", 19:44:30 DEBUG 1713552270, 19:44:30 DEBUG "result": [ 19:44:30 DEBUG 1713552270, 19:44:30 DEBUG "result": [ 19:44:30 DEBUG "0" 19:44:30 DEBUG { 19:44:30 DEBUG "0" 19:44:30 DEBUG { 19:44:30 DEBUG ] 19:44:30 DEBUG "metric": { 19:44:30 DEBUG ] 19:44:30 DEBUG "metric": { 19:44:30 DEBUG } 19:44:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:44:30 DEBUG } 19:44:30 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:44:30 DEBUG ] 19:44:30 DEBUG }, 19:44:30 DEBUG ] 19:44:30 DEBUG }, 19:44:30 DEBUG } 19:44:30 DEBUG "value": [ 19:44:30 DEBUG } 19:44:30 DEBUG "value": [ 19:44:30 DEBUG } 19:44:30 DEBUG 19:44:30 DEBUG } 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG [...] 19:44:30 DEBUG [...] 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:44:30 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:44:30 DEBUG }, 19:44:30 DEBUG }, 19:44:30 DEBUG "value": [ 19:44:30 DEBUG "value": [ 19:44:30 DEBUG 1713552270, 19:44:30 DEBUG 1713552270, 19:44:30 DEBUG "2237.7666666666664" 19:44:30 DEBUG "33880.54246417236" 19:44:30 DEBUG ] 19:44:30 DEBUG ] 19:44:30 DEBUG } 19:44:30 DEBUG } 19:44:30 DEBUG ] 19:44:30 DEBUG ] 19:44:30 DEBUG } 19:44:30 DEBUG } 19:44:30 DEBUG } 19:44:30 DEBUG } 19:44:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552270" 19:44:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552270" 19:44:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713552270" 19:44:30 INFO [http_cmd]: http status code OK 19:44:30 DEBUG --- status code --- 19:44:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:30 DEBUG --- http response --- 19:44:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:44:30 DEBUG ----- output ----- 19:44:30 DEBUG { 19:44:30 DEBUG "status": "success", 19:44:30 DEBUG "data": { 19:44:30 DEBUG "resultType": "vector", 19:44:30 DEBUG "result": [ 19:44:30 DEBUG { 19:44:30 DEBUG "metric": { 19:44:30 DEBUG "pod": "ds-cts-0" 19:44:30 DEBUG }, 19:44:30 DEBUG "value": [ 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG [...] 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG "pod": "ds-cts-2" 19:44:30 DEBUG }, 19:44:30 DEBUG "value": [ 19:44:30 DEBUG 1713552270, 19:44:30 DEBUG "0" 19:44:30 DEBUG ] 19:44:30 DEBUG } 19:44:30 DEBUG ] 19:44:30 DEBUG } 19:44:30 DEBUG } 19:44:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552270" 19:44:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552270" 19:44:30 INFO [http_cmd]: http status code OK 19:44:30 DEBUG --- status code --- 19:44:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:30 DEBUG --- http response --- 19:44:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:44:30 DEBUG ----- output ----- 19:44:30 DEBUG { 19:44:30 DEBUG "status": "success", 19:44:30 DEBUG "data": { 19:44:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713552270" 19:44:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552270" 19:44:30 DEBUG "resultType": "vector", 19:44:30 DEBUG "result": [ 19:44:30 DEBUG { 19:44:30 DEBUG "metric": { 19:44:30 DEBUG "pod": "ds-idrepo-0" 19:44:30 DEBUG }, 19:44:30 DEBUG "value": [ 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG [...] 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG "pod": "ds-cts-2" 19:44:30 DEBUG }, 19:44:30 DEBUG "value": [ 19:44:30 DEBUG 1713552270, 19:44:30 DEBUG "0" 19:44:30 DEBUG ] 19:44:30 DEBUG } 19:44:30 DEBUG ] 19:44:30 DEBUG } 19:44:30 DEBUG } 19:44:30 INFO [http_cmd]: http status code OK 19:44:30 DEBUG --- status code --- 19:44:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:30 DEBUG --- http response --- 19:44:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:44:30 DEBUG ----- output ----- 19:44:30 DEBUG { 19:44:30 DEBUG "status": "success", 19:44:30 DEBUG "data": { 19:44:30 DEBUG "resultType": "vector", 19:44:30 DEBUG "result": [ 19:44:30 DEBUG { 19:44:30 DEBUG "metric": { 19:44:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:44:30 DEBUG }, 19:44:30 DEBUG "value": [ 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG [...] 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:44:30 DEBUG }, 19:44:30 DEBUG "value": [ 19:44:30 DEBUG 1713552270, 19:44:30 DEBUG "2329.9" 19:44:30 DEBUG ] 19:44:30 DEBUG } 19:44:30 DEBUG ] 19:44:30 DEBUG } 19:44:30 DEBUG } 19:44:30 INFO [http_cmd]: http status code OK 19:44:30 DEBUG --- status code --- 19:44:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:30 DEBUG --- http response --- 19:44:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:44:30 DEBUG ----- output ----- 19:44:30 DEBUG { 19:44:30 DEBUG "status": "success", 19:44:30 DEBUG "data": { 19:44:30 DEBUG "resultType": "vector", 19:44:30 DEBUG "result": [ 19:44:30 DEBUG { 19:44:30 DEBUG "metric": { 19:44:30 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:44:30 DEBUG }, 19:44:30 DEBUG "value": [ 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG [...] 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:44:30 DEBUG }, 19:44:30 DEBUG "value": [ 19:44:30 DEBUG 1713552270, 19:44:30 DEBUG "0" 19:44:30 DEBUG ] 19:44:30 DEBUG } 19:44:30 DEBUG ] 19:44:30 DEBUG } 19:44:30 DEBUG } 19:44:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552270" 19:44:30 INFO [http_cmd]: http status code OK 19:44:30 DEBUG --- status code --- 19:44:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:30 DEBUG --- http response --- 19:44:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:44:30 DEBUG ----- output ----- 19:44:30 DEBUG { 19:44:30 DEBUG "status": "success", 19:44:30 DEBUG "data": { 19:44:30 DEBUG "resultType": "vector", 19:44:30 DEBUG "result": [ 19:44:30 DEBUG { 19:44:30 DEBUG "metric": { 19:44:30 DEBUG "instance": "10.142.0.107:9100" 19:44:30 DEBUG }, 19:44:30 DEBUG "value": [ 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG [...] 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG "instance": "10.142.0.43:9100" 19:44:30 DEBUG }, 19:44:30 DEBUG "value": [ 19:44:30 DEBUG 1713552270, 19:44:30 DEBUG "0.16140000000005159" 19:44:30 DEBUG ] 19:44:30 DEBUG } 19:44:30 DEBUG ] 19:44:30 DEBUG } 19:44:30 DEBUG } 19:44:30 INFO [http_cmd]: http status code OK 19:44:30 DEBUG --- status code --- 19:44:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:30 DEBUG --- http response --- 19:44:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:44:30 DEBUG ----- output ----- 19:44:30 DEBUG { 19:44:30 DEBUG "status": "success", 19:44:30 DEBUG "data": { 19:44:30 DEBUG "resultType": "vector", 19:44:30 DEBUG "result": [ 19:44:30 DEBUG { 19:44:30 DEBUG "metric": { 19:44:30 DEBUG "instance": "10.142.0.107:9100" 19:44:30 DEBUG }, 19:44:30 DEBUG "value": [ 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG [...] 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG "instance": "10.142.0.43:9100" 19:44:30 DEBUG }, 19:44:30 DEBUG "value": [ 19:44:30 DEBUG 1713552270, 19:44:30 DEBUG "0.07633333333333593" 19:44:30 DEBUG ] 19:44:30 DEBUG } 19:44:30 DEBUG ] 19:44:30 DEBUG } 19:44:30 DEBUG } 19:44:30 INFO [http_cmd]: http status code OK 19:44:30 DEBUG --- status code --- 19:44:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:30 DEBUG --- http response --- 19:44:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:44:30 DEBUG ----- output ----- 19:44:30 DEBUG { 19:44:30 DEBUG "status": "success", 19:44:30 DEBUG "data": { 19:44:30 DEBUG "resultType": "vector", 19:44:30 DEBUG "result": [ 19:44:30 DEBUG { 19:44:30 DEBUG "metric": { 19:44:30 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:44:30 DEBUG }, 19:44:30 DEBUG "value": [ 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG [...] 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:44:30 DEBUG }, 19:44:30 DEBUG "value": [ 19:44:30 DEBUG 1713552270, 19:44:30 DEBUG "7.693656097086681" 19:44:30 DEBUG ] 19:44:30 DEBUG } 19:44:30 DEBUG ] 19:44:30 DEBUG } 19:44:30 DEBUG } 19:44:30 INFO [http_cmd]: http status code OK 19:44:30 DEBUG --- status code --- 19:44:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:30 DEBUG --- http response --- 19:44:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:44:30 DEBUG ----- output ----- 19:44:30 DEBUG { 19:44:30 DEBUG "status": "success", 19:44:30 DEBUG "data": { 19:44:30 DEBUG "resultType": "vector", 19:44:30 DEBUG "result": [ 19:44:30 DEBUG { 19:44:30 DEBUG "metric": { 19:44:30 DEBUG "instance": "10.142.0.107:9100" 19:44:30 DEBUG }, 19:44:30 DEBUG "value": [ 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG [...] 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG "instance": "10.142.0.43:9100" 19:44:30 DEBUG }, 19:44:30 DEBUG "value": [ 19:44:30 DEBUG 1713552270, 19:44:30 DEBUG "70718026.9" 19:44:30 DEBUG ] 19:44:30 DEBUG } 19:44:30 DEBUG ] 19:44:30 DEBUG } 19:44:30 DEBUG } 19:44:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552270" 19:44:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552270" 19:44:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552270" 19:44:30 INFO [http_cmd]: http status code OK 19:44:30 DEBUG --- status code --- 19:44:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:30 DEBUG --- http response --- 19:44:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:44:30 DEBUG ----- output ----- 19:44:30 DEBUG { 19:44:30 DEBUG "status": "success", 19:44:30 DEBUG "data": { 19:44:30 DEBUG "resultType": "vector", 19:44:30 DEBUG "result": [ 19:44:30 DEBUG { 19:44:30 DEBUG "metric": { 19:44:30 DEBUG "pod": "ds-idrepo-0" 19:44:30 DEBUG }, 19:44:30 DEBUG "value": [ 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG [...] 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG "pod": "ds-idrepo-2" 19:44:30 DEBUG }, 19:44:30 DEBUG "value": [ 19:44:30 DEBUG 1713552270, 19:44:30 DEBUG "0" 19:44:30 DEBUG ] 19:44:30 DEBUG } 19:44:30 DEBUG ] 19:44:30 DEBUG } 19:44:30 DEBUG } 19:44:30 INFO [http_cmd]: http status code OK 19:44:30 DEBUG --- status code --- 19:44:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:30 DEBUG --- http response --- 19:44:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:44:30 INFO [http_cmd]: http status code OK 19:44:30 DEBUG ----- output ----- 19:44:30 DEBUG --- status code --- 19:44:30 DEBUG { 19:44:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:30 DEBUG "status": "success", 19:44:30 DEBUG --- http response --- 19:44:30 DEBUG "data": { 19:44:30 DEBUG "resultType": "vector", 19:44:30 DEBUG "result": [ 19:44:30 DEBUG { 19:44:30 DEBUG "metric": { 19:44:30 DEBUG "pod": "ds-cts-0" 19:44:30 DEBUG }, 19:44:30 DEBUG "value": [ 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG [...] 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG "pod": "ds-cts-2" 19:44:30 DEBUG }, 19:44:30 DEBUG "value": [ 19:44:30 DEBUG 1713552270, 19:44:30 DEBUG "0" 19:44:30 DEBUG ] 19:44:30 DEBUG } 19:44:30 DEBUG ] 19:44:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:44:30 DEBUG } 19:44:30 DEBUG ----- output ----- 19:44:30 DEBUG } 19:44:30 DEBUG { 19:44:30 DEBUG "status": "success", 19:44:30 DEBUG "data": { 19:44:30 DEBUG "resultType": "vector", 19:44:30 DEBUG "result": [ 19:44:30 DEBUG { 19:44:30 DEBUG "metric": { 19:44:30 DEBUG "pod": "ds-cts-0" 19:44:30 DEBUG }, 19:44:30 DEBUG "value": [ 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG [...] 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG "pod": "ds-cts-2" 19:44:30 DEBUG }, 19:44:30 DEBUG "value": [ 19:44:30 DEBUG 1713552270, 19:44:30 DEBUG "29.833333333333332" 19:44:30 DEBUG ] 19:44:30 DEBUG } 19:44:30 DEBUG ] 19:44:30 DEBUG } 19:44:30 DEBUG } 19:44:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552270" 19:44:30 INFO [http_cmd]: http status code OK 19:44:30 DEBUG --- status code --- 19:44:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:30 DEBUG --- http response --- 19:44:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:44:30 DEBUG ----- output ----- 19:44:30 DEBUG { 19:44:30 DEBUG "status": "success", 19:44:30 DEBUG "data": { 19:44:30 DEBUG "resultType": "vector", 19:44:30 DEBUG "result": [ 19:44:30 DEBUG { 19:44:30 DEBUG "metric": { 19:44:30 DEBUG "pod": "ds-idrepo-0" 19:44:30 DEBUG }, 19:44:30 DEBUG "value": [ 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG [...] 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG "pod": "ds-idrepo-2" 19:44:30 DEBUG }, 19:44:30 DEBUG "value": [ 19:44:30 DEBUG 1713552270, 19:44:30 DEBUG "0" 19:44:30 DEBUG ] 19:44:30 DEBUG } 19:44:30 DEBUG ] 19:44:30 DEBUG } 19:44:30 DEBUG } 19:44:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552270" 19:44:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713552270" 19:44:30 INFO [http_cmd]: http status code OK 19:44:30 INFO [http_cmd]: http status code OK 19:44:30 DEBUG --- status code --- 19:44:30 DEBUG --- status code --- 19:44:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:30 DEBUG --- http response --- 19:44:30 DEBUG --- http response --- 19:44:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:44:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:44:30 DEBUG ----- output ----- 19:44:30 DEBUG ----- output ----- 19:44:30 DEBUG { 19:44:30 DEBUG { 19:44:30 DEBUG "status": "success", 19:44:30 DEBUG "status": "success", 19:44:30 DEBUG "data": { 19:44:30 DEBUG "data": { 19:44:30 DEBUG "resultType": "vector", 19:44:30 DEBUG "resultType": "vector", 19:44:30 DEBUG "result": [ 19:44:30 DEBUG "result": [ 19:44:30 DEBUG { 19:44:30 DEBUG { 19:44:30 DEBUG "metric": { 19:44:30 DEBUG "metric": { 19:44:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:44:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:44:30 DEBUG }, 19:44:30 DEBUG }, 19:44:30 DEBUG "value": [ 19:44:30 DEBUG "value": [ 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG [...] 19:44:30 DEBUG [...] 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:44:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:44:30 DEBUG }, 19:44:30 DEBUG }, 19:44:30 DEBUG "value": [ 19:44:30 DEBUG "value": [ 19:44:30 DEBUG 1713552270, 19:44:30 DEBUG 1713552270, 19:44:30 DEBUG "2237.0333333333333" 19:44:30 DEBUG "0.012" 19:44:30 DEBUG ] 19:44:30 DEBUG ] 19:44:30 DEBUG } 19:44:30 DEBUG } 19:44:30 DEBUG ] 19:44:30 DEBUG ] 19:44:30 DEBUG } 19:44:30 DEBUG } 19:44:30 DEBUG } 19:44:30 DEBUG } 19:44:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552270" 19:44:30 INFO [http_cmd]: http status code OK 19:44:30 DEBUG --- status code --- 19:44:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:30 DEBUG --- http response --- 19:44:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:44:30 DEBUG ----- output ----- 19:44:30 DEBUG { 19:44:30 DEBUG "status": "success", 19:44:30 DEBUG "data": { 19:44:30 DEBUG "resultType": "vector", 19:44:30 DEBUG "result": [ 19:44:30 DEBUG { 19:44:30 DEBUG "metric": { 19:44:30 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:44:30 DEBUG }, 19:44:30 DEBUG "value": [ 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG [...] 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:44:30 DEBUG }, 19:44:30 DEBUG "value": [ 19:44:30 DEBUG 1713552270, 19:44:30 DEBUG "7.638281580423363" 19:44:30 DEBUG ] 19:44:30 DEBUG } 19:44:30 DEBUG ] 19:44:30 DEBUG } 19:44:30 DEBUG } 19:44:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552270" 19:44:30 INFO [http_cmd]: http status code OK 19:44:30 DEBUG --- status code --- 19:44:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:30 DEBUG --- http response --- 19:44:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:44:30 DEBUG ----- output ----- 19:44:30 DEBUG { 19:44:30 DEBUG "status": "success", 19:44:30 DEBUG "data": { 19:44:30 DEBUG "resultType": "vector", 19:44:30 DEBUG "result": [ 19:44:30 DEBUG { 19:44:30 DEBUG "metric": { 19:44:30 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:44:30 DEBUG }, 19:44:30 DEBUG "value": [ 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG [...] 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:44:30 DEBUG }, 19:44:30 DEBUG "value": [ 19:44:30 DEBUG 1713552270, 19:44:30 DEBUG "0" 19:44:30 DEBUG ] 19:44:30 DEBUG } 19:44:30 DEBUG ] 19:44:30 DEBUG } 19:44:30 DEBUG } 19:44:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552270" 19:44:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713552270" 19:44:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552270" 19:44:30 INFO [http_cmd]: http status code OK 19:44:30 DEBUG --- status code --- 19:44:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:30 DEBUG --- http response --- 19:44:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:44:30 DEBUG ----- output ----- 19:44:30 DEBUG { 19:44:30 DEBUG "status": "success", 19:44:30 DEBUG "data": { 19:44:30 DEBUG "resultType": "vector", 19:44:30 DEBUG "result": [ 19:44:30 DEBUG { 19:44:30 DEBUG "metric": { 19:44:30 DEBUG "instance": "10.142.0.107:9100" 19:44:30 DEBUG }, 19:44:30 DEBUG "value": [ 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG [...] 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG "instance": "10.142.0.43:9100" 19:44:30 DEBUG }, 19:44:30 DEBUG "value": [ 19:44:30 DEBUG 1713552270, 19:44:30 DEBUG "70716094.53333333" 19:44:30 DEBUG ] 19:44:30 DEBUG } 19:44:30 DEBUG ] 19:44:30 DEBUG } 19:44:30 DEBUG } 19:44:30 INFO [http_cmd]: http status code OK 19:44:30 DEBUG --- status code --- 19:44:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:30 DEBUG --- http response --- 19:44:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:44:30 DEBUG ----- output ----- 19:44:30 DEBUG { 19:44:30 DEBUG "status": "success", 19:44:30 DEBUG "data": { 19:44:30 DEBUG "resultType": "vector", 19:44:30 DEBUG "result": [ 19:44:30 DEBUG { 19:44:30 DEBUG "metric": { 19:44:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:44:30 DEBUG }, 19:44:30 DEBUG "value": [ 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG [...] 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:44:30 DEBUG }, 19:44:30 DEBUG "value": [ 19:44:30 DEBUG 1713552270, 19:44:30 DEBUG "2236.0666666666666" 19:44:30 DEBUG ] 19:44:30 DEBUG } 19:44:30 DEBUG ] 19:44:30 DEBUG } 19:44:30 DEBUG } 19:44:30 INFO [http_cmd]: http status code OK 19:44:30 DEBUG --- status code --- 19:44:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:30 DEBUG --- http response --- 19:44:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:44:30 DEBUG ----- output ----- 19:44:30 DEBUG { 19:44:30 DEBUG "status": "success", 19:44:30 DEBUG "data": { 19:44:30 DEBUG "resultType": "vector", 19:44:30 DEBUG "result": [ 19:44:30 DEBUG { 19:44:30 DEBUG "metric": { 19:44:30 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:44:30 DEBUG }, 19:44:30 DEBUG "value": [ 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG [...] 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG 19:44:30 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:44:30 DEBUG }, 19:44:30 DEBUG "value": [ 19:44:30 DEBUG 1713552270, 19:44:30 DEBUG "3423408128" 19:44:30 DEBUG ] 19:44:30 DEBUG } 19:44:30 DEBUG ] 19:44:30 DEBUG } 19:44:30 DEBUG } 19:44:31 INFO 19:44:31 INFO [loop_until]: kubectl --namespace=xlou top pods 19:44:31 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:44:31 INFO [loop_until]: OK (rc = 0) 19:44:31 DEBUG --- stdout --- 19:44:31 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 14261m 8993Mi am-767f6b67b5-qcqz2 14345m 9002Mi am-767f6b67b5-vbltz 14439m 8982Mi ds-cts-0 15609m 10919Mi ds-cts-1 14064m 11156Mi ds-cts-2 15164m 10925Mi ds-idrepo-0 6538m 3663Mi ds-idrepo-1 5757m 3533Mi ds-idrepo-2 5583m 3492Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 6760m 2675Mi 19:44:31 DEBUG --- stderr --- 19:44:31 DEBUG 19:44:31 INFO 19:44:31 INFO [loop_until]: kubectl --namespace=xlou top node 19:44:31 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:44:31 INFO [loop_until]: OK (rc = 0) 19:44:31 DEBUG --- stdout --- 19:44:31 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 71m 0% 6252Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 6384m 21% 7263Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 11822m 39% 10571Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 3337Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 15722m 52% 13924Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 138m 0% 5219Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 12844m 43% 11887Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 10811m 36% 2987Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 14381m 48% 11207Mi 10% gke-xlou-cdm-ds-0f601751-j488 5651m 18% 6739Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 14109m 47% 15934Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 5564m 18% 7431Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 15575m 52% 13394Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 7915m 49% 6944Mi 11% 19:44:31 DEBUG --- stderr --- 19:44:31 DEBUG 19:44:43 WARNING File /tmp/lodemon_data-20240419_191228/node_disk_read_bytes_total.json does not exist. Skipping average value computation... 127.0.0.1 - - [19/Apr/2024 19:44:43] "GET /monitoring/average?start_time=20240419_181530&stop_time=20240419_184242 HTTP/1.1" 200 - 19:45:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713552330" 19:45:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552330" 19:45:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552330" 19:45:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552330" 19:45:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713552330" 19:45:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552330" 19:45:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552330" 19:45:30 ERROR [http_cmd]: ERROR 19:45:30 ERROR --- status code --- 19:45:30 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:45:30 ERROR --- http response --- 19:45:30 INFO [http_cmd]: http status code OK 19:45:30 DEBUG --- status code --- 19:45:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:30 DEBUG --- http response --- 19:45:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552330" 19:45:30 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:45:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713552330" 19:45:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552330" 19:45:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552330" 19:45:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552330" 19:45:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:45:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552330" 19:45:30 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:45:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552330" 19:45:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552330" 19:45:30 INFO [http_cmd]: http status code OK 19:45:30 INFO [http_cmd]: http status code OK 19:45:30 INFO [http_cmd]: http status code OK 19:45:30 DEBUG ----- output ----- 19:45:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552330" 19:45:30 INFO [http_cmd]: http status code OK 19:45:30 DEBUG --- status code --- 19:45:30 DEBUG --- status code --- 19:45:30 DEBUG --- status code --- 19:45:30 DEBUG { 19:45:30 DEBUG --- status code --- 19:45:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:30 INFO [http_cmd]: http status code OK 19:45:30 INFO [http_cmd]: http status code OK 19:45:30 INFO [http_cmd]: http status code OK 19:45:30 INFO [http_cmd]: http status code OK 19:45:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:30 DEBUG "status": "success", 19:45:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:30 DEBUG --- http response --- 19:45:30 DEBUG --- status code --- 19:45:30 DEBUG --- status code --- 19:45:30 DEBUG --- status code --- 19:45:30 DEBUG --- status code --- 19:45:30 DEBUG --- http response --- 19:45:30 DEBUG --- http response --- 19:45:30 DEBUG "data": { 19:45:30 DEBUG --- http response --- 19:45:30 INFO [http_cmd]: http status code OK 19:45:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:45:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:30 INFO [http_cmd]: http status code OK 19:45:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:45:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:45:30 INFO [http_cmd]: http status code OK 19:45:30 INFO [http_cmd]: http status code OK 19:45:30 DEBUG "resultType": "vector", 19:45:30 INFO [http_cmd]: http status code OK 19:45:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:45:30 DEBUG --- status code --- 19:45:30 DEBUG --- http response --- 19:45:30 DEBUG ----- output ----- 19:45:30 DEBUG { 19:45:30 DEBUG --- http response --- 19:45:30 DEBUG --- http response --- 19:45:30 DEBUG --- http response --- 19:45:30 DEBUG ----- output ----- 19:45:30 DEBUG ----- output ----- 19:45:30 DEBUG --- status code --- 19:45:30 DEBUG --- status code --- 19:45:30 DEBUG "result": [ 19:45:30 DEBUG --- status code --- 19:45:30 DEBUG ----- output ----- 19:45:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:30 DEBUG --- status code --- 19:45:30 DEBUG "status": "success", 19:45:30 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:45:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:45:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:45:30 DEBUG { 19:45:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:45:30 DEBUG { 19:45:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:30 DEBUG { 19:45:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:30 DEBUG { 19:45:30 DEBUG --- http response --- 19:45:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:30 DEBUG "data": { 19:45:30 DEBUG ----- output ----- 19:45:30 DEBUG ----- output ----- 19:45:30 DEBUG "status": "success", 19:45:30 INFO [http_cmd]: http status code OK 19:45:30 DEBUG ----- output ----- 19:45:30 DEBUG "status": "success", 19:45:30 DEBUG --- http response --- 19:45:30 DEBUG --- http response --- 19:45:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552330" 19:45:30 DEBUG "metric": { 19:45:30 DEBUG --- http response --- 19:45:30 DEBUG "status": "success", 19:45:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:45:30 DEBUG --- http response --- 19:45:30 DEBUG "resultType": "vector", 19:45:30 DEBUG { 19:45:30 DEBUG { 19:45:30 DEBUG "data": { 19:45:30 DEBUG --- status code --- 19:45:30 DEBUG { 19:45:30 DEBUG "data": { 19:45:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:45:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:45:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:45:30 DEBUG "data": { 19:45:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:45:30 DEBUG ----- output ----- 19:45:30 DEBUG { 19:45:30 DEBUG "result": [ 19:45:30 DEBUG "status": "success", 19:45:30 DEBUG "status": "success", 19:45:30 DEBUG "resultType": "vector", 19:45:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:30 DEBUG "status": "success", 19:45:30 DEBUG "resultType": "vector", 19:45:30 DEBUG ----- output ----- 19:45:30 DEBUG ----- output ----- 19:45:30 DEBUG }, 19:45:30 DEBUG "resultType": "vector", 19:45:30 DEBUG ----- output ----- 19:45:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:45:30 DEBUG "status": "success", 19:45:30 DEBUG { 19:45:30 DEBUG "data": { 19:45:30 DEBUG "data": { 19:45:30 DEBUG "result": [ 19:45:30 DEBUG --- http response --- 19:45:30 DEBUG "data": { 19:45:30 DEBUG "result": [ 19:45:30 DEBUG { 19:45:30 DEBUG { 19:45:30 DEBUG "value": [ 19:45:30 DEBUG "result": [ 19:45:30 DEBUG { 19:45:30 DEBUG ----- output ----- 19:45:30 DEBUG "data": { 19:45:30 DEBUG "resultType": "vector", 19:45:30 DEBUG "resultType": "vector", 19:45:30 DEBUG "resultType": "vector", 19:45:30 DEBUG { 19:45:30 DEBUG "resultType": "vector", 19:45:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:45:30 DEBUG { 19:45:30 DEBUG "status": "success", 19:45:30 DEBUG "status": "success", 19:45:30 DEBUG 19:45:30 DEBUG { 19:45:30 DEBUG "status": "success", 19:45:30 DEBUG { 19:45:30 DEBUG "metric": { 19:45:30 DEBUG "result": [ 19:45:30 DEBUG "result": [ 19:45:30 DEBUG "result": [ 19:45:30 DEBUG "metric": { 19:45:30 DEBUG "result": [ 19:45:30 DEBUG ----- output ----- 19:45:30 DEBUG "metric": { 19:45:30 DEBUG "data": { 19:45:30 DEBUG "data": { 19:45:30 DEBUG 19:45:30 DEBUG "metric": { 19:45:30 DEBUG "data": { 19:45:30 DEBUG "status": "success", 19:45:30 DEBUG "pod": "ds-idrepo-0" 19:45:30 DEBUG { 19:45:30 DEBUG { 19:45:30 DEBUG { 19:45:30 DEBUG "instance": "10.142.0.107:9100" 19:45:30 DEBUG { 19:45:30 DEBUG { 19:45:30 DEBUG "instance": "10.142.0.107:9100" 19:45:30 DEBUG "resultType": "vector", 19:45:30 DEBUG "resultType": "vector", 19:45:30 DEBUG 19:45:30 DEBUG "instance": "10.142.0.107:9100" 19:45:30 DEBUG "resultType": "vector", 19:45:30 DEBUG "data": { 19:45:30 DEBUG }, 19:45:30 DEBUG "metric": { 19:45:30 DEBUG "metric": { 19:45:30 DEBUG "metric": { 19:45:30 DEBUG }, 19:45:30 DEBUG "metric": { 19:45:30 DEBUG "status": "success", 19:45:30 DEBUG }, 19:45:30 DEBUG "result": [ 19:45:30 DEBUG "result": [ 19:45:30 DEBUG [...] 19:45:30 DEBUG }, 19:45:30 DEBUG "result": [ 19:45:30 DEBUG "resultType": "vector", 19:45:30 DEBUG "value": [ 19:45:30 DEBUG "pod": "ds-idrepo-0" 19:45:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:45:30 DEBUG "pod": "ds-idrepo-0" 19:45:30 DEBUG "value": [ 19:45:30 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:45:30 DEBUG "data": { 19:45:30 DEBUG "value": [ 19:45:30 DEBUG { 19:45:30 DEBUG { 19:45:30 DEBUG 19:45:30 DEBUG "value": [ 19:45:30 DEBUG { 19:45:30 DEBUG "result": [ 19:45:30 DEBUG 19:45:30 DEBUG }, 19:45:30 DEBUG }, 19:45:30 DEBUG }, 19:45:30 DEBUG 19:45:30 DEBUG }, 19:45:30 DEBUG "resultType": "vector", 19:45:30 DEBUG 19:45:30 DEBUG "metric": { 19:45:30 DEBUG "metric": { 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG "metric": { 19:45:30 DEBUG { 19:45:30 DEBUG "metric": { 19:45:30 DEBUG "value": [ 19:45:30 DEBUG "value": [ 19:45:30 DEBUG "value": [ 19:45:30 DEBUG 19:45:30 DEBUG "value": [ 19:45:30 DEBUG "result": [ 19:45:30 DEBUG 19:45:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:45:30 DEBUG "instance": "10.142.0.107:9100" 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:45:30 DEBUG }, 19:45:30 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG { 19:45:30 DEBUG 19:45:30 DEBUG }, 19:45:30 DEBUG }, 19:45:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:45:30 DEBUG 19:45:30 INFO [http_cmd]: http status code OK 19:45:30 DEBUG "value": [ 19:45:30 DEBUG 19:45:30 DEBUG }, 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG "metric": { 19:45:30 DEBUG [...] 19:45:30 DEBUG "value": [ 19:45:30 DEBUG "value": [ 19:45:30 DEBUG }, 19:45:30 DEBUG [...] 19:45:30 DEBUG --- status code --- 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG [...] 19:45:30 DEBUG [...] 19:45:30 DEBUG 19:45:30 DEBUG "pod": "ds-cts-0" 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG "value": [ 19:45:30 DEBUG 19:45:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:30 DEBUG "value": [ 19:45:30 DEBUG 19:45:30 DEBUG [...] 19:45:30 DEBUG [...] 19:45:30 DEBUG [...] 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG }, 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG 1713552330, 19:45:30 DEBUG "319301.5" 19:45:30 DEBUG --- http response --- 19:45:30 DEBUG 19:45:30 DEBUG [...] 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG "pod": "ds-idrepo-2" 19:45:30 DEBUG "value": [ 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG ] 19:45:30 DEBUG 19:45:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG }, 19:45:30 DEBUG [...] 19:45:30 DEBUG 19:45:30 DEBUG "instance": "10.142.0.43:9100" 19:45:30 DEBUG [...] 19:45:30 DEBUG [...] 19:45:30 DEBUG 19:45:30 DEBUG } 19:45:30 DEBUG 19:45:30 DEBUG ----- output ----- 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG "instance": "10.142.0.43:9100" 19:45:30 DEBUG "value": [ 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG }, 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG "instance": "10.142.0.43:9100" 19:45:30 DEBUG ] 19:45:30 DEBUG [...] 19:45:30 DEBUG { 19:45:30 DEBUG 19:45:30 DEBUG "pod": "ds-cts-2" 19:45:30 DEBUG "pod": "ds-idrepo-2" 19:45:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:45:30 DEBUG }, 19:45:30 DEBUG 1713552330, 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG "value": [ 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG }, 19:45:30 DEBUG } 19:45:30 DEBUG 19:45:30 DEBUG "status": "success", 19:45:30 DEBUG "pod": "idm-b85f9d978-txpwv" 19:45:30 DEBUG }, 19:45:30 DEBUG }, 19:45:30 DEBUG }, 19:45:30 DEBUG "value": [ 19:45:30 DEBUG "0" 19:45:30 DEBUG 19:45:30 DEBUG [...] 19:45:30 DEBUG 1713552330, 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG "value": [ 19:45:30 DEBUG } 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG "pod": "idm-b85f9d978-txpwv" 19:45:30 DEBUG "value": [ 19:45:30 DEBUG "value": [ 19:45:30 DEBUG "value": [ 19:45:30 DEBUG 1713552330, 19:45:30 DEBUG ] 19:45:30 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:45:30 DEBUG 19:45:30 DEBUG "0.08666666666667122" 19:45:30 DEBUG ] 19:45:30 DEBUG "instance": "10.142.0.43:9100" 19:45:30 DEBUG 1713552330, 19:45:30 DEBUG "data": { 19:45:30 DEBUG }, 19:45:30 DEBUG "value": [ 19:45:30 DEBUG 1713552330, 19:45:30 DEBUG 1713552330, 19:45:30 DEBUG 1713552330, 19:45:30 DEBUG "1946585566208" 19:45:30 DEBUG } 19:45:30 DEBUG }, 19:45:30 DEBUG 19:45:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:45:30 DEBUG } 19:45:30 DEBUG }, 19:45:30 DEBUG "0.04866666666666409" 19:45:30 DEBUG "resultType": "vector", 19:45:30 DEBUG }, 19:45:30 DEBUG 1713552330, 19:45:30 DEBUG "432.46287041599214" 19:45:30 DEBUG "0" 19:45:30 DEBUG "0" 19:45:30 DEBUG ] 19:45:30 DEBUG ] 19:45:30 DEBUG "value": [ 19:45:30 DEBUG 19:45:30 DEBUG }, 19:45:30 DEBUG ] 19:45:30 DEBUG "value": [ 19:45:30 DEBUG ] 19:45:30 DEBUG "result": [ 19:45:30 DEBUG "value": [ 19:45:30 DEBUG "0" 19:45:30 DEBUG ] 19:45:30 DEBUG ] 19:45:30 DEBUG ] 19:45:30 DEBUG } 19:45:30 DEBUG } 19:45:30 DEBUG 1713552330, 19:45:30 DEBUG "pod": "ds-cts-2" 19:45:30 DEBUG }, 19:45:30 DEBUG } 19:45:30 DEBUG 1713552330, 19:45:30 DEBUG } 19:45:30 DEBUG { 19:45:30 DEBUG 1713552330, 19:45:30 DEBUG ] 19:45:30 DEBUG } 19:45:30 DEBUG } 19:45:30 DEBUG } 19:45:30 DEBUG ] 19:45:30 DEBUG } 19:45:30 DEBUG "0.8722402764287601" 19:45:30 DEBUG "value": [ 19:45:30 DEBUG "value": [ 19:45:30 DEBUG } 19:45:30 DEBUG "16416886.633333333" 19:45:30 DEBUG ] 19:45:30 DEBUG "metric": { 19:45:30 DEBUG "2162.3143520799604" 19:45:30 DEBUG ] 19:45:30 DEBUG ] 19:45:30 DEBUG } 19:45:30 DEBUG ] 19:45:30 DEBUG } 19:45:30 DEBUG ] 19:45:30 DEBUG 1713552330, 19:45:30 DEBUG 1713552330, 19:45:30 DEBUG ] 19:45:30 DEBUG } 19:45:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:45:30 DEBUG } 19:45:30 DEBUG } 19:45:30 DEBUG ] 19:45:30 DEBUG } 19:45:30 DEBUG } 19:45:30 DEBUG } 19:45:30 DEBUG } 19:45:30 DEBUG "0" 19:45:30 DEBUG ] 19:45:30 DEBUG } 19:45:30 DEBUG } 19:45:30 DEBUG }, 19:45:30 DEBUG ] 19:45:30 DEBUG ] 19:45:30 DEBUG } 19:45:30 DEBUG } 19:45:30 DEBUG } 19:45:30 DEBUG ] 19:45:30 DEBUG "459.96666666666664" 19:45:30 DEBUG } 19:45:30 DEBUG ] 19:45:30 DEBUG "value": [ 19:45:30 DEBUG } 19:45:30 DEBUG } 19:45:30 DEBUG } 19:45:30 DEBUG ] 19:45:30 DEBUG ] 19:45:30 DEBUG } 19:45:30 DEBUG 19:45:30 DEBUG } 19:45:30 DEBUG } 19:45:30 DEBUG } 19:45:30 DEBUG } 19:45:30 DEBUG } 19:45:30 DEBUG } 19:45:30 DEBUG 19:45:30 DEBUG ] 19:45:30 DEBUG } 19:45:30 DEBUG 19:45:30 DEBUG } 19:45:30 DEBUG [...] 19:45:30 DEBUG } 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:45:30 DEBUG }, 19:45:30 DEBUG "value": [ 19:45:30 DEBUG 1713552330, 19:45:30 DEBUG "2331.4" 19:45:30 DEBUG ] 19:45:30 DEBUG } 19:45:30 DEBUG ] 19:45:30 DEBUG } 19:45:30 DEBUG } 19:45:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552330" 19:45:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552330" 19:45:30 INFO [http_cmd]: http status code OK 19:45:30 DEBUG --- status code --- 19:45:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:30 DEBUG --- http response --- 19:45:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:45:30 DEBUG ----- output ----- 19:45:30 DEBUG { 19:45:30 DEBUG "status": "success", 19:45:30 DEBUG "data": { 19:45:30 DEBUG "resultType": "vector", 19:45:30 DEBUG "result": [ 19:45:30 DEBUG { 19:45:30 DEBUG "metric": { 19:45:30 DEBUG "pod": "ds-cts-0" 19:45:30 DEBUG }, 19:45:30 DEBUG "value": [ 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG [...] 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG "pod": "ds-cts-2" 19:45:30 DEBUG }, 19:45:30 DEBUG "value": [ 19:45:30 DEBUG 1713552330, 19:45:30 DEBUG "0.000020437777777777778" 19:45:30 DEBUG ] 19:45:30 DEBUG } 19:45:30 DEBUG ] 19:45:30 DEBUG } 19:45:30 DEBUG } 19:45:30 INFO [http_cmd]: http status code OK 19:45:30 DEBUG --- status code --- 19:45:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:30 DEBUG --- http response --- 19:45:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:45:30 DEBUG ----- output ----- 19:45:30 DEBUG { 19:45:30 DEBUG "status": "success", 19:45:30 DEBUG "data": { 19:45:30 DEBUG "resultType": "vector", 19:45:30 DEBUG "result": [ 19:45:30 DEBUG { 19:45:30 DEBUG "metric": { 19:45:30 DEBUG "pod": "ds-cts-0" 19:45:30 DEBUG }, 19:45:30 DEBUG "value": [ 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG [...] 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG "pod": "ds-cts-2" 19:45:30 DEBUG }, 19:45:30 DEBUG "value": [ 19:45:30 DEBUG 1713552330, 19:45:30 DEBUG "29.9" 19:45:30 DEBUG ] 19:45:30 DEBUG } 19:45:30 DEBUG ] 19:45:30 DEBUG } 19:45:30 DEBUG } 19:45:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552330" 19:45:30 INFO [http_cmd]: http status code OK 19:45:30 DEBUG --- status code --- 19:45:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:30 DEBUG --- http response --- 19:45:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:45:30 DEBUG ----- output ----- 19:45:30 DEBUG { 19:45:30 DEBUG "status": "success", 19:45:30 DEBUG "data": { 19:45:30 DEBUG "resultType": "vector", 19:45:30 DEBUG "result": [ 19:45:30 DEBUG { 19:45:30 DEBUG "metric": { 19:45:30 DEBUG "pod": "ds-idrepo-0" 19:45:30 DEBUG }, 19:45:30 DEBUG "value": [ 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG [...] 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG "pod": "ds-idrepo-2" 19:45:30 DEBUG }, 19:45:30 DEBUG "value": [ 19:45:30 DEBUG 1713552330, 19:45:30 DEBUG "0" 19:45:30 DEBUG ] 19:45:30 DEBUG } 19:45:30 DEBUG ] 19:45:30 DEBUG } 19:45:30 DEBUG } 19:45:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713552330" 19:45:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552330" 19:45:30 INFO [http_cmd]: http status code OK 19:45:30 DEBUG --- status code --- 19:45:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:30 DEBUG --- http response --- 19:45:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:45:30 DEBUG ----- output ----- 19:45:30 DEBUG { 19:45:30 DEBUG "status": "success", 19:45:30 DEBUG "data": { 19:45:30 DEBUG "resultType": "vector", 19:45:30 DEBUG "result": [ 19:45:30 DEBUG { 19:45:30 DEBUG "metric": { 19:45:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:45:30 DEBUG }, 19:45:30 DEBUG "value": [ 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG [...] 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:45:30 DEBUG }, 19:45:30 DEBUG "value": [ 19:45:30 DEBUG 1713552330, 19:45:30 DEBUG "0.012" 19:45:30 DEBUG ] 19:45:30 DEBUG } 19:45:30 DEBUG ] 19:45:30 DEBUG } 19:45:30 DEBUG } 19:45:30 INFO [http_cmd]: http status code OK 19:45:30 DEBUG --- status code --- 19:45:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:30 DEBUG --- http response --- 19:45:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:45:30 DEBUG ----- output ----- 19:45:30 DEBUG { 19:45:30 DEBUG "status": "success", 19:45:30 DEBUG "data": { 19:45:30 DEBUG "resultType": "vector", 19:45:30 DEBUG "result": [ 19:45:30 DEBUG { 19:45:30 DEBUG "metric": { 19:45:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:45:30 DEBUG }, 19:45:30 DEBUG "value": [ 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG [...] 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:45:30 DEBUG }, 19:45:30 DEBUG "value": [ 19:45:30 DEBUG 1713552330, 19:45:30 DEBUG "460.6666666666667" 19:45:30 DEBUG ] 19:45:30 DEBUG } 19:45:30 DEBUG ] 19:45:30 DEBUG } 19:45:30 DEBUG } 19:45:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552330" 19:45:30 INFO [http_cmd]: http status code OK 19:45:30 DEBUG --- status code --- 19:45:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:30 DEBUG --- http response --- 19:45:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:45:30 DEBUG ----- output ----- 19:45:30 DEBUG { 19:45:30 DEBUG "status": "success", 19:45:30 DEBUG "data": { 19:45:30 DEBUG "resultType": "vector", 19:45:30 DEBUG "result": [ 19:45:30 DEBUG { 19:45:30 DEBUG "metric": { 19:45:30 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:45:30 DEBUG }, 19:45:30 DEBUG "value": [ 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG [...] 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG "pod": "idm-b85f9d978-txpwv" 19:45:30 DEBUG }, 19:45:30 DEBUG "value": [ 19:45:30 DEBUG 1713552330, 19:45:30 DEBUG "0" 19:45:30 DEBUG ] 19:45:30 DEBUG } 19:45:30 DEBUG ] 19:45:30 DEBUG } 19:45:30 DEBUG } 19:45:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552330" 19:45:30 INFO [http_cmd]: http status code OK 19:45:30 DEBUG --- status code --- 19:45:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:30 DEBUG --- http response --- 19:45:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:45:30 DEBUG ----- output ----- 19:45:30 DEBUG { 19:45:30 DEBUG "status": "success", 19:45:30 DEBUG "data": { 19:45:30 DEBUG "resultType": "vector", 19:45:30 DEBUG "result": [ 19:45:30 DEBUG { 19:45:30 DEBUG "metric": { 19:45:30 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:45:30 DEBUG }, 19:45:30 DEBUG "value": [ 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG [...] 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG "pod": "idm-b85f9d978-txpwv" 19:45:30 DEBUG }, 19:45:30 DEBUG "value": [ 19:45:30 DEBUG 1713552330, 19:45:30 DEBUG "0.0033731611177607486" 19:45:30 DEBUG ] 19:45:30 DEBUG } 19:45:30 DEBUG ] 19:45:30 DEBUG } 19:45:30 DEBUG } 19:45:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552330" 19:45:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552330" 19:45:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713552330" 19:45:30 INFO [http_cmd]: http status code OK 19:45:30 DEBUG --- status code --- 19:45:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:30 DEBUG --- http response --- 19:45:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:45:30 DEBUG ----- output ----- 19:45:30 DEBUG { 19:45:30 DEBUG "status": "success", 19:45:30 DEBUG "data": { 19:45:30 DEBUG "resultType": "vector", 19:45:30 DEBUG "result": [ 19:45:30 DEBUG { 19:45:30 DEBUG "metric": { 19:45:30 DEBUG "instance": "10.142.0.107:9100" 19:45:30 DEBUG }, 19:45:30 DEBUG "value": [ 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG [...] 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG "instance": "10.142.0.43:9100" 19:45:30 DEBUG }, 19:45:30 DEBUG "value": [ 19:45:30 DEBUG 1713552330, 19:45:30 DEBUG "16417378.2" 19:45:30 DEBUG ] 19:45:30 DEBUG } 19:45:30 DEBUG ] 19:45:30 DEBUG } 19:45:30 DEBUG } 19:45:30 INFO [http_cmd]: http status code OK 19:45:30 DEBUG --- status code --- 19:45:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:30 DEBUG --- http response --- 19:45:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:45:30 DEBUG ----- output ----- 19:45:30 DEBUG { 19:45:30 DEBUG "status": "success", 19:45:30 DEBUG "data": { 19:45:30 DEBUG "resultType": "vector", 19:45:30 DEBUG "result": [ 19:45:30 DEBUG { 19:45:30 DEBUG "metric": { 19:45:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:45:30 DEBUG }, 19:45:30 DEBUG "value": [ 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG [...] 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:45:30 DEBUG }, 19:45:30 DEBUG "value": [ 19:45:30 DEBUG 1713552330, 19:45:30 DEBUG "460.96666666666664" 19:45:30 DEBUG ] 19:45:30 DEBUG } 19:45:30 DEBUG ] 19:45:30 DEBUG } 19:45:30 DEBUG } 19:45:30 INFO [http_cmd]: http status code OK 19:45:30 DEBUG --- status code --- 19:45:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:30 DEBUG --- http response --- 19:45:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:45:30 DEBUG ----- output ----- 19:45:30 DEBUG { 19:45:30 DEBUG "status": "success", 19:45:30 DEBUG "data": { 19:45:30 DEBUG "resultType": "vector", 19:45:30 DEBUG "result": [ 19:45:30 DEBUG { 19:45:30 DEBUG "metric": { 19:45:30 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:45:30 DEBUG }, 19:45:30 DEBUG "value": [ 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG [...] 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG 19:45:30 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:45:30 DEBUG }, 19:45:30 DEBUG "value": [ 19:45:30 DEBUG 1713552330, 19:45:30 DEBUG "3280084992" 19:45:30 DEBUG ] 19:45:30 DEBUG } 19:45:30 DEBUG ] 19:45:30 DEBUG } 19:45:30 DEBUG } 19:45:31 INFO 19:45:31 INFO [loop_until]: kubectl --namespace=xlou top pods 19:45:31 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:45:31 INFO [loop_until]: OK (rc = 0) 19:45:31 DEBUG --- stdout --- 19:45:31 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 4695m 7849Mi am-767f6b67b5-qcqz2 3005m 9000Mi am-767f6b67b5-vbltz 3059m 8984Mi ds-cts-0 9164m 10941Mi ds-cts-1 7878m 11148Mi ds-cts-2 8365m 10899Mi ds-idrepo-0 1396m 3663Mi ds-idrepo-1 1940m 3533Mi ds-idrepo-2 1052m 3493Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 3227m 3100Mi 19:45:31 DEBUG --- stderr --- 19:45:31 DEBUG 19:45:32 INFO 19:45:32 INFO [loop_until]: kubectl --namespace=xlou top node 19:45:32 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:45:32 INFO [loop_until]: OK (rc = 0) 19:45:32 DEBUG --- stdout --- 19:45:32 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 72m 0% 6251Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 1466m 4% 7258Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 5803m 19% 10560Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 80m 0% 3345Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 8067m 27% 13907Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 158m 0% 5212Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 5310m 17% 11613Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 2848m 9% 2972Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 6087m 20% 11194Mi 10% gke-xlou-cdm-ds-0f601751-j488 2666m 8% 6745Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 7631m 25% 15922Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 2658m 8% 7427Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 7904m 26% 13394Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 3442m 21% 6868Mi 11% 19:45:32 DEBUG --- stderr --- 19:45:32 DEBUG 19:46:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552390" 19:46:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713552390" 19:46:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713552390" 19:46:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552390" 19:46:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552390" 19:46:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552390" 19:46:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552390" 19:46:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552390" 19:46:30 INFO [http_cmd]: http status code OK 19:46:30 DEBUG --- status code --- 19:46:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:30 DEBUG --- http response --- 19:46:30 INFO [http_cmd]: http status code OK 19:46:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:46:30 DEBUG --- status code --- 19:46:30 DEBUG ----- output ----- 19:46:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:30 DEBUG { 19:46:30 DEBUG --- http response --- 19:46:30 DEBUG "status": "success", 19:46:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:46:30 DEBUG "data": { 19:46:30 ERROR [http_cmd]: ERROR 19:46:30 DEBUG ----- output ----- 19:46:30 DEBUG "resultType": "vector", 19:46:30 ERROR --- status code --- 19:46:30 DEBUG { 19:46:30 DEBUG "result": [ 19:46:30 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:46:30 DEBUG "status": "success", 19:46:30 DEBUG { 19:46:30 ERROR --- http response --- 19:46:30 DEBUG "data": { 19:46:30 DEBUG "metric": { 19:46:30 DEBUG "resultType": "vector", 19:46:30 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:46:30 DEBUG "pod": "ds-idrepo-0" 19:46:30 DEBUG "result": [ 19:46:30 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:46:30 DEBUG }, 19:46:30 DEBUG { 19:46:30 DEBUG "value": [ 19:46:30 DEBUG "metric": { 19:46:30 DEBUG 19:46:30 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:46:30 DEBUG 19:46:30 DEBUG }, 19:46:30 DEBUG 19:46:30 DEBUG "value": [ 19:46:30 DEBUG [...] 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG [...] 19:46:30 DEBUG "pod": "ds-idrepo-2" 19:46:30 DEBUG 19:46:30 DEBUG }, 19:46:30 DEBUG 19:46:30 DEBUG "value": [ 19:46:30 DEBUG 19:46:30 DEBUG 1713552390, 19:46:30 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:46:30 DEBUG "0" 19:46:30 DEBUG }, 19:46:30 DEBUG ] 19:46:30 DEBUG "value": [ 19:46:30 DEBUG } 19:46:30 DEBUG 1713552390, 19:46:30 DEBUG ] 19:46:30 DEBUG "7.200794917573868" 19:46:30 DEBUG } 19:46:30 DEBUG ] 19:46:30 DEBUG } 19:46:30 DEBUG } 19:46:30 DEBUG ] 19:46:30 DEBUG } 19:46:30 DEBUG } 19:46:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552390" 19:46:30 INFO [http_cmd]: http status code OK 19:46:30 DEBUG --- status code --- 19:46:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:30 DEBUG --- http response --- 19:46:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552390" 19:46:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:46:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552390" 19:46:30 DEBUG ----- output ----- 19:46:30 INFO [http_cmd]: http status code OK 19:46:30 DEBUG { 19:46:30 INFO [http_cmd]: http status code OK 19:46:30 DEBUG --- status code --- 19:46:30 DEBUG "status": "success", 19:46:30 DEBUG --- status code --- 19:46:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:30 DEBUG "data": { 19:46:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:30 DEBUG --- http response --- 19:46:30 DEBUG "resultType": "vector", 19:46:30 DEBUG --- http response --- 19:46:30 DEBUG "result": [ 19:46:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:46:30 INFO [http_cmd]: http status code OK 19:46:30 INFO [http_cmd]: http status code OK 19:46:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552390" 19:46:30 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:46:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552390" 19:46:30 DEBUG { 19:46:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552390" 19:46:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552390" 19:46:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713552390" 19:46:30 DEBUG ----- output ----- 19:46:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552390" 19:46:30 DEBUG --- status code --- 19:46:30 DEBUG --- status code --- 19:46:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552390" 19:46:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552390" 19:46:30 DEBUG "metric": { 19:46:30 DEBUG { 19:46:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:30 INFO [http_cmd]: http status code OK 19:46:30 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:46:30 INFO [http_cmd]: http status code OK 19:46:30 INFO [http_cmd]: http status code OK 19:46:30 DEBUG "status": "success", 19:46:30 DEBUG --- http response --- 19:46:30 DEBUG --- http response --- 19:46:30 DEBUG --- status code --- 19:46:30 DEBUG }, 19:46:30 DEBUG --- status code --- 19:46:30 DEBUG --- status code --- 19:46:30 DEBUG "data": { 19:46:30 INFO [http_cmd]: http status code OK 19:46:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:46:30 INFO [http_cmd]: http status code OK 19:46:30 INFO [http_cmd]: http status code OK 19:46:30 INFO [http_cmd]: http status code OK 19:46:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:46:30 DEBUG "value": [ 19:46:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:30 DEBUG "resultType": "vector", 19:46:30 DEBUG --- status code --- 19:46:30 DEBUG ----- output ----- 19:46:30 DEBUG --- status code --- 19:46:30 DEBUG --- status code --- 19:46:30 DEBUG --- status code --- 19:46:30 INFO [http_cmd]: http status code OK 19:46:30 INFO [http_cmd]: http status code OK 19:46:30 INFO [http_cmd]: http status code OK 19:46:30 DEBUG --- http response --- 19:46:30 DEBUG ----- output ----- 19:46:30 DEBUG 19:46:30 DEBUG --- http response --- 19:46:30 DEBUG --- http response --- 19:46:30 DEBUG "result": [ 19:46:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:30 DEBUG { 19:46:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:30 DEBUG --- status code --- 19:46:30 DEBUG --- status code --- 19:46:30 DEBUG --- status code --- 19:46:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:46:30 DEBUG { 19:46:30 DEBUG 19:46:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:46:30 DEBUG { 19:46:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:46:30 DEBUG --- http response --- 19:46:30 DEBUG "status": "success", 19:46:30 DEBUG --- http response --- 19:46:30 DEBUG --- http response --- 19:46:30 DEBUG --- http response --- 19:46:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:30 DEBUG ----- output ----- 19:46:30 DEBUG "status": "success", 19:46:30 DEBUG 19:46:30 DEBUG ----- output ----- 19:46:30 DEBUG "metric": { 19:46:30 DEBUG ----- output ----- 19:46:30 DEBUG "data": { 19:46:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:46:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:46:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:46:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:46:30 INFO [http_cmd]: http status code OK 19:46:30 DEBUG --- http response --- 19:46:30 DEBUG --- http response --- 19:46:30 DEBUG --- http response --- 19:46:30 DEBUG { 19:46:30 DEBUG "data": { 19:46:30 DEBUG [...] 19:46:30 DEBUG { 19:46:30 DEBUG "pod": "ds-idrepo-0" 19:46:30 DEBUG { 19:46:30 DEBUG "resultType": "vector", 19:46:30 DEBUG ----- output ----- 19:46:30 DEBUG ----- output ----- 19:46:30 DEBUG ----- output ----- 19:46:30 DEBUG ----- output ----- 19:46:30 DEBUG --- status code --- 19:46:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:46:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:46:30 DEBUG "status": "success", 19:46:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:46:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552390" 19:46:30 DEBUG "resultType": "vector", 19:46:30 DEBUG 19:46:30 DEBUG "status": "success", 19:46:30 DEBUG }, 19:46:30 DEBUG "status": "success", 19:46:30 DEBUG "result": [ 19:46:30 DEBUG { 19:46:30 DEBUG { 19:46:30 DEBUG { 19:46:30 DEBUG { 19:46:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:30 DEBUG ----- output ----- 19:46:30 DEBUG ----- output ----- 19:46:30 DEBUG "data": { 19:46:30 DEBUG ----- output ----- 19:46:30 DEBUG "result": [ 19:46:30 DEBUG 19:46:30 DEBUG "data": { 19:46:30 DEBUG "value": [ 19:46:30 DEBUG "data": { 19:46:30 DEBUG { 19:46:30 DEBUG "status": "success", 19:46:30 DEBUG "status": "success", 19:46:30 DEBUG "status": "success", 19:46:30 DEBUG "status": "success", 19:46:30 DEBUG --- http response --- 19:46:30 DEBUG { 19:46:30 DEBUG { 19:46:30 DEBUG "resultType": "vector", 19:46:30 DEBUG "result": [ 19:46:30 DEBUG { 19:46:30 DEBUG 19:46:30 DEBUG "resultType": "vector", 19:46:30 DEBUG 19:46:30 DEBUG "resultType": "vector", 19:46:30 DEBUG "metric": { 19:46:30 DEBUG "data": { 19:46:30 DEBUG "data": { 19:46:30 DEBUG "data": { 19:46:30 DEBUG "data": { 19:46:30 DEBUG "status": "success", 19:46:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:46:30 DEBUG "status": "success", 19:46:30 DEBUG { 19:46:30 DEBUG { 19:46:30 DEBUG "status": "success", 19:46:30 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:46:30 DEBUG "result": [ 19:46:30 DEBUG 19:46:30 DEBUG "result": [ 19:46:30 DEBUG "pod": "ds-cts-0" 19:46:30 DEBUG "resultType": "vector", 19:46:30 DEBUG "resultType": "vector", 19:46:30 DEBUG "resultType": "vector", 19:46:30 DEBUG "resultType": "vector", 19:46:30 DEBUG "data": { 19:46:30 DEBUG ----- output ----- 19:46:30 DEBUG "data": { 19:46:30 DEBUG "metric": { 19:46:30 DEBUG "metric": { 19:46:30 DEBUG "data": { 19:46:30 DEBUG }, 19:46:30 DEBUG { 19:46:30 DEBUG 19:46:30 DEBUG { 19:46:30 DEBUG }, 19:46:30 DEBUG "result": [ 19:46:30 DEBUG "result": [ 19:46:30 DEBUG "result": [ 19:46:30 DEBUG "result": [ 19:46:30 DEBUG "resultType": "vector", 19:46:30 DEBUG { 19:46:30 DEBUG "resultType": "vector", 19:46:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:46:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:46:30 DEBUG "resultType": "vector", 19:46:30 DEBUG "value": [ 19:46:30 DEBUG "metric": { 19:46:30 DEBUG [...] 19:46:30 DEBUG "metric": { 19:46:30 DEBUG "value": [ 19:46:30 DEBUG { 19:46:30 DEBUG { 19:46:30 DEBUG { 19:46:30 DEBUG { 19:46:30 DEBUG "result": [ 19:46:30 DEBUG "status": "success", 19:46:30 DEBUG "result": [ 19:46:30 DEBUG }, 19:46:30 DEBUG }, 19:46:30 DEBUG "result": [ 19:46:30 DEBUG 1713552390, 19:46:30 DEBUG "instance": "10.142.0.107:9100" 19:46:30 DEBUG 19:46:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:46:30 DEBUG 19:46:30 DEBUG "metric": { 19:46:30 DEBUG "metric": { 19:46:30 DEBUG "metric": { 19:46:30 DEBUG "metric": { 19:46:30 DEBUG { 19:46:30 DEBUG "data": { 19:46:30 DEBUG { 19:46:30 DEBUG "value": [ 19:46:30 DEBUG "value": [ 19:46:30 DEBUG { 19:46:30 DEBUG "0" 19:46:30 DEBUG }, 19:46:30 DEBUG 19:46:30 DEBUG }, 19:46:30 DEBUG 19:46:30 DEBUG "pod": "ds-idrepo-0" 19:46:30 DEBUG "instance": "10.142.0.107:9100" 19:46:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:46:30 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:46:30 DEBUG "metric": { 19:46:30 DEBUG "resultType": "vector", 19:46:30 DEBUG "metric": { 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG "metric": { 19:46:30 DEBUG ] 19:46:30 DEBUG "value": [ 19:46:30 DEBUG 19:46:30 DEBUG "value": [ 19:46:30 DEBUG 19:46:30 DEBUG [...] 19:46:30 DEBUG }, 19:46:30 DEBUG }, 19:46:30 DEBUG }, 19:46:30 DEBUG "pod": "ds-cts-0" 19:46:30 DEBUG "result": [ 19:46:30 DEBUG "instance": "10.142.0.107:9100" 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG "pod": "ds-idrepo-0" 19:46:30 DEBUG } 19:46:30 DEBUG 19:46:30 DEBUG "pod": "ds-idrepo-2" 19:46:30 DEBUG 19:46:30 DEBUG }, 19:46:30 DEBUG 19:46:30 DEBUG "value": [ 19:46:30 DEBUG "value": [ 19:46:30 DEBUG 19:46:30 DEBUG }, 19:46:30 DEBUG { 19:46:30 DEBUG }, 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG }, 19:46:30 DEBUG ] 19:46:30 DEBUG 19:46:30 DEBUG }, 19:46:30 INFO [http_cmd]: http status code OK 19:46:30 DEBUG 19:46:30 DEBUG "value": [ 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG "value": [ 19:46:30 DEBUG 19:46:30 DEBUG "value": [ 19:46:30 DEBUG "metric": { 19:46:30 DEBUG "value": [ 19:46:30 DEBUG [...] 19:46:30 DEBUG [...] 19:46:30 DEBUG "value": [ 19:46:30 DEBUG } 19:46:30 DEBUG 19:46:30 DEBUG "value": [ 19:46:30 DEBUG --- status code --- 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG "instance": "10.142.0.107:9100" 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG } 19:46:30 DEBUG [...] 19:46:30 DEBUG 1713552390, 19:46:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:30 DEBUG [...] 19:46:30 DEBUG 19:46:30 DEBUG "pod": "ds-cts-2" 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG [...] 19:46:30 DEBUG 19:46:30 DEBUG }, 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG "0" 19:46:30 DEBUG --- http response --- 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG }, 19:46:30 DEBUG [...] 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG "value": [ 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG ] 19:46:30 DEBUG 19:46:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:46:30 DEBUG [...] 19:46:30 DEBUG "value": [ 19:46:30 DEBUG 19:46:30 DEBUG [...] 19:46:30 DEBUG 19:46:30 DEBUG [...] 19:46:30 DEBUG 19:46:30 DEBUG [...] 19:46:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:46:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:46:30 DEBUG [...] 19:46:30 DEBUG 19:46:30 DEBUG } 19:46:30 DEBUG 19:46:30 DEBUG ----- output ----- 19:46:30 DEBUG 19:46:30 DEBUG 1713552390, 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG }, 19:46:30 DEBUG }, 19:46:30 DEBUG 19:46:30 DEBUG "instance": "10.142.0.43:9100" 19:46:30 DEBUG ] 19:46:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:46:30 DEBUG { 19:46:30 DEBUG 19:46:30 DEBUG "0.000033333333333333335" 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG "value": [ 19:46:30 DEBUG "value": [ 19:46:30 DEBUG 19:46:30 DEBUG }, 19:46:30 DEBUG } 19:46:30 DEBUG }, 19:46:30 DEBUG "status": "success", 19:46:30 DEBUG 19:46:30 DEBUG ] 19:46:30 DEBUG "pod": "ds-cts-2" 19:46:30 DEBUG 19:46:30 DEBUG }, 19:46:30 DEBUG 19:46:30 DEBUG [...] 19:46:30 DEBUG 19:46:30 DEBUG 1713552390, 19:46:30 DEBUG 1713552390, 19:46:30 DEBUG 19:46:30 DEBUG "value": [ 19:46:30 DEBUG } 19:46:30 DEBUG "value": [ 19:46:30 DEBUG "data": { 19:46:30 DEBUG "instance": "10.142.0.43:9100" 19:46:30 DEBUG } 19:46:30 DEBUG }, 19:46:30 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:46:30 DEBUG "value": [ 19:46:30 DEBUG "pod": "ds-cts-2" 19:46:30 DEBUG 19:46:30 DEBUG "instance": "10.142.0.43:9100" 19:46:30 DEBUG "0" 19:46:30 DEBUG "2499.133333333333" 19:46:30 DEBUG "pod": "ds-idrepo-2" 19:46:30 DEBUG 1713552390, 19:46:30 DEBUG 1713552390, 19:46:30 DEBUG "resultType": "vector", 19:46:30 DEBUG }, 19:46:30 DEBUG ] 19:46:30 DEBUG "value": [ 19:46:30 DEBUG }, 19:46:30 DEBUG 1713552390, 19:46:30 DEBUG }, 19:46:30 DEBUG 19:46:30 DEBUG }, 19:46:30 DEBUG ] 19:46:30 DEBUG ] 19:46:30 DEBUG }, 19:46:30 DEBUG "0.15989999999989474" 19:46:30 DEBUG "292470" 19:46:30 DEBUG "result": [ 19:46:30 DEBUG "value": [ 19:46:30 DEBUG } 19:46:30 DEBUG 1713552390, 19:46:30 DEBUG "value": [ 19:46:30 DEBUG "2336.866666666667" 19:46:30 DEBUG "value": [ 19:46:30 DEBUG 19:46:30 DEBUG "value": [ 19:46:30 DEBUG } 19:46:30 DEBUG } 19:46:30 DEBUG "value": [ 19:46:30 DEBUG ] 19:46:30 DEBUG ] 19:46:30 DEBUG { 19:46:30 DEBUG 1713552390, 19:46:30 DEBUG } 19:46:30 DEBUG "0" 19:46:30 DEBUG 1713552390, 19:46:30 DEBUG ] 19:46:30 DEBUG 1713552390, 19:46:30 DEBUG "instance": "10.142.0.43:9100" 19:46:30 DEBUG 1713552390, 19:46:30 DEBUG ] 19:46:30 DEBUG ] 19:46:30 DEBUG 1713552390, 19:46:30 DEBUG } 19:46:30 DEBUG } 19:46:30 DEBUG "metric": { 19:46:30 DEBUG "1949587002368" 19:46:30 DEBUG ] 19:46:30 DEBUG ] 19:46:30 DEBUG } 19:46:30 DEBUG "0" 19:46:30 DEBUG }, 19:46:30 DEBUG "0.06599999999999871" 19:46:30 DEBUG } 19:46:30 DEBUG } 19:46:30 DEBUG "0" 19:46:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713552390" 19:46:30 DEBUG ] 19:46:30 DEBUG ] 19:46:30 DEBUG "pod": "ds-cts-0" 19:46:30 DEBUG "2131980.4766824613" 19:46:30 DEBUG } 19:46:30 DEBUG } 19:46:30 DEBUG ] 19:46:30 DEBUG ] 19:46:30 DEBUG "value": [ 19:46:30 DEBUG 1713552390, 19:46:30 DEBUG } 19:46:30 DEBUG } 19:46:30 DEBUG ] 19:46:30 DEBUG } 19:46:30 DEBUG } 19:46:30 DEBUG }, 19:46:30 DEBUG ] 19:46:30 DEBUG ] 19:46:30 DEBUG ] 19:46:30 DEBUG } 19:46:30 DEBUG } 19:46:30 DEBUG ] 19:46:30 DEBUG ] 19:46:30 DEBUG } 19:46:30 DEBUG } 19:46:30 DEBUG } 19:46:30 DEBUG "value": [ 19:46:30 DEBUG } 19:46:30 DEBUG } 19:46:30 DEBUG } 19:46:30 DEBUG } 19:46:30 DEBUG "74607343.83333334" 19:46:30 DEBUG } 19:46:30 DEBUG } 19:46:30 DEBUG ] 19:46:30 DEBUG 19:46:30 DEBUG ] 19:46:30 DEBUG } 19:46:30 DEBUG } 19:46:30 DEBUG ] 19:46:30 DEBUG } 19:46:30 DEBUG ] 19:46:30 DEBUG } 19:46:30 DEBUG 19:46:30 DEBUG } 19:46:30 DEBUG } 19:46:30 DEBUG ] 19:46:30 DEBUG } 19:46:30 DEBUG 19:46:30 DEBUG } 19:46:30 DEBUG } 19:46:30 DEBUG } 19:46:30 DEBUG [...] 19:46:30 DEBUG } 19:46:30 DEBUG } 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG "pod": "ds-cts-2" 19:46:30 DEBUG }, 19:46:30 DEBUG "value": [ 19:46:30 DEBUG 1713552390, 19:46:30 DEBUG "30" 19:46:30 DEBUG ] 19:46:30 DEBUG } 19:46:30 DEBUG ] 19:46:30 DEBUG } 19:46:30 DEBUG } 19:46:30 INFO [http_cmd]: http status code OK 19:46:30 DEBUG --- status code --- 19:46:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:30 DEBUG --- http response --- 19:46:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:46:30 DEBUG ----- output ----- 19:46:30 DEBUG { 19:46:30 DEBUG "status": "success", 19:46:30 DEBUG "data": { 19:46:30 DEBUG "resultType": "vector", 19:46:30 DEBUG "result": [ 19:46:30 DEBUG { 19:46:30 DEBUG "metric": { 19:46:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:46:30 DEBUG }, 19:46:30 DEBUG "value": [ 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG [...] 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:46:30 DEBUG }, 19:46:30 DEBUG "value": [ 19:46:30 DEBUG 1713552390, 19:46:30 DEBUG "0.012" 19:46:30 DEBUG ] 19:46:30 DEBUG } 19:46:30 DEBUG ] 19:46:30 DEBUG } 19:46:30 DEBUG } 19:46:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552390" 19:46:30 INFO [http_cmd]: http status code OK 19:46:30 DEBUG --- status code --- 19:46:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:30 DEBUG --- http response --- 19:46:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:46:30 DEBUG ----- output ----- 19:46:30 DEBUG { 19:46:30 DEBUG "status": "success", 19:46:30 DEBUG "data": { 19:46:30 DEBUG "resultType": "vector", 19:46:30 DEBUG "result": [ 19:46:30 DEBUG { 19:46:30 DEBUG "metric": { 19:46:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:46:30 DEBUG }, 19:46:30 DEBUG "value": [ 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG [...] 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:46:30 DEBUG }, 19:46:30 DEBUG "value": [ 19:46:30 DEBUG 1713552390, 19:46:30 DEBUG "2499.9333333333334" 19:46:30 DEBUG ] 19:46:30 DEBUG } 19:46:30 DEBUG ] 19:46:30 DEBUG } 19:46:30 DEBUG } 19:46:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552390" 19:46:30 INFO [http_cmd]: http status code OK 19:46:30 DEBUG --- status code --- 19:46:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:30 DEBUG --- http response --- 19:46:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:46:30 DEBUG ----- output ----- 19:46:30 DEBUG { 19:46:30 DEBUG "status": "success", 19:46:30 DEBUG "data": { 19:46:30 DEBUG "resultType": "vector", 19:46:30 DEBUG "result": [ 19:46:30 DEBUG { 19:46:30 DEBUG "metric": { 19:46:30 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:46:30 DEBUG }, 19:46:30 DEBUG "value": [ 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG [...] 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:46:30 DEBUG }, 19:46:30 DEBUG "value": [ 19:46:30 DEBUG 1713552390, 19:46:30 DEBUG "0" 19:46:30 DEBUG ] 19:46:30 DEBUG } 19:46:30 DEBUG ] 19:46:30 DEBUG } 19:46:30 DEBUG } 19:46:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552390" 19:46:30 INFO [http_cmd]: http status code OK 19:46:30 DEBUG --- status code --- 19:46:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:30 DEBUG --- http response --- 19:46:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:46:30 DEBUG ----- output ----- 19:46:30 DEBUG { 19:46:30 DEBUG "status": "success", 19:46:30 DEBUG "data": { 19:46:30 DEBUG "resultType": "vector", 19:46:30 DEBUG "result": [ 19:46:30 DEBUG { 19:46:30 DEBUG "metric": { 19:46:30 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:46:30 DEBUG }, 19:46:30 DEBUG "value": [ 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG [...] 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:46:30 DEBUG }, 19:46:30 DEBUG "value": [ 19:46:30 DEBUG 1713552390, 19:46:30 DEBUG "8.003535760233873" 19:46:30 DEBUG ] 19:46:30 DEBUG } 19:46:30 DEBUG ] 19:46:30 DEBUG } 19:46:30 DEBUG } 19:46:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552390" 19:46:30 INFO [http_cmd]: http status code OK 19:46:30 DEBUG --- status code --- 19:46:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:30 DEBUG --- http response --- 19:46:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:46:30 DEBUG ----- output ----- 19:46:30 DEBUG { 19:46:30 DEBUG "status": "success", 19:46:30 DEBUG "data": { 19:46:30 DEBUG "resultType": "vector", 19:46:30 DEBUG "result": [ 19:46:30 DEBUG { 19:46:30 DEBUG "metric": { 19:46:30 DEBUG "instance": "10.142.0.107:9100" 19:46:30 DEBUG }, 19:46:30 DEBUG "value": [ 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG [...] 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG "instance": "10.142.0.43:9100" 19:46:30 DEBUG }, 19:46:30 DEBUG "value": [ 19:46:30 DEBUG 1713552390, 19:46:30 DEBUG "74604885.03333333" 19:46:30 DEBUG ] 19:46:30 DEBUG } 19:46:30 DEBUG ] 19:46:30 DEBUG } 19:46:30 DEBUG } 19:46:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552390" 19:46:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713552390" 19:46:30 INFO [http_cmd]: http status code OK 19:46:30 DEBUG --- status code --- 19:46:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:30 DEBUG --- http response --- 19:46:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:46:30 DEBUG ----- output ----- 19:46:30 DEBUG { 19:46:30 DEBUG "status": "success", 19:46:30 DEBUG "data": { 19:46:30 DEBUG "resultType": "vector", 19:46:30 DEBUG "result": [ 19:46:30 DEBUG { 19:46:30 DEBUG "metric": { 19:46:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:46:30 DEBUG }, 19:46:30 DEBUG "value": [ 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG [...] 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:46:30 DEBUG }, 19:46:30 DEBUG "value": [ 19:46:30 DEBUG 1713552390, 19:46:30 DEBUG "2499.1666666666665" 19:46:30 DEBUG ] 19:46:30 DEBUG } 19:46:30 DEBUG ] 19:46:30 DEBUG } 19:46:30 DEBUG } 19:46:30 INFO [http_cmd]: http status code OK 19:46:30 DEBUG --- status code --- 19:46:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:30 DEBUG --- http response --- 19:46:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:46:30 DEBUG ----- output ----- 19:46:30 DEBUG { 19:46:30 DEBUG "status": "success", 19:46:30 DEBUG "data": { 19:46:30 DEBUG "resultType": "vector", 19:46:30 DEBUG "result": [ 19:46:30 DEBUG { 19:46:30 DEBUG "metric": { 19:46:30 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:46:30 DEBUG }, 19:46:30 DEBUG "value": [ 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG [...] 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG 19:46:30 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:46:30 DEBUG }, 19:46:30 DEBUG "value": [ 19:46:30 DEBUG 1713552390, 19:46:30 DEBUG "3417948160" 19:46:30 DEBUG ] 19:46:30 DEBUG } 19:46:30 DEBUG ] 19:46:30 DEBUG } 19:46:30 DEBUG } 19:46:31 INFO 19:46:31 INFO [loop_until]: kubectl --namespace=xlou top pods 19:46:31 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:46:31 INFO [loop_until]: OK (rc = 0) 19:46:31 DEBUG --- stdout --- 19:46:31 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 15730m 7851Mi am-767f6b67b5-qcqz2 16064m 8738Mi am-767f6b67b5-vbltz 17311m 8989Mi ds-cts-0 15887m 10920Mi ds-cts-1 14344m 11172Mi ds-cts-2 16628m 10982Mi ds-idrepo-0 6954m 3665Mi ds-idrepo-1 7087m 3535Mi ds-idrepo-2 7247m 3493Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8062m 3206Mi 19:46:31 DEBUG --- stderr --- 19:46:31 DEBUG 19:46:32 INFO 19:46:32 INFO [loop_until]: kubectl --namespace=xlou top node 19:46:32 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:46:32 INFO [loop_until]: OK (rc = 0) 19:46:32 DEBUG --- stdout --- 19:46:32 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 68m 0% 6249Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7386m 24% 7265Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 15911m 53% 10558Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 82m 0% 3331Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 15734m 52% 13928Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 157m 0% 5213Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 16245m 54% 11625Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15459m 51% 3001Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 17903m 59% 11198Mi 10% gke-xlou-cdm-ds-0f601751-j488 7583m 25% 6744Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 14258m 47% 15935Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 7393m 24% 7430Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 16084m 53% 13440Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 8198m 51% 6935Mi 11% 19:46:32 DEBUG --- stderr --- 19:46:32 DEBUG 19:47:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552450" 19:47:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713552450" 19:47:30 INFO [http_cmd]: http status code OK 19:47:30 DEBUG --- status code --- 19:47:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:30 DEBUG --- http response --- 19:47:30 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:47:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552450" 19:47:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552450" 19:47:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713552450" 19:47:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552450" 19:47:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552450" 19:47:30 ERROR [http_cmd]: ERROR 19:47:30 ERROR --- status code --- 19:47:30 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:47:30 ERROR --- http response --- 19:47:30 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:47:30 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:47:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552450" 19:47:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713552450" 19:47:30 INFO [http_cmd]: http status code OK 19:47:30 DEBUG --- status code --- 19:47:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:30 DEBUG --- http response --- 19:47:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:47:30 DEBUG ----- output ----- 19:47:30 DEBUG { 19:47:30 DEBUG "status": "success", 19:47:30 DEBUG "data": { 19:47:30 DEBUG "resultType": "vector", 19:47:30 DEBUG "result": [ 19:47:30 DEBUG { 19:47:30 INFO [http_cmd]: http status code OK 19:47:30 INFO [http_cmd]: http status code OK 19:47:30 INFO [http_cmd]: http status code OK 19:47:30 DEBUG "metric": { 19:47:30 INFO [http_cmd]: http status code OK 19:47:30 DEBUG --- status code --- 19:47:30 DEBUG --- status code --- 19:47:30 DEBUG --- status code --- 19:47:30 DEBUG "pod": "ds-idrepo-0" 19:47:30 DEBUG --- status code --- 19:47:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:30 DEBUG }, 19:47:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:30 DEBUG --- http response --- 19:47:30 DEBUG --- http response --- 19:47:30 DEBUG --- http response --- 19:47:30 DEBUG "value": [ 19:47:30 DEBUG --- http response --- 19:47:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:47:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:47:30 INFO [http_cmd]: http status code OK 19:47:30 DEBUG 19:47:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:47:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:47:30 DEBUG ----- output ----- 19:47:30 DEBUG ----- output ----- 19:47:30 DEBUG --- status code --- 19:47:30 DEBUG 19:47:30 DEBUG ----- output ----- 19:47:30 DEBUG ----- output ----- 19:47:30 DEBUG { 19:47:30 DEBUG { 19:47:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:30 DEBUG 19:47:30 DEBUG { 19:47:30 DEBUG { 19:47:30 DEBUG "status": "success", 19:47:30 DEBUG "status": "success", 19:47:30 DEBUG --- http response --- 19:47:30 DEBUG [...] 19:47:30 DEBUG "status": "success", 19:47:30 DEBUG "status": "success", 19:47:30 DEBUG "data": { 19:47:30 DEBUG "data": { 19:47:30 DEBUG 19:47:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:47:30 DEBUG "data": { 19:47:30 DEBUG "data": { 19:47:30 DEBUG "resultType": "vector", 19:47:30 DEBUG "resultType": "vector", 19:47:30 DEBUG 19:47:30 DEBUG ----- output ----- 19:47:30 DEBUG "resultType": "vector", 19:47:30 DEBUG "resultType": "vector", 19:47:30 DEBUG "result": [ 19:47:30 DEBUG "result": [ 19:47:30 DEBUG 19:47:30 DEBUG { 19:47:30 DEBUG "result": [ 19:47:30 DEBUG "result": [ 19:47:30 DEBUG { 19:47:30 DEBUG { 19:47:30 DEBUG "pod": "ds-idrepo-2" 19:47:30 DEBUG "status": "success", 19:47:30 DEBUG { 19:47:30 DEBUG { 19:47:30 DEBUG "metric": { 19:47:30 DEBUG "metric": { 19:47:30 DEBUG }, 19:47:30 DEBUG "data": { 19:47:30 DEBUG "metric": { 19:47:30 DEBUG "metric": { 19:47:30 DEBUG "pod": "ds-cts-0" 19:47:30 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:47:30 DEBUG "value": [ 19:47:30 DEBUG "resultType": "vector", 19:47:30 DEBUG "pod": "ds-idrepo-0" 19:47:30 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:47:30 DEBUG }, 19:47:30 DEBUG }, 19:47:30 DEBUG 1713552450, 19:47:30 DEBUG "result": [ 19:47:30 DEBUG }, 19:47:30 DEBUG }, 19:47:30 DEBUG "value": [ 19:47:30 DEBUG "value": [ 19:47:30 DEBUG "0" 19:47:30 DEBUG { 19:47:30 DEBUG "value": [ 19:47:30 DEBUG "value": [ 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG ] 19:47:30 DEBUG "metric": { 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG } 19:47:30 DEBUG "pod": "ds-cts-0" 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG ] 19:47:30 DEBUG }, 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG [...] 19:47:30 DEBUG [...] 19:47:30 DEBUG } 19:47:30 DEBUG "value": [ 19:47:30 DEBUG [...] 19:47:30 DEBUG [...] 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG } 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG "pod": "ds-cts-2" 19:47:30 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 19:47:30 DEBUG [...] 19:47:30 DEBUG "pod": "ds-idrepo-2" 19:47:30 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:47:30 DEBUG }, 19:47:30 DEBUG }, 19:47:30 DEBUG 19:47:30 DEBUG }, 19:47:30 DEBUG }, 19:47:30 DEBUG "value": [ 19:47:30 DEBUG "value": [ 19:47:30 DEBUG 19:47:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552450" 19:47:30 DEBUG "value": [ 19:47:30 DEBUG "value": [ 19:47:30 DEBUG 1713552450, 19:47:30 DEBUG 1713552450, 19:47:30 DEBUG 19:47:30 DEBUG 1713552450, 19:47:30 DEBUG 1713552450, 19:47:30 DEBUG "0.00003333333333333336" 19:47:30 INFO [http_cmd]: http status code OK 19:47:30 DEBUG "0" 19:47:30 DEBUG "pod": "ds-cts-2" 19:47:30 DEBUG "0" 19:47:30 DEBUG "8.033493176000169" 19:47:30 DEBUG ] 19:47:30 DEBUG --- status code --- 19:47:30 DEBUG ] 19:47:30 DEBUG }, 19:47:30 DEBUG ] 19:47:30 DEBUG ] 19:47:30 DEBUG } 19:47:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:30 DEBUG } 19:47:30 DEBUG "value": [ 19:47:30 DEBUG } 19:47:30 DEBUG } 19:47:30 DEBUG ] 19:47:30 DEBUG --- http response --- 19:47:30 DEBUG ] 19:47:30 DEBUG 1713552450, 19:47:30 DEBUG ] 19:47:30 DEBUG ] 19:47:30 DEBUG } 19:47:30 DEBUG } 19:47:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:47:30 DEBUG "29.966666666666665" 19:47:30 DEBUG } 19:47:30 DEBUG } 19:47:30 DEBUG } 19:47:30 DEBUG } 19:47:30 DEBUG ----- output ----- 19:47:30 DEBUG ] 19:47:30 DEBUG } 19:47:30 DEBUG } 19:47:30 DEBUG { 19:47:30 DEBUG } 19:47:30 DEBUG "status": "success", 19:47:30 DEBUG ] 19:47:30 DEBUG "data": { 19:47:30 DEBUG } 19:47:30 DEBUG "resultType": "vector", 19:47:30 DEBUG } 19:47:30 DEBUG "result": [ 19:47:30 DEBUG { 19:47:30 DEBUG "metric": { 19:47:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:47:30 DEBUG }, 19:47:30 DEBUG "value": [ 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG [...] 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:47:30 DEBUG }, 19:47:30 DEBUG "value": [ 19:47:30 DEBUG 1713552450, 19:47:30 DEBUG "0.013" 19:47:30 DEBUG ] 19:47:30 DEBUG } 19:47:30 DEBUG ] 19:47:30 DEBUG } 19:47:30 DEBUG } 19:47:30 INFO [http_cmd]: http status code OK 19:47:30 DEBUG --- status code --- 19:47:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:30 DEBUG --- http response --- 19:47:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:47:30 DEBUG ----- output ----- 19:47:30 DEBUG { 19:47:30 DEBUG "status": "success", 19:47:30 DEBUG "data": { 19:47:30 DEBUG "resultType": "vector", 19:47:30 DEBUG "result": [ 19:47:30 DEBUG { 19:47:30 DEBUG "metric": { 19:47:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:47:30 DEBUG }, 19:47:30 DEBUG "value": [ 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG [...] 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:47:30 DEBUG }, 19:47:30 DEBUG "value": [ 19:47:30 DEBUG 1713552450, 19:47:30 DEBUG "303949.5" 19:47:30 DEBUG ] 19:47:30 DEBUG } 19:47:30 DEBUG ] 19:47:30 DEBUG } 19:47:30 DEBUG } 19:47:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552450" 19:47:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552450" 19:47:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552450" 19:47:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552450" 19:47:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552450" 19:47:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552450" 19:47:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552450" 19:47:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552450" 19:47:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713552450" 19:47:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552450" 19:47:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552450" 19:47:30 INFO [http_cmd]: http status code OK 19:47:30 INFO [http_cmd]: http status code OK 19:47:30 INFO [http_cmd]: http status code OK 19:47:30 INFO [http_cmd]: http status code OK 19:47:30 DEBUG --- status code --- 19:47:30 DEBUG --- status code --- 19:47:30 DEBUG --- status code --- 19:47:30 DEBUG --- status code --- 19:47:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:30 INFO [http_cmd]: http status code OK 19:47:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:30 DEBUG --- http response --- 19:47:30 INFO [http_cmd]: http status code OK 19:47:30 DEBUG --- http response --- 19:47:30 DEBUG --- http response --- 19:47:30 DEBUG --- status code --- 19:47:30 DEBUG --- http response --- 19:47:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:47:30 DEBUG --- status code --- 19:47:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:47:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:47:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:47:30 DEBUG ----- output ----- 19:47:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:30 DEBUG ----- output ----- 19:47:30 DEBUG ----- output ----- 19:47:30 DEBUG --- http response --- 19:47:30 DEBUG ----- output ----- 19:47:30 DEBUG { 19:47:30 DEBUG --- http response --- 19:47:30 INFO [http_cmd]: http status code OK 19:47:30 INFO [http_cmd]: http status code OK 19:47:30 DEBUG { 19:47:30 INFO [http_cmd]: http status code OK 19:47:30 DEBUG { 19:47:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:47:30 DEBUG { 19:47:30 DEBUG "status": "success", 19:47:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:47:30 DEBUG --- status code --- 19:47:30 DEBUG --- status code --- 19:47:30 DEBUG "status": "success", 19:47:30 DEBUG --- status code --- 19:47:30 DEBUG "status": "success", 19:47:30 DEBUG ----- output ----- 19:47:30 DEBUG "status": "success", 19:47:30 DEBUG "data": { 19:47:30 DEBUG ----- output ----- 19:47:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:30 DEBUG "data": { 19:47:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:30 INFO [http_cmd]: http status code OK 19:47:30 DEBUG "data": { 19:47:30 DEBUG { 19:47:30 DEBUG "data": { 19:47:30 DEBUG "resultType": "vector", 19:47:30 DEBUG { 19:47:30 DEBUG --- http response --- 19:47:30 DEBUG --- http response --- 19:47:30 DEBUG "resultType": "vector", 19:47:30 DEBUG --- http response --- 19:47:30 DEBUG --- status code --- 19:47:30 DEBUG "resultType": "vector", 19:47:30 DEBUG "status": "success", 19:47:30 DEBUG "resultType": "vector", 19:47:30 INFO [http_cmd]: http status code OK 19:47:30 DEBUG "result": [ 19:47:30 DEBUG "status": "success", 19:47:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:47:30 DEBUG "result": [ 19:47:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:47:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:47:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:30 DEBUG "result": [ 19:47:30 DEBUG "data": { 19:47:30 DEBUG "result": [ 19:47:30 DEBUG --- status code --- 19:47:30 DEBUG { 19:47:30 DEBUG "data": { 19:47:30 DEBUG ----- output ----- 19:47:30 DEBUG { 19:47:30 DEBUG ----- output ----- 19:47:30 DEBUG ----- output ----- 19:47:30 DEBUG --- http response --- 19:47:30 DEBUG { 19:47:30 DEBUG "resultType": "vector", 19:47:30 DEBUG { 19:47:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552450" 19:47:30 DEBUG "metric": { 19:47:30 DEBUG "resultType": "vector", 19:47:30 DEBUG { 19:47:30 DEBUG "status": "success", 19:47:30 DEBUG { 19:47:30 DEBUG { 19:47:30 DEBUG "metric": { 19:47:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:47:30 DEBUG "result": [ 19:47:30 DEBUG "metric": { 19:47:30 DEBUG --- http response --- 19:47:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:47:30 DEBUG "result": [ 19:47:30 DEBUG { 19:47:30 DEBUG "metric": { 19:47:30 DEBUG "status": "success", 19:47:30 DEBUG "status": "success", 19:47:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:47:30 DEBUG ----- output ----- 19:47:30 DEBUG { 19:47:30 DEBUG "instance": "10.142.0.107:9100" 19:47:30 DEBUG }, 19:47:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:47:30 DEBUG "data": { 19:47:30 DEBUG "metric": { 19:47:30 DEBUG "instance": "10.142.0.107:9100" 19:47:30 DEBUG "data": { 19:47:30 DEBUG "data": { 19:47:30 DEBUG }, 19:47:30 DEBUG { 19:47:30 DEBUG "metric": { 19:47:30 DEBUG }, 19:47:30 DEBUG "value": [ 19:47:30 DEBUG ----- output ----- 19:47:30 DEBUG "resultType": "vector", 19:47:30 DEBUG "pod": "ds-cts-0" 19:47:30 DEBUG }, 19:47:30 DEBUG "resultType": "vector", 19:47:30 DEBUG "resultType": "vector", 19:47:30 DEBUG "value": [ 19:47:30 DEBUG "status": "success", 19:47:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:47:30 DEBUG "value": [ 19:47:30 DEBUG 19:47:30 DEBUG { 19:47:30 DEBUG "result": [ 19:47:30 DEBUG }, 19:47:30 DEBUG "value": [ 19:47:30 DEBUG "result": [ 19:47:30 DEBUG "result": [ 19:47:30 DEBUG 19:47:30 DEBUG "data": { 19:47:30 DEBUG }, 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG "status": "success", 19:47:30 DEBUG { 19:47:30 DEBUG "value": [ 19:47:30 DEBUG 19:47:30 DEBUG { 19:47:30 DEBUG { 19:47:30 DEBUG 19:47:30 DEBUG "resultType": "vector", 19:47:30 DEBUG "value": [ 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG "data": { 19:47:30 DEBUG "metric": { 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG "metric": { 19:47:30 DEBUG "metric": { 19:47:30 DEBUG 19:47:30 DEBUG "result": [ 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG [...] 19:47:30 DEBUG "resultType": "vector", 19:47:30 DEBUG "instance": "10.142.0.107:9100" 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG "pod": "ds-idrepo-0" 19:47:30 DEBUG "instance": "10.142.0.107:9100" 19:47:30 DEBUG [...] 19:47:30 DEBUG { 19:47:30 DEBUG 19:47:30 DEBUG [...] 19:47:30 DEBUG 19:47:30 DEBUG "result": [ 19:47:30 DEBUG }, 19:47:30 DEBUG 19:47:30 DEBUG [...] 19:47:30 DEBUG }, 19:47:30 DEBUG }, 19:47:30 DEBUG 19:47:30 DEBUG "metric": { 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG { 19:47:30 DEBUG "value": [ 19:47:30 DEBUG [...] 19:47:30 DEBUG 19:47:30 DEBUG "value": [ 19:47:30 DEBUG "value": [ 19:47:30 DEBUG 19:47:30 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:47:30 DEBUG [...] 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG "metric": { 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG }, 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:47:30 DEBUG "pod": "ds-idrepo-0" 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:47:30 DEBUG "value": [ 19:47:30 DEBUG 19:47:30 DEBUG "instance": "10.142.0.43:9100" 19:47:30 DEBUG }, 19:47:30 DEBUG }, 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG "instance": "10.142.0.43:9100" 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG }, 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG "value": [ 19:47:30 DEBUG }, 19:47:30 DEBUG "value": [ 19:47:30 DEBUG [...] 19:47:30 DEBUG "pod": "ds-cts-2" 19:47:30 DEBUG }, 19:47:30 DEBUG [...] 19:47:30 DEBUG [...] 19:47:30 DEBUG "value": [ 19:47:30 DEBUG 19:47:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:47:30 DEBUG 1713552450, 19:47:30 DEBUG "value": [ 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG }, 19:47:30 DEBUG "value": [ 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG 1713552450, 19:47:30 DEBUG 19:47:30 DEBUG }, 19:47:30 DEBUG "0.16783333333336828" 19:47:30 DEBUG 1713552450, 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG "value": [ 19:47:30 DEBUG 1713552450, 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG "2502.733333333333" 19:47:30 DEBUG [...] 19:47:30 DEBUG "value": [ 19:47:30 DEBUG ] 19:47:30 DEBUG "0" 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG 1713552450, 19:47:30 DEBUG "1952769205248" 19:47:30 DEBUG 19:47:30 DEBUG "pod": "ds-cts-2" 19:47:30 DEBUG ] 19:47:30 DEBUG 19:47:30 DEBUG 1713552450, 19:47:30 DEBUG } 19:47:30 DEBUG ] 19:47:30 DEBUG [...] 19:47:30 DEBUG "instance": "10.142.0.43:9100" 19:47:30 DEBUG "0" 19:47:30 DEBUG ] 19:47:30 DEBUG 19:47:30 DEBUG }, 19:47:30 DEBUG "value": [ 19:47:30 DEBUG 19:47:30 DEBUG "2238" 19:47:30 DEBUG ] 19:47:30 DEBUG } 19:47:30 DEBUG 19:47:30 INFO [http_cmd]: http status code OK 19:47:30 DEBUG }, 19:47:30 DEBUG ] 19:47:30 DEBUG } 19:47:30 DEBUG "instance": "10.142.0.43:9100" 19:47:30 DEBUG 1713552450, 19:47:30 DEBUG } 19:47:30 DEBUG 19:47:30 DEBUG ] 19:47:30 DEBUG } 19:47:30 DEBUG ] 19:47:30 DEBUG 19:47:30 DEBUG --- status code --- 19:47:30 DEBUG "value": [ 19:47:30 DEBUG } 19:47:30 DEBUG ] 19:47:30 DEBUG }, 19:47:30 DEBUG "0" 19:47:30 DEBUG ] 19:47:30 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:47:30 DEBUG } 19:47:30 DEBUG } 19:47:30 DEBUG } 19:47:30 DEBUG 19:47:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:30 DEBUG 1713552450, 19:47:30 DEBUG ] 19:47:30 DEBUG } 19:47:30 DEBUG "value": [ 19:47:30 DEBUG ] 19:47:30 DEBUG } 19:47:30 DEBUG }, 19:47:30 DEBUG ] 19:47:30 DEBUG } 19:47:30 DEBUG "pod": "ds-idrepo-2" 19:47:30 DEBUG --- http response --- 19:47:30 DEBUG "0.07266666666667446" 19:47:30 DEBUG ] 19:47:30 DEBUG } 19:47:30 DEBUG 1713552450, 19:47:30 DEBUG } 19:47:30 DEBUG } 19:47:30 DEBUG "value": [ 19:47:30 DEBUG } 19:47:30 DEBUG }, 19:47:30 DEBUG } 19:47:30 DEBUG } 19:47:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:47:30 DEBUG } 19:47:30 DEBUG "74831621.76666667" 19:47:30 DEBUG ] 19:47:30 DEBUG 1713552450, 19:47:30 DEBUG } 19:47:30 DEBUG "value": [ 19:47:30 DEBUG ] 19:47:30 DEBUG ----- output ----- 19:47:30 DEBUG ] 19:47:30 DEBUG } 19:47:30 DEBUG "0" 19:47:30 DEBUG 1713552450, 19:47:30 DEBUG } 19:47:30 DEBUG { 19:47:30 DEBUG "status": "success", 19:47:30 DEBUG } 19:47:30 DEBUG ] 19:47:30 DEBUG "0" 19:47:30 DEBUG } 19:47:30 DEBUG } 19:47:30 DEBUG "data": { 19:47:30 DEBUG } 19:47:30 DEBUG ] 19:47:30 DEBUG ] 19:47:30 DEBUG "resultType": "vector", 19:47:30 DEBUG ] 19:47:30 DEBUG } 19:47:30 DEBUG } 19:47:30 DEBUG "result": [ 19:47:30 DEBUG } 19:47:30 DEBUG ] 19:47:30 DEBUG } 19:47:30 DEBUG { 19:47:30 DEBUG } 19:47:30 DEBUG } 19:47:30 DEBUG "metric": { 19:47:30 DEBUG } 19:47:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:47:30 DEBUG }, 19:47:30 DEBUG "value": [ 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG [...] 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:47:30 DEBUG }, 19:47:30 DEBUG "value": [ 19:47:30 DEBUG 1713552450, 19:47:30 DEBUG "2504.9666666666667" 19:47:30 DEBUG ] 19:47:30 DEBUG } 19:47:30 DEBUG ] 19:47:30 DEBUG } 19:47:30 DEBUG } 19:47:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552450" 19:47:30 INFO [http_cmd]: http status code OK 19:47:30 DEBUG --- status code --- 19:47:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:30 DEBUG --- http response --- 19:47:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:47:30 DEBUG ----- output ----- 19:47:30 DEBUG { 19:47:30 DEBUG "status": "success", 19:47:30 DEBUG "data": { 19:47:30 DEBUG "resultType": "vector", 19:47:30 DEBUG "result": [ 19:47:30 DEBUG { 19:47:30 DEBUG "metric": { 19:47:30 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:47:30 DEBUG }, 19:47:30 DEBUG "value": [ 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG [...] 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 19:47:30 DEBUG }, 19:47:30 DEBUG "value": [ 19:47:30 DEBUG 1713552450, 19:47:30 DEBUG "0" 19:47:30 DEBUG ] 19:47:30 DEBUG } 19:47:30 DEBUG ] 19:47:30 DEBUG } 19:47:30 DEBUG } 19:47:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552450" 19:47:30 INFO [http_cmd]: http status code OK 19:47:30 DEBUG --- status code --- 19:47:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:30 DEBUG --- http response --- 19:47:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:47:30 DEBUG ----- output ----- 19:47:30 DEBUG { 19:47:30 DEBUG "status": "success", 19:47:30 DEBUG "data": { 19:47:30 DEBUG "resultType": "vector", 19:47:30 DEBUG "result": [ 19:47:30 DEBUG { 19:47:30 DEBUG "metric": { 19:47:30 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:47:30 DEBUG }, 19:47:30 DEBUG "value": [ 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG [...] 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:47:30 DEBUG }, 19:47:30 DEBUG "value": [ 19:47:30 DEBUG 1713552450, 19:47:30 DEBUG "0" 19:47:30 DEBUG ] 19:47:30 DEBUG } 19:47:30 DEBUG ] 19:47:30 DEBUG } 19:47:30 DEBUG } 19:47:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552450" 19:47:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552450" 19:47:30 INFO [http_cmd]: http status code OK 19:47:30 DEBUG --- status code --- 19:47:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:30 DEBUG --- http response --- 19:47:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:47:30 DEBUG ----- output ----- 19:47:30 DEBUG { 19:47:30 DEBUG "status": "success", 19:47:30 DEBUG "data": { 19:47:30 DEBUG "resultType": "vector", 19:47:30 DEBUG "result": [ 19:47:30 DEBUG { 19:47:30 DEBUG "metric": { 19:47:30 DEBUG "instance": "10.142.0.107:9100" 19:47:30 DEBUG }, 19:47:30 DEBUG "value": [ 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG [...] 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG "instance": "10.142.0.43:9100" 19:47:30 DEBUG }, 19:47:30 DEBUG "value": [ 19:47:30 DEBUG 1713552450, 19:47:30 DEBUG "74827854.46666667" 19:47:30 DEBUG ] 19:47:30 DEBUG } 19:47:30 DEBUG ] 19:47:30 DEBUG } 19:47:30 DEBUG } 19:47:30 INFO [http_cmd]: http status code OK 19:47:30 DEBUG --- status code --- 19:47:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:30 DEBUG --- http response --- 19:47:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:47:30 DEBUG ----- output ----- 19:47:30 DEBUG { 19:47:30 DEBUG "status": "success", 19:47:30 DEBUG "data": { 19:47:30 DEBUG "resultType": "vector", 19:47:30 DEBUG "result": [ 19:47:30 DEBUG { 19:47:30 DEBUG "metric": { 19:47:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:47:30 DEBUG }, 19:47:30 DEBUG "value": [ 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG [...] 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:47:30 DEBUG }, 19:47:30 DEBUG "value": [ 19:47:30 DEBUG 1713552450, 19:47:30 DEBUG "2503.5333333333333" 19:47:30 DEBUG ] 19:47:30 DEBUG } 19:47:30 DEBUG ] 19:47:30 DEBUG } 19:47:30 DEBUG } 19:47:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713552450" 19:47:30 INFO [http_cmd]: http status code OK 19:47:30 DEBUG --- status code --- 19:47:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:30 DEBUG --- http response --- 19:47:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:47:30 DEBUG ----- output ----- 19:47:30 DEBUG { 19:47:30 DEBUG "status": "success", 19:47:30 DEBUG "data": { 19:47:30 DEBUG "resultType": "vector", 19:47:30 DEBUG "result": [ 19:47:30 DEBUG { 19:47:30 DEBUG "metric": { 19:47:30 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:47:30 DEBUG }, 19:47:30 DEBUG "value": [ 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG [...] 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG 19:47:30 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:47:30 DEBUG }, 19:47:30 DEBUG "value": [ 19:47:30 DEBUG 1713552450, 19:47:30 DEBUG "3461279744" 19:47:30 DEBUG ] 19:47:30 DEBUG } 19:47:30 DEBUG ] 19:47:30 DEBUG } 19:47:30 DEBUG } 19:47:31 INFO 19:47:31 INFO [loop_until]: kubectl --namespace=xlou top pods 19:47:31 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:47:31 INFO [loop_until]: OK (rc = 0) 19:47:31 DEBUG --- stdout --- 19:47:31 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 15737m 7850Mi am-767f6b67b5-qcqz2 15478m 8740Mi am-767f6b67b5-vbltz 16816m 8989Mi ds-cts-0 15780m 10915Mi ds-cts-1 14006m 11172Mi ds-cts-2 16407m 10927Mi ds-idrepo-0 7134m 3666Mi ds-idrepo-1 7193m 3537Mi ds-idrepo-2 7146m 3492Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 6m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8231m 3210Mi 19:47:31 DEBUG --- stderr --- 19:47:31 DEBUG 19:47:32 INFO 19:47:32 INFO [loop_until]: kubectl --namespace=xlou top node 19:47:32 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:47:32 INFO [loop_until]: OK (rc = 0) 19:47:32 DEBUG --- stdout --- 19:47:32 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 69m 0% 6245Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7403m 24% 7266Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 15564m 52% 10549Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 88m 0% 3343Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 16112m 53% 13899Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 156m 0% 5220Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 15842m 53% 11632Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15659m 52% 2986Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 17524m 58% 11197Mi 10% gke-xlou-cdm-ds-0f601751-j488 7390m 24% 6747Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 14173m 47% 15943Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 7207m 24% 7433Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 16178m 54% 13400Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 7940m 49% 6975Mi 11% 19:47:32 DEBUG --- stderr --- 19:47:32 DEBUG 19:48:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713552510" 19:48:30 ERROR [http_cmd]: ERROR 19:48:30 ERROR --- status code --- 19:48:30 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:48:30 ERROR --- http response --- 19:48:30 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:48:30 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:48:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552510" 19:48:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713552510" 19:48:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552510" 19:48:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552510" 19:48:30 INFO [http_cmd]: http status code OK 19:48:30 DEBUG --- status code --- 19:48:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:30 DEBUG --- http response --- 19:48:30 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:48:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552510" 19:48:30 INFO [http_cmd]: http status code OK 19:48:30 DEBUG --- status code --- 19:48:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:30 DEBUG --- http response --- 19:48:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:48:30 DEBUG ----- output ----- 19:48:30 DEBUG { 19:48:30 DEBUG "status": "success", 19:48:30 DEBUG "data": { 19:48:30 DEBUG "resultType": "vector", 19:48:30 DEBUG "result": [ 19:48:30 DEBUG { 19:48:30 DEBUG "metric": { 19:48:30 DEBUG "pod": "ds-cts-0" 19:48:30 DEBUG }, 19:48:30 DEBUG "value": [ 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG [...] 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 INFO [http_cmd]: http status code OK 19:48:30 DEBUG "pod": "ds-cts-2" 19:48:30 DEBUG --- status code --- 19:48:30 DEBUG }, 19:48:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:30 DEBUG "value": [ 19:48:30 DEBUG --- http response --- 19:48:30 DEBUG 1713552510, 19:48:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:48:30 DEBUG "0.00047006888888888885" 19:48:30 DEBUG ----- output ----- 19:48:30 DEBUG ] 19:48:30 DEBUG { 19:48:30 DEBUG } 19:48:30 DEBUG "status": "success", 19:48:30 DEBUG ] 19:48:30 DEBUG "data": { 19:48:30 DEBUG } 19:48:30 DEBUG "resultType": "vector", 19:48:30 DEBUG } 19:48:30 DEBUG "result": [ 19:48:30 DEBUG { 19:48:30 DEBUG "metric": { 19:48:30 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:48:30 DEBUG }, 19:48:30 DEBUG "value": [ 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG [...] 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:48:30 DEBUG }, 19:48:30 DEBUG "value": [ 19:48:30 DEBUG 1713552510, 19:48:30 DEBUG "8.041388870065616" 19:48:30 DEBUG ] 19:48:30 DEBUG } 19:48:30 DEBUG ] 19:48:30 DEBUG } 19:48:30 DEBUG } 19:48:30 INFO [http_cmd]: http status code OK 19:48:30 INFO [http_cmd]: http status code OK 19:48:30 DEBUG --- status code --- 19:48:30 DEBUG --- status code --- 19:48:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:30 DEBUG --- http response --- 19:48:30 DEBUG --- http response --- 19:48:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:48:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:48:30 DEBUG ----- output ----- 19:48:30 DEBUG ----- output ----- 19:48:30 DEBUG { 19:48:30 DEBUG { 19:48:30 DEBUG "status": "success", 19:48:30 DEBUG "status": "success", 19:48:30 DEBUG "data": { 19:48:30 DEBUG "data": { 19:48:30 DEBUG "resultType": "vector", 19:48:30 DEBUG "resultType": "vector", 19:48:30 DEBUG "result": [ 19:48:30 DEBUG "result": [ 19:48:30 DEBUG { 19:48:30 DEBUG { 19:48:30 DEBUG "metric": { 19:48:30 DEBUG "metric": { 19:48:30 DEBUG "pod": "ds-idrepo-0" 19:48:30 DEBUG "pod": "ds-idrepo-0" 19:48:30 DEBUG }, 19:48:30 DEBUG }, 19:48:30 DEBUG "value": [ 19:48:30 DEBUG "value": [ 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG [...] 19:48:30 DEBUG [...] 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG "pod": "ds-idrepo-2" 19:48:30 DEBUG "pod": "ds-idrepo-2" 19:48:30 DEBUG }, 19:48:30 DEBUG }, 19:48:30 DEBUG "value": [ 19:48:30 DEBUG "value": [ 19:48:30 DEBUG 1713552510, 19:48:30 DEBUG 1713552510, 19:48:30 DEBUG "0" 19:48:30 DEBUG "0" 19:48:30 DEBUG ] 19:48:30 DEBUG ] 19:48:30 DEBUG } 19:48:30 DEBUG } 19:48:30 DEBUG ] 19:48:30 DEBUG ] 19:48:30 DEBUG } 19:48:30 DEBUG } 19:48:30 DEBUG } 19:48:30 DEBUG } 19:48:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552510" 19:48:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713552510" 19:48:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552510" 19:48:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552510" 19:48:30 INFO [http_cmd]: http status code OK 19:48:30 DEBUG --- status code --- 19:48:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:30 DEBUG --- http response --- 19:48:30 INFO [http_cmd]: http status code OK 19:48:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:48:30 DEBUG --- status code --- 19:48:30 DEBUG ----- output ----- 19:48:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:30 DEBUG { 19:48:30 DEBUG --- http response --- 19:48:30 DEBUG "status": "success", 19:48:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 19:48:30 DEBUG "data": { 19:48:30 DEBUG ----- output ----- 19:48:30 DEBUG "resultType": "vector", 19:48:30 DEBUG { 19:48:30 INFO [http_cmd]: http status code OK 19:48:30 DEBUG "result": [ 19:48:30 INFO [http_cmd]: http status code OK 19:48:30 DEBUG "status": "success", 19:48:30 DEBUG --- status code --- 19:48:30 DEBUG { 19:48:30 DEBUG --- status code --- 19:48:30 DEBUG "data": { 19:48:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:30 DEBUG "metric": { 19:48:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:30 DEBUG "resultType": "vector", 19:48:30 DEBUG --- http response --- 19:48:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:48:30 DEBUG --- http response --- 19:48:30 DEBUG "result": [ 19:48:30 DEBUG }, 19:48:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:48:30 DEBUG { 19:48:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:48:30 DEBUG "value": [ 19:48:30 DEBUG ----- output ----- 19:48:30 DEBUG "metric": { 19:48:30 DEBUG ----- output ----- 19:48:30 DEBUG 19:48:30 DEBUG { 19:48:30 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:48:30 DEBUG { 19:48:30 DEBUG 19:48:30 DEBUG "status": "success", 19:48:30 DEBUG }, 19:48:30 DEBUG "status": "success", 19:48:30 DEBUG 19:48:30 DEBUG "data": { 19:48:30 DEBUG "value": [ 19:48:30 DEBUG "data": { 19:48:30 DEBUG [...] 19:48:30 DEBUG "resultType": "vector", 19:48:30 DEBUG 19:48:30 DEBUG "resultType": "vector", 19:48:30 DEBUG 19:48:30 DEBUG "result": [ 19:48:30 DEBUG 19:48:30 DEBUG "result": [ 19:48:30 DEBUG 19:48:30 DEBUG { 19:48:30 DEBUG 19:48:30 DEBUG { 19:48:30 DEBUG 19:48:30 DEBUG "metric": { 19:48:30 DEBUG [...] 19:48:30 DEBUG "metric": { 19:48:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:48:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:48:30 DEBUG 19:48:30 DEBUG "pod": "ds-cts-0" 19:48:30 DEBUG }, 19:48:30 DEBUG }, 19:48:30 DEBUG 19:48:30 DEBUG }, 19:48:30 DEBUG "value": [ 19:48:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552510" 19:48:30 DEBUG "value": [ 19:48:30 DEBUG 19:48:30 DEBUG "value": [ 19:48:30 DEBUG 1713552510, 19:48:30 DEBUG 19:48:30 DEBUG "pod": "ds-idrepo-2" 19:48:30 DEBUG 19:48:30 DEBUG "413869" 19:48:30 DEBUG 19:48:30 DEBUG }, 19:48:30 DEBUG 19:48:30 DEBUG ] 19:48:30 DEBUG 19:48:30 DEBUG "value": [ 19:48:30 DEBUG 19:48:30 DEBUG } 19:48:30 DEBUG [...] 19:48:30 DEBUG 1713552510, 19:48:30 DEBUG [...] 19:48:30 DEBUG ] 19:48:30 DEBUG 19:48:30 DEBUG "4124.132475097989" 19:48:30 DEBUG 19:48:30 DEBUG } 19:48:30 DEBUG 19:48:30 DEBUG ] 19:48:30 DEBUG 19:48:30 DEBUG } 19:48:30 DEBUG 19:48:30 DEBUG } 19:48:30 DEBUG 19:48:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:48:30 DEBUG ] 19:48:30 DEBUG "pod": "ds-cts-2" 19:48:30 DEBUG }, 19:48:30 DEBUG } 19:48:30 DEBUG }, 19:48:30 DEBUG "value": [ 19:48:30 DEBUG } 19:48:30 DEBUG "value": [ 19:48:30 DEBUG 1713552510, 19:48:30 DEBUG 1713552510, 19:48:30 DEBUG "0.013" 19:48:30 DEBUG "29.933333333333334" 19:48:30 DEBUG ] 19:48:30 DEBUG ] 19:48:30 DEBUG } 19:48:30 DEBUG } 19:48:30 DEBUG ] 19:48:30 DEBUG ] 19:48:30 DEBUG } 19:48:30 DEBUG } 19:48:30 DEBUG } 19:48:30 DEBUG } 19:48:30 INFO [http_cmd]: http status code OK 19:48:30 DEBUG --- status code --- 19:48:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:30 DEBUG --- http response --- 19:48:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:48:30 DEBUG ----- output ----- 19:48:30 DEBUG { 19:48:30 DEBUG "status": "success", 19:48:30 DEBUG "data": { 19:48:30 DEBUG "resultType": "vector", 19:48:30 DEBUG "result": [ 19:48:30 DEBUG { 19:48:30 DEBUG "metric": { 19:48:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:48:30 DEBUG }, 19:48:30 DEBUG "value": [ 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG [...] 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:48:30 DEBUG }, 19:48:30 DEBUG "value": [ 19:48:30 DEBUG 1713552510, 19:48:30 DEBUG "0" 19:48:30 DEBUG ] 19:48:30 DEBUG } 19:48:30 DEBUG ] 19:48:30 DEBUG } 19:48:30 DEBUG } 19:48:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713552510" 19:48:30 INFO [http_cmd]: http status code OK 19:48:30 DEBUG --- status code --- 19:48:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:30 DEBUG --- http response --- 19:48:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:48:30 DEBUG ----- output ----- 19:48:30 DEBUG { 19:48:30 DEBUG "status": "success", 19:48:30 DEBUG "data": { 19:48:30 DEBUG "resultType": "vector", 19:48:30 DEBUG "result": [ 19:48:30 DEBUG { 19:48:30 DEBUG "metric": { 19:48:30 DEBUG "pod": "ds-idrepo-0" 19:48:30 DEBUG }, 19:48:30 DEBUG "value": [ 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG [...] 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG "pod": "ds-cts-2" 19:48:30 DEBUG }, 19:48:30 DEBUG "value": [ 19:48:30 DEBUG 1713552510, 19:48:30 DEBUG "0" 19:48:30 DEBUG ] 19:48:30 DEBUG } 19:48:30 DEBUG ] 19:48:30 DEBUG } 19:48:30 DEBUG } 19:48:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552510" 19:48:30 INFO [http_cmd]: http status code OK 19:48:30 DEBUG --- status code --- 19:48:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:30 DEBUG --- http response --- 19:48:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:48:30 DEBUG ----- output ----- 19:48:30 DEBUG { 19:48:30 DEBUG "status": "success", 19:48:30 DEBUG "data": { 19:48:30 DEBUG "resultType": "vector", 19:48:30 DEBUG "result": [ 19:48:30 DEBUG { 19:48:30 DEBUG "metric": { 19:48:30 DEBUG "instance": "10.142.0.107:9100" 19:48:30 DEBUG }, 19:48:30 DEBUG "value": [ 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG [...] 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG "instance": "10.142.0.43:9100" 19:48:30 DEBUG }, 19:48:30 DEBUG "value": [ 19:48:30 DEBUG 1713552510, 19:48:30 DEBUG "0.05336666666669127" 19:48:30 DEBUG ] 19:48:30 DEBUG } 19:48:30 DEBUG ] 19:48:30 DEBUG } 19:48:30 DEBUG } 19:48:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552510" 19:48:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552510" 19:48:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552510" 19:48:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552510" 19:48:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552510" 19:48:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552510" 19:48:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552510" 19:48:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552510" 19:48:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552510" 19:48:30 INFO [http_cmd]: http status code OK 19:48:30 INFO [http_cmd]: http status code OK 19:48:30 DEBUG --- status code --- 19:48:30 DEBUG --- status code --- 19:48:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:30 DEBUG --- http response --- 19:48:30 DEBUG --- http response --- 19:48:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:48:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:48:30 DEBUG ----- output ----- 19:48:30 INFO [http_cmd]: http status code OK 19:48:30 INFO [http_cmd]: http status code OK 19:48:30 DEBUG ----- output ----- 19:48:30 DEBUG { 19:48:30 DEBUG --- status code --- 19:48:30 DEBUG --- status code --- 19:48:30 DEBUG { 19:48:30 DEBUG "status": "success", 19:48:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:30 DEBUG "status": "success", 19:48:30 DEBUG "data": { 19:48:30 DEBUG --- http response --- 19:48:30 DEBUG --- http response --- 19:48:30 DEBUG "data": { 19:48:30 DEBUG "resultType": "vector", 19:48:30 INFO [http_cmd]: http status code OK 19:48:30 INFO [http_cmd]: http status code OK 19:48:30 DEBUG "resultType": "vector", 19:48:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:48:30 DEBUG "result": [ 19:48:30 INFO [http_cmd]: http status code OK 19:48:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:48:30 DEBUG --- status code --- 19:48:30 DEBUG --- status code --- 19:48:30 DEBUG "result": [ 19:48:30 DEBUG ----- output ----- 19:48:30 DEBUG { 19:48:30 DEBUG --- status code --- 19:48:30 DEBUG ----- output ----- 19:48:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:30 DEBUG { 19:48:30 DEBUG { 19:48:30 DEBUG "metric": { 19:48:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:30 DEBUG { 19:48:30 DEBUG --- http response --- 19:48:30 DEBUG --- http response --- 19:48:30 DEBUG "metric": { 19:48:30 DEBUG "status": "success", 19:48:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:48:30 DEBUG --- http response --- 19:48:30 DEBUG "status": "success", 19:48:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:48:30 DEBUG "instance": "10.142.0.107:9100" 19:48:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:48:30 INFO [http_cmd]: http status code OK 19:48:30 DEBUG "data": { 19:48:30 DEBUG }, 19:48:30 DEBUG "data": { 19:48:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:48:30 DEBUG ----- output ----- 19:48:30 DEBUG }, 19:48:30 DEBUG ----- output ----- 19:48:30 DEBUG --- status code --- 19:48:30 DEBUG "resultType": "vector", 19:48:30 DEBUG "value": [ 19:48:30 DEBUG "resultType": "vector", 19:48:30 DEBUG ----- output ----- 19:48:30 DEBUG { 19:48:30 DEBUG "value": [ 19:48:30 DEBUG { 19:48:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:30 DEBUG "result": [ 19:48:30 DEBUG 19:48:30 DEBUG "result": [ 19:48:30 DEBUG { 19:48:30 DEBUG "status": "success", 19:48:30 DEBUG 19:48:30 DEBUG "status": "success", 19:48:30 DEBUG --- http response --- 19:48:30 INFO [http_cmd]: http status code OK 19:48:30 DEBUG { 19:48:30 DEBUG 19:48:30 DEBUG { 19:48:30 DEBUG "status": "success", 19:48:30 DEBUG "data": { 19:48:30 DEBUG 19:48:30 DEBUG "data": { 19:48:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:48:30 DEBUG --- status code --- 19:48:30 DEBUG "metric": { 19:48:30 DEBUG 19:48:30 DEBUG "metric": { 19:48:30 DEBUG "data": { 19:48:30 DEBUG "resultType": "vector", 19:48:30 DEBUG 19:48:30 DEBUG "resultType": "vector", 19:48:30 DEBUG ----- output ----- 19:48:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:30 DEBUG "pod": "ds-cts-0" 19:48:30 DEBUG [...] 19:48:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:48:30 DEBUG "resultType": "vector", 19:48:30 DEBUG "result": [ 19:48:30 DEBUG [...] 19:48:30 DEBUG "result": [ 19:48:30 DEBUG { 19:48:30 DEBUG --- http response --- 19:48:30 DEBUG }, 19:48:30 DEBUG 19:48:30 DEBUG }, 19:48:30 DEBUG "result": [ 19:48:30 DEBUG { 19:48:30 DEBUG 19:48:30 DEBUG { 19:48:30 DEBUG "status": "success", 19:48:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:48:30 DEBUG "value": [ 19:48:30 DEBUG 19:48:30 DEBUG "value": [ 19:48:30 DEBUG { 19:48:30 DEBUG "metric": { 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG "data": { 19:48:30 DEBUG ----- output ----- 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG "metric": { 19:48:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:48:30 DEBUG "metric": { 19:48:30 DEBUG "instance": "10.142.0.43:9100" 19:48:30 DEBUG "resultType": "vector", 19:48:30 DEBUG { 19:48:30 DEBUG 19:48:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:48:30 DEBUG 19:48:30 DEBUG "pod": "ds-idrepo-0" 19:48:30 DEBUG }, 19:48:30 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:48:30 DEBUG }, 19:48:30 DEBUG "result": [ 19:48:30 DEBUG "status": "success", 19:48:30 DEBUG 19:48:30 DEBUG }, 19:48:30 DEBUG 19:48:30 DEBUG }, 19:48:30 DEBUG "value": [ 19:48:30 DEBUG }, 19:48:30 DEBUG "value": [ 19:48:30 DEBUG { 19:48:30 DEBUG "data": { 19:48:30 DEBUG [...] 19:48:30 DEBUG "value": [ 19:48:30 DEBUG [...] 19:48:30 DEBUG "value": [ 19:48:30 DEBUG 19:48:30 DEBUG "value": [ 19:48:30 DEBUG 1713552510, 19:48:30 DEBUG "metric": { 19:48:30 DEBUG "resultType": "vector", 19:48:30 DEBUG 19:48:30 DEBUG 1713552510, 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG "1955586243584" 19:48:30 DEBUG "instance": "10.142.0.107:9100" 19:48:30 DEBUG "result": [ 19:48:30 DEBUG 19:48:30 DEBUG "2500.1" 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG ] 19:48:30 DEBUG }, 19:48:30 DEBUG { 19:48:30 DEBUG 19:48:30 DEBUG ] 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG [...] 19:48:30 DEBUG 19:48:30 DEBUG } 19:48:30 DEBUG "value": [ 19:48:30 DEBUG "metric": { 19:48:30 DEBUG "pod": "ds-cts-2" 19:48:30 DEBUG } 19:48:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:48:30 DEBUG [...] 19:48:30 DEBUG 19:48:30 DEBUG [...] 19:48:30 DEBUG ] 19:48:30 DEBUG 19:48:30 DEBUG "instance": "10.142.0.107:9100" 19:48:30 DEBUG }, 19:48:30 DEBUG ] 19:48:30 DEBUG }, 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG } 19:48:30 DEBUG 19:48:30 DEBUG }, 19:48:30 DEBUG "value": [ 19:48:30 DEBUG } 19:48:30 DEBUG "value": [ 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG } 19:48:30 DEBUG 19:48:30 DEBUG "value": [ 19:48:30 DEBUG 1713552510, 19:48:30 DEBUG } 19:48:30 DEBUG 1713552510, 19:48:30 DEBUG 19:48:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:48:30 DEBUG }, 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG "0" 19:48:30 DEBUG "459.6666666666667" 19:48:30 DEBUG "pod": "ds-idrepo-2" 19:48:30 DEBUG [...] 19:48:30 DEBUG "value": [ 19:48:30 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:48:30 DEBUG 19:48:30 DEBUG ] 19:48:30 DEBUG ] 19:48:30 DEBUG }, 19:48:30 DEBUG 19:48:30 DEBUG 1713552510, 19:48:30 DEBUG }, 19:48:30 DEBUG 19:48:30 DEBUG } 19:48:30 DEBUG } 19:48:30 DEBUG "value": [ 19:48:30 DEBUG 19:48:30 DEBUG "2500.233333333333" 19:48:30 DEBUG "value": [ 19:48:30 DEBUG [...] 19:48:30 DEBUG ] 19:48:30 DEBUG ] 19:48:30 DEBUG 1713552510, 19:48:30 DEBUG 19:48:30 DEBUG ] 19:48:30 DEBUG 1713552510, 19:48:30 DEBUG 19:48:30 DEBUG } 19:48:30 DEBUG } 19:48:30 DEBUG "0" 19:48:30 DEBUG "instance": "10.142.0.43:9100" 19:48:30 DEBUG } 19:48:30 DEBUG "0" 19:48:30 DEBUG 19:48:30 DEBUG } 19:48:30 DEBUG } 19:48:30 DEBUG ] 19:48:30 DEBUG }, 19:48:30 DEBUG ] 19:48:30 DEBUG ] 19:48:30 DEBUG 19:48:30 DEBUG } 19:48:30 DEBUG "value": [ 19:48:30 DEBUG } 19:48:30 DEBUG } 19:48:30 DEBUG "instance": "10.142.0.43:9100" 19:48:30 DEBUG ] 19:48:30 DEBUG 1713552510, 19:48:30 DEBUG } 19:48:30 DEBUG ] 19:48:30 DEBUG }, 19:48:30 DEBUG } 19:48:30 DEBUG "0.03733333333334296" 19:48:30 DEBUG } 19:48:30 DEBUG "value": [ 19:48:30 DEBUG } 19:48:30 DEBUG ] 19:48:30 DEBUG } 19:48:30 DEBUG 1713552510, 19:48:30 DEBUG } 19:48:30 DEBUG "59784554.13333333" 19:48:30 DEBUG ] 19:48:30 DEBUG ] 19:48:30 DEBUG } 19:48:30 DEBUG } 19:48:30 DEBUG } 19:48:30 DEBUG ] 19:48:30 DEBUG } 19:48:30 DEBUG } 19:48:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552510" 19:48:30 INFO [http_cmd]: http status code OK 19:48:30 DEBUG --- status code --- 19:48:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:30 DEBUG --- http response --- 19:48:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 19:48:30 DEBUG ----- output ----- 19:48:30 DEBUG { 19:48:30 DEBUG "status": "success", 19:48:30 DEBUG "data": { 19:48:30 DEBUG "resultType": "vector", 19:48:30 DEBUG "result": [ 19:48:30 DEBUG { 19:48:30 DEBUG "metric": { 19:48:30 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:48:30 DEBUG }, 19:48:30 DEBUG "value": [ 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG [...] 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG "pod": "ds-idrepo-2" 19:48:30 DEBUG }, 19:48:30 DEBUG "value": [ 19:48:30 DEBUG 1713552510, 19:48:30 DEBUG "0" 19:48:30 DEBUG ] 19:48:30 DEBUG } 19:48:30 DEBUG ] 19:48:30 DEBUG } 19:48:30 DEBUG } 19:48:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552510" 19:48:30 INFO [http_cmd]: http status code OK 19:48:30 DEBUG --- status code --- 19:48:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:30 DEBUG --- http response --- 19:48:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:48:30 DEBUG ----- output ----- 19:48:30 DEBUG { 19:48:30 DEBUG "status": "success", 19:48:30 DEBUG "data": { 19:48:30 DEBUG "resultType": "vector", 19:48:30 DEBUG "result": [ 19:48:30 DEBUG { 19:48:30 DEBUG "metric": { 19:48:30 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:48:30 DEBUG }, 19:48:30 DEBUG "value": [ 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG [...] 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:48:30 DEBUG }, 19:48:30 DEBUG "value": [ 19:48:30 DEBUG 1713552510, 19:48:30 DEBUG "0" 19:48:30 DEBUG ] 19:48:30 DEBUG } 19:48:30 DEBUG ] 19:48:30 DEBUG } 19:48:30 DEBUG } 19:48:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552510" 19:48:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552510" 19:48:30 INFO [http_cmd]: http status code OK 19:48:30 DEBUG --- status code --- 19:48:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:30 DEBUG --- http response --- 19:48:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:48:30 DEBUG ----- output ----- 19:48:30 DEBUG { 19:48:30 DEBUG "status": "success", 19:48:30 DEBUG "data": { 19:48:30 DEBUG "resultType": "vector", 19:48:30 DEBUG "result": [ 19:48:30 DEBUG { 19:48:30 DEBUG "metric": { 19:48:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:48:30 DEBUG }, 19:48:30 DEBUG "value": [ 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG [...] 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:48:30 DEBUG }, 19:48:30 DEBUG "value": [ 19:48:30 DEBUG 1713552510, 19:48:30 DEBUG "2503.766666666667" 19:48:30 DEBUG ] 19:48:30 DEBUG } 19:48:30 DEBUG ] 19:48:30 DEBUG } 19:48:30 DEBUG } 19:48:30 INFO [http_cmd]: http status code OK 19:48:30 DEBUG --- status code --- 19:48:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:30 DEBUG --- http response --- 19:48:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:48:30 DEBUG ----- output ----- 19:48:30 DEBUG { 19:48:30 DEBUG "status": "success", 19:48:30 DEBUG "data": { 19:48:30 DEBUG "resultType": "vector", 19:48:30 DEBUG "result": [ 19:48:30 DEBUG { 19:48:30 DEBUG "metric": { 19:48:30 DEBUG "instance": "10.142.0.107:9100" 19:48:30 DEBUG }, 19:48:30 DEBUG "value": [ 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG [...] 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG "instance": "10.142.0.43:9100" 19:48:30 DEBUG }, 19:48:30 DEBUG "value": [ 19:48:30 DEBUG 1713552510, 19:48:30 DEBUG "59780336.766666666" 19:48:30 DEBUG ] 19:48:30 DEBUG } 19:48:30 DEBUG ] 19:48:30 DEBUG } 19:48:30 DEBUG } 19:48:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713552510" 19:48:30 INFO [http_cmd]: http status code OK 19:48:30 DEBUG --- status code --- 19:48:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:30 DEBUG --- http response --- 19:48:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:48:30 DEBUG ----- output ----- 19:48:30 DEBUG { 19:48:30 DEBUG "status": "success", 19:48:30 DEBUG "data": { 19:48:30 DEBUG "resultType": "vector", 19:48:30 DEBUG "result": [ 19:48:30 DEBUG { 19:48:30 DEBUG "metric": { 19:48:30 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:48:30 DEBUG }, 19:48:30 DEBUG "value": [ 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG [...] 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG 19:48:30 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:48:30 DEBUG }, 19:48:30 DEBUG "value": [ 19:48:30 DEBUG 1713552510, 19:48:30 DEBUG "3464101888" 19:48:30 DEBUG ] 19:48:30 DEBUG } 19:48:30 DEBUG ] 19:48:30 DEBUG } 19:48:30 DEBUG } 19:48:31 INFO 19:48:31 INFO [loop_until]: kubectl --namespace=xlou top pods 19:48:31 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:48:31 INFO [loop_until]: OK (rc = 0) 19:48:31 DEBUG --- stdout --- 19:48:31 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 13997m 8972Mi am-767f6b67b5-qcqz2 13697m 8736Mi am-767f6b67b5-vbltz 14445m 8985Mi ds-cts-0 9534m 10952Mi ds-cts-1 7480m 11189Mi ds-cts-2 9187m 10958Mi ds-idrepo-0 6363m 3666Mi ds-idrepo-1 6429m 3537Mi ds-idrepo-2 6566m 3493Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 4m 1217Mi lodemon-f8459c597-6tj6f 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8008m 3213Mi 19:48:31 DEBUG --- stderr --- 19:48:31 DEBUG 19:48:32 INFO 19:48:32 INFO [loop_until]: kubectl --namespace=xlou top node 19:48:32 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:48:32 INFO [loop_until]: OK (rc = 0) 19:48:32 DEBUG --- stdout --- 19:48:32 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 67m 0% 6254Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 6642m 22% 7261Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 14000m 46% 11671Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 76m 0% 3346Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 9142m 30% 13957Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 156m 0% 5192Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 14177m 47% 11623Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15789m 52% 2994Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 15688m 52% 11187Mi 10% gke-xlou-cdm-ds-0f601751-j488 6885m 23% 6740Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 8667m 29% 15973Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 6843m 22% 7428Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 8996m 30% 13430Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 8203m 51% 6977Mi 11% 19:48:32 DEBUG --- stderr --- 19:48:32 DEBUG 19:49:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713552570" 19:49:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552570" 19:49:30 ERROR [http_cmd]: ERROR 19:49:30 ERROR --- status code --- 19:49:30 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:49:30 ERROR --- http response --- 19:49:30 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:49:30 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:49:30 INFO [http_cmd]: http status code OK 19:49:30 DEBUG --- status code --- 19:49:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:30 DEBUG --- http response --- 19:49:30 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:49:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552570" 19:49:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713552570" 19:49:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552570" 19:49:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552570" 19:49:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552570" 19:49:30 INFO [http_cmd]: http status code OK 19:49:30 DEBUG --- status code --- 19:49:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:30 DEBUG --- http response --- 19:49:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:49:30 DEBUG ----- output ----- 19:49:30 DEBUG { 19:49:30 DEBUG "status": "success", 19:49:30 DEBUG "data": { 19:49:30 DEBUG "resultType": "vector", 19:49:30 DEBUG "result": [ 19:49:30 DEBUG { 19:49:30 DEBUG "metric": { 19:49:30 DEBUG "pod": "ds-idrepo-0" 19:49:30 DEBUG }, 19:49:30 DEBUG "value": [ 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG [...] 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:49:30 DEBUG }, 19:49:30 DEBUG "value": [ 19:49:30 DEBUG 1713552570, 19:49:30 DEBUG "0" 19:49:30 DEBUG ] 19:49:30 DEBUG } 19:49:30 DEBUG ] 19:49:30 DEBUG } 19:49:30 DEBUG } 19:49:30 INFO [http_cmd]: http status code OK 19:49:30 DEBUG --- status code --- 19:49:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:30 DEBUG --- http response --- 19:49:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:49:30 DEBUG ----- output ----- 19:49:30 DEBUG { 19:49:30 DEBUG "status": "success", 19:49:30 DEBUG "data": { 19:49:30 DEBUG "resultType": "vector", 19:49:30 DEBUG "result": [ 19:49:30 DEBUG { 19:49:30 DEBUG "metric": { 19:49:30 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:49:30 DEBUG }, 19:49:30 DEBUG "value": [ 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG [...] 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:49:30 DEBUG }, 19:49:30 DEBUG "value": [ 19:49:30 DEBUG 1713552570, 19:49:30 DEBUG "8.043842028869063" 19:49:30 DEBUG ] 19:49:30 DEBUG } 19:49:30 DEBUG ] 19:49:30 DEBUG } 19:49:30 DEBUG } 19:49:30 INFO [http_cmd]: http status code OK 19:49:30 DEBUG --- status code --- 19:49:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:30 DEBUG --- http response --- 19:49:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:49:30 DEBUG ----- output ----- 19:49:30 DEBUG { 19:49:30 DEBUG "status": "success", 19:49:30 DEBUG "data": { 19:49:30 DEBUG "resultType": "vector", 19:49:30 DEBUG "result": [ 19:49:30 DEBUG { 19:49:30 DEBUG "metric": { 19:49:30 DEBUG "pod": "ds-cts-0" 19:49:30 DEBUG }, 19:49:30 DEBUG "value": [ 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG [...] 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG "pod": "ds-cts-2" 19:49:30 DEBUG }, 19:49:30 DEBUG "value": [ 19:49:30 DEBUG 1713552570, 19:49:30 DEBUG "0.0004" 19:49:30 DEBUG ] 19:49:30 DEBUG } 19:49:30 DEBUG ] 19:49:30 DEBUG } 19:49:30 DEBUG } 19:49:30 INFO [http_cmd]: http status code OK 19:49:30 DEBUG --- status code --- 19:49:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:30 DEBUG --- http response --- 19:49:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:49:30 DEBUG ----- output ----- 19:49:30 DEBUG { 19:49:30 DEBUG "status": "success", 19:49:30 DEBUG "data": { 19:49:30 DEBUG "resultType": "vector", 19:49:30 DEBUG "result": [ 19:49:30 DEBUG { 19:49:30 DEBUG "metric": { 19:49:30 DEBUG "pod": "ds-idrepo-0" 19:49:30 DEBUG }, 19:49:30 DEBUG "value": [ 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG [...] 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG "pod": "ds-idrepo-2" 19:49:30 DEBUG }, 19:49:30 DEBUG "value": [ 19:49:30 DEBUG 1713552570, 19:49:30 DEBUG "0" 19:49:30 DEBUG ] 19:49:30 DEBUG } 19:49:30 DEBUG ] 19:49:30 DEBUG } 19:49:30 DEBUG } 19:49:30 INFO [http_cmd]: http status code OK 19:49:30 DEBUG --- status code --- 19:49:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:30 DEBUG --- http response --- 19:49:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:49:30 DEBUG ----- output ----- 19:49:30 DEBUG { 19:49:30 DEBUG "status": "success", 19:49:30 DEBUG "data": { 19:49:30 DEBUG "resultType": "vector", 19:49:30 DEBUG "result": [ 19:49:30 DEBUG { 19:49:30 DEBUG "metric": { 19:49:30 DEBUG "pod": "ds-idrepo-0" 19:49:30 DEBUG }, 19:49:30 DEBUG "value": [ 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG [...] 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG "pod": "ds-idrepo-2" 19:49:30 DEBUG }, 19:49:30 DEBUG "value": [ 19:49:30 DEBUG 1713552570, 19:49:30 DEBUG "0" 19:49:30 DEBUG ] 19:49:30 DEBUG } 19:49:30 DEBUG ] 19:49:30 DEBUG } 19:49:30 DEBUG } 19:49:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552570" 19:49:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713552570" 19:49:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552570" 19:49:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552570" 19:49:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552570" 19:49:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713552570" 19:49:30 INFO [http_cmd]: http status code OK 19:49:30 DEBUG --- status code --- 19:49:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:30 DEBUG --- http response --- 19:49:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:49:30 DEBUG ----- output ----- 19:49:30 DEBUG { 19:49:30 DEBUG "status": "success", 19:49:30 DEBUG "data": { 19:49:30 INFO [http_cmd]: http status code OK 19:49:30 DEBUG "resultType": "vector", 19:49:30 DEBUG --- status code --- 19:49:30 DEBUG "result": [ 19:49:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:30 DEBUG { 19:49:30 DEBUG --- http response --- 19:49:30 DEBUG "metric": { 19:49:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:49:30 DEBUG }, 19:49:30 DEBUG "value": [ 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG [...] 19:49:30 DEBUG 19:49:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:49:30 DEBUG 19:49:30 DEBUG ----- output ----- 19:49:30 DEBUG 19:49:30 DEBUG { 19:49:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:49:30 DEBUG "status": "success", 19:49:30 DEBUG }, 19:49:30 DEBUG "data": { 19:49:30 DEBUG "value": [ 19:49:30 DEBUG "resultType": "vector", 19:49:30 DEBUG 1713552570, 19:49:30 DEBUG "result": [ 19:49:30 DEBUG "450604" 19:49:30 DEBUG { 19:49:30 DEBUG ] 19:49:30 DEBUG "metric": { 19:49:30 DEBUG } 19:49:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:49:30 DEBUG ] 19:49:30 DEBUG }, 19:49:30 DEBUG } 19:49:30 DEBUG "value": [ 19:49:30 DEBUG } 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG [...] 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:49:30 DEBUG }, 19:49:30 DEBUG "value": [ 19:49:30 DEBUG 1713552570, 19:49:30 DEBUG "0.0135" 19:49:30 DEBUG ] 19:49:30 DEBUG } 19:49:30 DEBUG ] 19:49:30 DEBUG } 19:49:30 DEBUG } 19:49:30 INFO [http_cmd]: http status code OK 19:49:30 INFO [http_cmd]: http status code OK 19:49:30 INFO [http_cmd]: http status code OK 19:49:30 DEBUG --- status code --- 19:49:30 DEBUG --- status code --- 19:49:30 DEBUG --- status code --- 19:49:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:30 DEBUG --- http response --- 19:49:30 DEBUG --- http response --- 19:49:30 DEBUG --- http response --- 19:49:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:49:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:49:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:49:30 DEBUG ----- output ----- 19:49:30 DEBUG ----- output ----- 19:49:30 DEBUG ----- output ----- 19:49:30 DEBUG { 19:49:30 DEBUG { 19:49:30 DEBUG { 19:49:30 DEBUG "status": "success", 19:49:30 DEBUG "status": "success", 19:49:30 DEBUG "status": "success", 19:49:30 DEBUG "data": { 19:49:30 DEBUG "data": { 19:49:30 DEBUG "data": { 19:49:30 DEBUG "resultType": "vector", 19:49:30 DEBUG "resultType": "vector", 19:49:30 DEBUG "resultType": "vector", 19:49:30 DEBUG "result": [ 19:49:30 DEBUG "result": [ 19:49:30 DEBUG "result": [ 19:49:30 DEBUG { 19:49:30 DEBUG { 19:49:30 DEBUG { 19:49:30 DEBUG "metric": { 19:49:30 DEBUG "metric": { 19:49:30 DEBUG "metric": { 19:49:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:49:30 INFO [http_cmd]: http status code OK 19:49:30 DEBUG "instance": "10.142.0.107:9100" 19:49:30 DEBUG "pod": "ds-cts-0" 19:49:30 DEBUG }, 19:49:30 DEBUG --- status code --- 19:49:30 DEBUG }, 19:49:30 DEBUG }, 19:49:30 DEBUG "value": [ 19:49:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:30 DEBUG "value": [ 19:49:30 DEBUG "value": [ 19:49:30 DEBUG 19:49:30 DEBUG --- http response --- 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG ----- output ----- 19:49:30 DEBUG 19:49:30 DEBUG [...] 19:49:30 DEBUG [...] 19:49:30 DEBUG { 19:49:30 DEBUG [...] 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG "status": "success", 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG "data": { 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG "resultType": "vector", 19:49:30 DEBUG 19:49:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:49:30 DEBUG "instance": "10.142.0.43:9100" 19:49:30 DEBUG "result": [ 19:49:30 DEBUG "pod": "ds-cts-2" 19:49:30 DEBUG }, 19:49:30 DEBUG }, 19:49:30 DEBUG { 19:49:30 DEBUG }, 19:49:30 DEBUG "value": [ 19:49:30 DEBUG "value": [ 19:49:30 DEBUG "metric": { 19:49:30 DEBUG "value": [ 19:49:30 DEBUG 1713552570, 19:49:30 DEBUG 1713552570, 19:49:30 DEBUG "pod": "ds-idrepo-0" 19:49:30 DEBUG 1713552570, 19:49:30 DEBUG "0" 19:49:30 DEBUG "0.10039999999992612" 19:49:30 DEBUG }, 19:49:30 DEBUG "29.966666666666665" 19:49:30 DEBUG ] 19:49:30 DEBUG ] 19:49:30 DEBUG "value": [ 19:49:30 DEBUG ] 19:49:30 DEBUG } 19:49:30 DEBUG } 19:49:30 DEBUG 19:49:30 DEBUG } 19:49:30 DEBUG ] 19:49:30 DEBUG ] 19:49:30 DEBUG 19:49:30 DEBUG ] 19:49:30 DEBUG } 19:49:30 DEBUG } 19:49:30 DEBUG 19:49:30 DEBUG } 19:49:30 DEBUG } 19:49:30 DEBUG } 19:49:30 DEBUG [...] 19:49:30 DEBUG } 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG "pod": "ds-cts-2" 19:49:30 DEBUG }, 19:49:30 DEBUG "value": [ 19:49:30 DEBUG 1713552570, 19:49:30 DEBUG "0" 19:49:30 DEBUG ] 19:49:30 DEBUG } 19:49:30 DEBUG ] 19:49:30 DEBUG } 19:49:30 DEBUG } 19:49:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552570" 19:49:30 INFO [http_cmd]: http status code OK 19:49:30 DEBUG --- status code --- 19:49:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:30 DEBUG --- http response --- 19:49:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:49:30 DEBUG ----- output ----- 19:49:30 DEBUG { 19:49:30 DEBUG "status": "success", 19:49:30 DEBUG "data": { 19:49:30 DEBUG "resultType": "vector", 19:49:30 DEBUG "result": [ 19:49:30 DEBUG { 19:49:30 DEBUG "metric": { 19:49:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:49:30 DEBUG }, 19:49:30 DEBUG "value": [ 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG [...] 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:49:30 DEBUG }, 19:49:30 DEBUG "value": [ 19:49:30 DEBUG 1713552570, 19:49:30 DEBUG "2495.1666666666665" 19:49:30 DEBUG ] 19:49:30 DEBUG } 19:49:30 DEBUG ] 19:49:30 DEBUG } 19:49:30 DEBUG } 19:49:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552570" 19:49:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552570" 19:49:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552570" 19:49:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552570" 19:49:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552570" 19:49:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552570" 19:49:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552570" 19:49:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552570" 19:49:30 INFO [http_cmd]: http status code OK 19:49:30 DEBUG --- status code --- 19:49:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:30 DEBUG --- http response --- 19:49:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:49:30 INFO [http_cmd]: http status code OK 19:49:30 DEBUG ----- output ----- 19:49:30 INFO [http_cmd]: http status code OK 19:49:30 DEBUG --- status code --- 19:49:30 DEBUG { 19:49:30 DEBUG --- status code --- 19:49:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:30 DEBUG "status": "success", 19:49:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:30 DEBUG --- http response --- 19:49:30 DEBUG "data": { 19:49:30 DEBUG --- http response --- 19:49:30 DEBUG "resultType": "vector", 19:49:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:49:30 DEBUG "result": [ 19:49:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:49:30 DEBUG ----- output ----- 19:49:30 INFO [http_cmd]: http status code OK 19:49:30 DEBUG { 19:49:30 INFO [http_cmd]: http status code OK 19:49:30 DEBUG ----- output ----- 19:49:30 DEBUG { 19:49:30 DEBUG --- status code --- 19:49:30 DEBUG "metric": { 19:49:30 DEBUG --- status code --- 19:49:30 DEBUG { 19:49:30 DEBUG "status": "success", 19:49:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:49:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:30 DEBUG "status": "success", 19:49:30 DEBUG "data": { 19:49:30 DEBUG --- http response --- 19:49:30 DEBUG }, 19:49:30 DEBUG --- http response --- 19:49:30 DEBUG "data": { 19:49:30 DEBUG "resultType": "vector", 19:49:30 DEBUG "value": [ 19:49:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:49:30 DEBUG "resultType": "vector", 19:49:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:49:30 DEBUG "result": [ 19:49:30 DEBUG 19:49:30 DEBUG ----- output ----- 19:49:30 DEBUG "result": [ 19:49:30 DEBUG ----- output ----- 19:49:30 DEBUG { 19:49:30 DEBUG 19:49:30 DEBUG { 19:49:30 DEBUG { 19:49:30 DEBUG { 19:49:30 DEBUG "metric": { 19:49:30 DEBUG 19:49:30 DEBUG "status": "success", 19:49:30 DEBUG "metric": { 19:49:30 DEBUG "status": "success", 19:49:30 DEBUG "pod": "ds-cts-0" 19:49:30 DEBUG [...] 19:49:30 DEBUG "data": { 19:49:30 DEBUG "instance": "10.142.0.107:9100" 19:49:30 DEBUG "data": { 19:49:30 DEBUG }, 19:49:30 DEBUG 19:49:30 DEBUG "resultType": "vector", 19:49:30 DEBUG }, 19:49:30 DEBUG "resultType": "vector", 19:49:30 DEBUG "value": [ 19:49:30 DEBUG 19:49:30 DEBUG "result": [ 19:49:30 DEBUG "value": [ 19:49:30 DEBUG "result": [ 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG { 19:49:30 DEBUG 19:49:30 DEBUG { 19:49:30 DEBUG 19:49:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:49:30 DEBUG "metric": { 19:49:30 DEBUG 19:49:30 DEBUG "metric": { 19:49:30 DEBUG 19:49:30 DEBUG }, 19:49:30 DEBUG "pod": "ds-idrepo-0" 19:49:30 DEBUG 19:49:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:49:30 DEBUG [...] 19:49:30 DEBUG "value": [ 19:49:30 DEBUG }, 19:49:30 DEBUG [...] 19:49:30 DEBUG }, 19:49:30 DEBUG 19:49:30 DEBUG 1713552570, 19:49:30 DEBUG "value": [ 19:49:30 DEBUG 19:49:30 DEBUG "value": [ 19:49:30 DEBUG 19:49:30 DEBUG "2495.133333333333" 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 INFO [http_cmd]: http status code OK 19:49:30 DEBUG 19:49:30 INFO [http_cmd]: http status code OK 19:49:30 DEBUG ] 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG --- status code --- 19:49:30 DEBUG "pod": "ds-cts-2" 19:49:30 DEBUG --- status code --- 19:49:30 DEBUG } 19:49:30 DEBUG 19:49:30 DEBUG "instance": "10.142.0.43:9100" 19:49:30 DEBUG 19:49:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:30 DEBUG }, 19:49:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:30 DEBUG ] 19:49:30 DEBUG [...] 19:49:30 DEBUG }, 19:49:30 DEBUG [...] 19:49:30 DEBUG --- http response --- 19:49:30 DEBUG "value": [ 19:49:30 DEBUG --- http response --- 19:49:30 DEBUG } 19:49:30 DEBUG 19:49:30 DEBUG "value": [ 19:49:30 DEBUG 19:49:30 DEBUG 1713552570, 19:49:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:49:30 DEBUG } 19:49:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:49:30 DEBUG 19:49:30 DEBUG 1713552570, 19:49:30 DEBUG 19:49:30 INFO [http_cmd]: http status code OK 19:49:30 DEBUG "0" 19:49:30 DEBUG ----- output ----- 19:49:30 DEBUG ----- output ----- 19:49:30 DEBUG 19:49:30 DEBUG "1958477360128" 19:49:30 DEBUG 19:49:30 DEBUG --- status code --- 19:49:30 DEBUG ] 19:49:30 DEBUG { 19:49:30 DEBUG { 19:49:30 DEBUG "pod": "ds-idrepo-2" 19:49:30 DEBUG ] 19:49:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:49:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:30 DEBUG } 19:49:30 DEBUG "status": "success", 19:49:30 DEBUG "status": "success", 19:49:30 DEBUG }, 19:49:30 DEBUG } 19:49:30 DEBUG }, 19:49:30 DEBUG --- http response --- 19:49:30 DEBUG ] 19:49:30 DEBUG "data": { 19:49:30 DEBUG "data": { 19:49:30 DEBUG "value": [ 19:49:30 DEBUG ] 19:49:30 DEBUG "value": [ 19:49:30 DEBUG } 19:49:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:49:30 DEBUG "resultType": "vector", 19:49:30 DEBUG "resultType": "vector", 19:49:30 DEBUG 1713552570, 19:49:30 DEBUG } 19:49:30 DEBUG 1713552570, 19:49:30 DEBUG } 19:49:30 DEBUG ----- output ----- 19:49:30 DEBUG "result": [ 19:49:30 DEBUG "result": [ 19:49:30 DEBUG "0" 19:49:30 DEBUG } 19:49:30 DEBUG "2490.4333333333334" 19:49:30 DEBUG { 19:49:30 DEBUG { 19:49:30 DEBUG { 19:49:30 DEBUG ] 19:49:30 DEBUG ] 19:49:30 DEBUG "status": "success", 19:49:30 DEBUG "metric": { 19:49:30 DEBUG "metric": { 19:49:30 DEBUG } 19:49:30 DEBUG } 19:49:30 DEBUG "data": { 19:49:30 DEBUG "instance": "10.142.0.107:9100" 19:49:30 DEBUG "pod": "ds-idrepo-0" 19:49:30 DEBUG ] 19:49:30 DEBUG ] 19:49:30 DEBUG "resultType": "vector", 19:49:30 DEBUG }, 19:49:30 DEBUG }, 19:49:30 DEBUG } 19:49:30 DEBUG } 19:49:30 DEBUG "result": [ 19:49:30 DEBUG "value": [ 19:49:30 DEBUG "value": [ 19:49:30 DEBUG } 19:49:30 DEBUG } 19:49:30 DEBUG { 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG "metric": { 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG "instance": "10.142.0.107:9100" 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG }, 19:49:30 DEBUG [...] 19:49:30 DEBUG [...] 19:49:30 DEBUG "value": [ 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG "instance": "10.142.0.43:9100" 19:49:30 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:49:30 DEBUG [...] 19:49:30 DEBUG }, 19:49:30 DEBUG }, 19:49:30 DEBUG 19:49:30 DEBUG "value": [ 19:49:30 DEBUG "value": [ 19:49:30 DEBUG 19:49:30 DEBUG 1713552570, 19:49:30 DEBUG 1713552570, 19:49:30 DEBUG 19:49:30 DEBUG "75422002.66666666" 19:49:30 DEBUG "0" 19:49:30 DEBUG "instance": "10.142.0.43:9100" 19:49:30 DEBUG ] 19:49:30 DEBUG ] 19:49:30 DEBUG }, 19:49:30 DEBUG } 19:49:30 DEBUG } 19:49:30 DEBUG "value": [ 19:49:30 DEBUG ] 19:49:30 DEBUG ] 19:49:30 DEBUG 1713552570, 19:49:30 DEBUG } 19:49:30 DEBUG } 19:49:30 DEBUG "0.045999999999996044" 19:49:30 DEBUG } 19:49:30 DEBUG } 19:49:30 DEBUG ] 19:49:30 DEBUG } 19:49:30 DEBUG ] 19:49:30 DEBUG } 19:49:30 DEBUG } 19:49:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552570" 19:49:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552570" 19:49:30 INFO [http_cmd]: http status code OK 19:49:30 DEBUG --- status code --- 19:49:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:30 DEBUG --- http response --- 19:49:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:49:30 DEBUG ----- output ----- 19:49:30 DEBUG { 19:49:30 DEBUG "status": "success", 19:49:30 DEBUG "data": { 19:49:30 DEBUG "resultType": "vector", 19:49:30 DEBUG "result": [ 19:49:30 DEBUG { 19:49:30 DEBUG "metric": { 19:49:30 DEBUG "pod": "ds-idrepo-0" 19:49:30 DEBUG }, 19:49:30 DEBUG "value": [ 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG [...] 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:49:30 DEBUG }, 19:49:30 DEBUG "value": [ 19:49:30 DEBUG 1713552570, 19:49:30 DEBUG "0" 19:49:30 DEBUG ] 19:49:30 DEBUG } 19:49:30 DEBUG ] 19:49:30 DEBUG } 19:49:30 DEBUG } 19:49:30 INFO [http_cmd]: http status code OK 19:49:30 DEBUG --- status code --- 19:49:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:30 DEBUG --- http response --- 19:49:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:49:30 DEBUG ----- output ----- 19:49:30 DEBUG { 19:49:30 DEBUG "status": "success", 19:49:30 DEBUG "data": { 19:49:30 DEBUG "resultType": "vector", 19:49:30 DEBUG "result": [ 19:49:30 DEBUG { 19:49:30 DEBUG "metric": { 19:49:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:49:30 DEBUG }, 19:49:30 DEBUG "value": [ 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG [...] 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:49:30 DEBUG }, 19:49:30 DEBUG "value": [ 19:49:30 DEBUG 1713552570, 19:49:30 DEBUG "2496.633333333333" 19:49:30 DEBUG ] 19:49:30 DEBUG } 19:49:30 DEBUG ] 19:49:30 DEBUG } 19:49:30 DEBUG } 19:49:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552570" 19:49:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552570" 19:49:30 INFO [http_cmd]: http status code OK 19:49:30 DEBUG --- status code --- 19:49:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:30 DEBUG --- http response --- 19:49:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:49:30 DEBUG ----- output ----- 19:49:30 DEBUG { 19:49:30 DEBUG "status": "success", 19:49:30 DEBUG "data": { 19:49:30 DEBUG "resultType": "vector", 19:49:30 DEBUG "result": [ 19:49:30 DEBUG { 19:49:30 DEBUG "metric": { 19:49:30 DEBUG "instance": "10.142.0.107:9100" 19:49:30 DEBUG }, 19:49:30 DEBUG "value": [ 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG [...] 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG "instance": "10.142.0.43:9100" 19:49:30 DEBUG }, 19:49:30 DEBUG "value": [ 19:49:30 DEBUG 1713552570, 19:49:30 DEBUG "75419473.93333334" 19:49:30 DEBUG ] 19:49:30 DEBUG } 19:49:30 DEBUG ] 19:49:30 DEBUG } 19:49:30 DEBUG } 19:49:30 INFO [http_cmd]: http status code OK 19:49:30 DEBUG --- status code --- 19:49:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:30 DEBUG --- http response --- 19:49:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:49:30 DEBUG ----- output ----- 19:49:30 DEBUG { 19:49:30 DEBUG "status": "success", 19:49:30 DEBUG "data": { 19:49:30 DEBUG "resultType": "vector", 19:49:30 DEBUG "result": [ 19:49:30 DEBUG { 19:49:30 DEBUG "metric": { 19:49:30 DEBUG "pod": "ds-idrepo-0" 19:49:30 DEBUG }, 19:49:30 DEBUG "value": [ 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG [...] 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:49:30 DEBUG }, 19:49:30 DEBUG "value": [ 19:49:30 DEBUG 1713552570, 19:49:30 DEBUG "0.000025730840999136396" 19:49:30 DEBUG ] 19:49:30 DEBUG } 19:49:30 DEBUG ] 19:49:30 DEBUG } 19:49:30 DEBUG } 19:49:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713552570" 19:49:30 INFO [http_cmd]: http status code OK 19:49:30 DEBUG --- status code --- 19:49:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:30 DEBUG --- http response --- 19:49:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:49:30 DEBUG ----- output ----- 19:49:30 DEBUG { 19:49:30 DEBUG "status": "success", 19:49:30 DEBUG "data": { 19:49:30 DEBUG "resultType": "vector", 19:49:30 DEBUG "result": [ 19:49:30 DEBUG { 19:49:30 DEBUG "metric": { 19:49:30 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:49:30 DEBUG }, 19:49:30 DEBUG "value": [ 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG [...] 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG 19:49:30 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:49:30 DEBUG }, 19:49:30 DEBUG "value": [ 19:49:30 DEBUG 1713552570, 19:49:30 DEBUG "3469725696" 19:49:30 DEBUG ] 19:49:30 DEBUG } 19:49:30 DEBUG ] 19:49:30 DEBUG } 19:49:30 DEBUG } 19:49:31 INFO 19:49:31 INFO [loop_until]: kubectl --namespace=xlou top pods 19:49:31 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:49:31 INFO [loop_until]: OK (rc = 0) 19:49:31 DEBUG --- stdout --- 19:49:31 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 15896m 8996Mi am-767f6b67b5-qcqz2 15941m 8745Mi am-767f6b67b5-vbltz 17396m 8987Mi ds-cts-0 17205m 10931Mi ds-cts-1 15763m 11173Mi ds-cts-2 16519m 10943Mi ds-idrepo-0 6952m 3666Mi ds-idrepo-1 7187m 3537Mi ds-idrepo-2 7365m 3493Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 1m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8189m 3217Mi 19:49:31 DEBUG --- stderr --- 19:49:31 DEBUG 19:49:32 INFO 19:49:32 INFO [loop_until]: kubectl --namespace=xlou top node 19:49:32 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:49:32 INFO [loop_until]: OK (rc = 0) 19:49:32 DEBUG --- stdout --- 19:49:32 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 68m 0% 6249Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7406m 24% 7266Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16100m 53% 11689Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 84m 0% 3334Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 17148m 57% 13937Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 140m 0% 5208Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 16392m 54% 11633Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15414m 51% 2985Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 17549m 58% 11196Mi 10% gke-xlou-cdm-ds-0f601751-j488 7664m 25% 6732Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 15402m 51% 15941Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 7225m 24% 7417Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 17037m 57% 13395Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 8215m 51% 6979Mi 11% 19:49:32 DEBUG --- stderr --- 19:49:32 DEBUG 19:50:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552630" 19:50:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713552630" 19:50:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552630" 19:50:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552630" 19:50:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552630" 19:50:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713552630" 19:50:30 ERROR [http_cmd]: ERROR 19:50:30 INFO [http_cmd]: http status code OK 19:50:30 ERROR --- status code --- 19:50:30 DEBUG --- status code --- 19:50:30 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:50:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:30 ERROR --- http response --- 19:50:30 DEBUG --- http response --- 19:50:30 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:50:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:50:30 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:50:30 DEBUG ----- output ----- 19:50:30 DEBUG { 19:50:30 DEBUG "status": "success", 19:50:30 DEBUG "data": { 19:50:30 DEBUG "resultType": "vector", 19:50:30 DEBUG "result": [ 19:50:30 DEBUG { 19:50:30 DEBUG "metric": { 19:50:30 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:50:30 DEBUG }, 19:50:30 DEBUG "value": [ 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG [...] 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG "pod": "idm-b85f9d978-txpwv" 19:50:30 DEBUG }, 19:50:30 DEBUG "value": [ 19:50:30 DEBUG 1713552630, 19:50:30 DEBUG "451.21699596345866" 19:50:30 DEBUG ] 19:50:30 DEBUG } 19:50:30 DEBUG ] 19:50:30 DEBUG } 19:50:30 DEBUG } 19:50:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552630" 19:50:30 INFO [http_cmd]: http status code OK 19:50:30 INFO [http_cmd]: http status code OK 19:50:30 DEBUG --- status code --- 19:50:30 DEBUG --- status code --- 19:50:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:30 DEBUG --- http response --- 19:50:30 DEBUG --- http response --- 19:50:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:50:30 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:50:30 DEBUG ----- output ----- 19:50:30 DEBUG { 19:50:30 DEBUG "status": "success", 19:50:30 DEBUG "data": { 19:50:30 DEBUG "resultType": "vector", 19:50:30 DEBUG "result": [ 19:50:30 DEBUG { 19:50:30 DEBUG "metric": { 19:50:30 DEBUG "pod": "ds-idrepo-0" 19:50:30 DEBUG }, 19:50:30 DEBUG "value": [ 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG [...] 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG "pod": "ds-idrepo-2" 19:50:30 DEBUG }, 19:50:30 DEBUG "value": [ 19:50:30 DEBUG 1713552630, 19:50:30 DEBUG "0" 19:50:30 DEBUG ] 19:50:30 DEBUG } 19:50:30 DEBUG ] 19:50:30 DEBUG } 19:50:30 DEBUG } 19:50:30 INFO [http_cmd]: http status code OK 19:50:30 DEBUG --- status code --- 19:50:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:30 DEBUG --- http response --- 19:50:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:50:30 DEBUG ----- output ----- 19:50:30 DEBUG { 19:50:30 DEBUG "status": "success", 19:50:30 DEBUG "data": { 19:50:30 DEBUG "resultType": "vector", 19:50:30 DEBUG "result": [ 19:50:30 DEBUG { 19:50:30 DEBUG "metric": { 19:50:30 DEBUG "pod": "ds-cts-0" 19:50:30 DEBUG }, 19:50:30 DEBUG "value": [ 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG [...] 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG "pod": "ds-cts-2" 19:50:30 DEBUG }, 19:50:30 DEBUG "value": [ 19:50:30 DEBUG 1713552630, 19:50:30 DEBUG "0.0004666666666666667" 19:50:30 DEBUG ] 19:50:30 DEBUG } 19:50:30 DEBUG ] 19:50:30 DEBUG } 19:50:30 DEBUG } 19:50:30 INFO [http_cmd]: http status code OK 19:50:30 DEBUG --- status code --- 19:50:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:30 DEBUG --- http response --- 19:50:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:50:30 DEBUG ----- output ----- 19:50:30 DEBUG { 19:50:30 DEBUG "status": "success", 19:50:30 DEBUG "data": { 19:50:30 DEBUG "resultType": "vector", 19:50:30 DEBUG "result": [ 19:50:30 DEBUG { 19:50:30 DEBUG "metric": { 19:50:30 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:50:30 DEBUG }, 19:50:30 DEBUG "value": [ 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG [...] 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:50:30 DEBUG }, 19:50:30 DEBUG "value": [ 19:50:30 DEBUG 1713552630, 19:50:30 DEBUG "8.039759859925557" 19:50:30 DEBUG ] 19:50:30 DEBUG } 19:50:30 DEBUG ] 19:50:30 DEBUG } 19:50:30 DEBUG } 19:50:30 INFO [http_cmd]: http status code OK 19:50:30 DEBUG --- status code --- 19:50:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:30 DEBUG --- http response --- 19:50:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:50:30 DEBUG ----- output ----- 19:50:30 DEBUG { 19:50:30 DEBUG "status": "success", 19:50:30 DEBUG "data": { 19:50:30 DEBUG "resultType": "vector", 19:50:30 DEBUG "result": [ 19:50:30 DEBUG { 19:50:30 DEBUG "metric": { 19:50:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:50:30 DEBUG }, 19:50:30 DEBUG "value": [ 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG [...] 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:50:30 DEBUG }, 19:50:30 DEBUG "value": [ 19:50:30 DEBUG 1713552630, 19:50:30 DEBUG "450530.5" 19:50:30 DEBUG ] 19:50:30 DEBUG } 19:50:30 DEBUG ] 19:50:30 DEBUG } 19:50:30 DEBUG } 19:50:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552630" 19:50:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552630" 19:50:30 INFO [http_cmd]: http status code OK 19:50:30 DEBUG --- status code --- 19:50:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:30 DEBUG --- http response --- 19:50:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:50:30 DEBUG ----- output ----- 19:50:30 DEBUG { 19:50:30 DEBUG "status": "success", 19:50:30 DEBUG "data": { 19:50:30 DEBUG "resultType": "vector", 19:50:30 DEBUG "result": [ 19:50:30 DEBUG { 19:50:30 DEBUG "metric": { 19:50:30 DEBUG "pod": "ds-idrepo-0" 19:50:30 DEBUG }, 19:50:30 DEBUG "value": [ 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG [...] 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG "pod": "ds-idrepo-2" 19:50:30 DEBUG }, 19:50:30 DEBUG "value": [ 19:50:30 DEBUG 1713552630, 19:50:30 DEBUG "0" 19:50:30 DEBUG ] 19:50:30 DEBUG } 19:50:30 DEBUG ] 19:50:30 DEBUG } 19:50:30 DEBUG } 19:50:30 INFO [http_cmd]: http status code OK 19:50:30 DEBUG --- status code --- 19:50:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:30 DEBUG --- http response --- 19:50:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:50:30 DEBUG ----- output ----- 19:50:30 DEBUG { 19:50:30 DEBUG "status": "success", 19:50:30 DEBUG "data": { 19:50:30 DEBUG "resultType": "vector", 19:50:30 DEBUG "result": [ 19:50:30 DEBUG { 19:50:30 DEBUG "metric": { 19:50:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:50:30 DEBUG }, 19:50:30 DEBUG "value": [ 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG [...] 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:50:30 DEBUG }, 19:50:30 DEBUG "value": [ 19:50:30 DEBUG 1713552630, 19:50:30 DEBUG "0" 19:50:30 DEBUG ] 19:50:30 DEBUG } 19:50:30 DEBUG ] 19:50:30 DEBUG } 19:50:30 DEBUG } 19:50:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713552630" 19:50:30 INFO [http_cmd]: http status code OK 19:50:30 DEBUG --- status code --- 19:50:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:30 DEBUG --- http response --- 19:50:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:50:30 DEBUG ----- output ----- 19:50:30 DEBUG { 19:50:30 DEBUG "status": "success", 19:50:30 DEBUG "data": { 19:50:30 DEBUG "resultType": "vector", 19:50:30 DEBUG "result": [ 19:50:30 DEBUG { 19:50:30 DEBUG "metric": { 19:50:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:50:30 DEBUG }, 19:50:30 DEBUG "value": [ 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG [...] 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:50:30 DEBUG }, 19:50:30 DEBUG "value": [ 19:50:30 DEBUG 1713552630, 19:50:30 DEBUG "0.014" 19:50:30 DEBUG ] 19:50:30 DEBUG } 19:50:30 DEBUG ] 19:50:30 DEBUG } 19:50:30 DEBUG } 19:50:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713552630" 19:50:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552630" 19:50:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552630" 19:50:30 INFO [http_cmd]: http status code OK 19:50:30 DEBUG --- status code --- 19:50:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:30 DEBUG --- http response --- 19:50:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:50:30 DEBUG ----- output ----- 19:50:30 DEBUG { 19:50:30 DEBUG "status": "success", 19:50:30 DEBUG "data": { 19:50:30 DEBUG "resultType": "vector", 19:50:30 DEBUG "result": [ 19:50:30 DEBUG { 19:50:30 DEBUG "metric": { 19:50:30 DEBUG "pod": "ds-idrepo-0" 19:50:30 DEBUG }, 19:50:30 DEBUG "value": [ 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG [...] 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG "pod": "ds-cts-2" 19:50:30 DEBUG }, 19:50:30 DEBUG "value": [ 19:50:30 DEBUG 1713552630, 19:50:30 DEBUG "0" 19:50:30 DEBUG ] 19:50:30 DEBUG } 19:50:30 DEBUG ] 19:50:30 DEBUG } 19:50:30 DEBUG } 19:50:30 INFO [http_cmd]: http status code OK 19:50:30 DEBUG --- status code --- 19:50:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:30 DEBUG --- http response --- 19:50:30 INFO [http_cmd]: http status code OK 19:50:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:50:30 DEBUG --- status code --- 19:50:30 DEBUG ----- output ----- 19:50:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:30 DEBUG { 19:50:30 DEBUG --- http response --- 19:50:30 DEBUG "status": "success", 19:50:30 DEBUG "data": { 19:50:30 DEBUG "resultType": "vector", 19:50:30 DEBUG "result": [ 19:50:30 DEBUG { 19:50:30 DEBUG "metric": { 19:50:30 DEBUG "pod": "ds-cts-0" 19:50:30 DEBUG }, 19:50:30 DEBUG "value": [ 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG [...] 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG "pod": "ds-cts-2" 19:50:30 DEBUG }, 19:50:30 DEBUG "value": [ 19:50:30 DEBUG 1713552630, 19:50:30 DEBUG "30" 19:50:30 DEBUG ] 19:50:30 DEBUG } 19:50:30 DEBUG ] 19:50:30 DEBUG } 19:50:30 DEBUG } 19:50:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:50:30 DEBUG ----- output ----- 19:50:30 DEBUG { 19:50:30 DEBUG "status": "success", 19:50:30 DEBUG "data": { 19:50:30 DEBUG "resultType": "vector", 19:50:30 DEBUG "result": [ 19:50:30 DEBUG { 19:50:30 DEBUG "metric": { 19:50:30 DEBUG "instance": "10.142.0.107:9100" 19:50:30 DEBUG }, 19:50:30 DEBUG "value": [ 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG [...] 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG "instance": "10.142.0.43:9100" 19:50:30 DEBUG }, 19:50:30 DEBUG "value": [ 19:50:30 DEBUG 1713552630, 19:50:30 DEBUG "0.1603999999999966" 19:50:30 DEBUG ] 19:50:30 DEBUG } 19:50:30 DEBUG ] 19:50:30 DEBUG } 19:50:30 DEBUG } 19:50:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552630" 19:50:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552630" 19:50:30 INFO [http_cmd]: http status code OK 19:50:30 DEBUG --- status code --- 19:50:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:30 DEBUG --- http response --- 19:50:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:50:30 DEBUG ----- output ----- 19:50:30 DEBUG { 19:50:30 DEBUG "status": "success", 19:50:30 DEBUG "data": { 19:50:30 DEBUG "resultType": "vector", 19:50:30 DEBUG "result": [ 19:50:30 DEBUG { 19:50:30 DEBUG "metric": { 19:50:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:50:30 DEBUG }, 19:50:30 DEBUG "value": [ 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG [...] 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:50:30 DEBUG }, 19:50:30 DEBUG "value": [ 19:50:30 DEBUG 1713552630, 19:50:30 DEBUG "2508.3" 19:50:30 DEBUG ] 19:50:30 DEBUG } 19:50:30 DEBUG ] 19:50:30 DEBUG } 19:50:30 DEBUG } 19:50:30 INFO [http_cmd]: http status code OK 19:50:30 DEBUG --- status code --- 19:50:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:30 DEBUG --- http response --- 19:50:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:50:30 DEBUG ----- output ----- 19:50:30 DEBUG { 19:50:30 DEBUG "status": "success", 19:50:30 DEBUG "data": { 19:50:30 DEBUG "resultType": "vector", 19:50:30 DEBUG "result": [ 19:50:30 DEBUG { 19:50:30 DEBUG "metric": { 19:50:30 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:50:30 DEBUG }, 19:50:30 DEBUG "value": [ 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG [...] 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:50:30 DEBUG }, 19:50:30 DEBUG "value": [ 19:50:30 DEBUG 1713552630, 19:50:30 DEBUG "0" 19:50:30 DEBUG ] 19:50:30 DEBUG } 19:50:30 DEBUG ] 19:50:30 DEBUG } 19:50:30 DEBUG } 19:50:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552630" 19:50:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552630" 19:50:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552630" 19:50:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552630" 19:50:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552630" 19:50:30 INFO [http_cmd]: http status code OK 19:50:30 DEBUG --- status code --- 19:50:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:30 DEBUG --- http response --- 19:50:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:50:30 DEBUG ----- output ----- 19:50:30 DEBUG { 19:50:30 DEBUG "status": "success", 19:50:30 DEBUG "data": { 19:50:30 DEBUG "resultType": "vector", 19:50:30 DEBUG "result": [ 19:50:30 DEBUG { 19:50:30 DEBUG "metric": { 19:50:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:50:30 DEBUG }, 19:50:30 DEBUG "value": [ 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG [...] 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552630" 19:50:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:50:30 INFO [http_cmd]: http status code OK 19:50:30 DEBUG }, 19:50:30 DEBUG --- status code --- 19:50:30 DEBUG "value": [ 19:50:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:30 DEBUG 1713552630, 19:50:30 DEBUG --- http response --- 19:50:30 DEBUG "2502.4333333333334" 19:50:30 DEBUG ] 19:50:30 DEBUG } 19:50:30 DEBUG ] 19:50:30 DEBUG } 19:50:30 DEBUG } 19:50:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:50:30 DEBUG ----- output ----- 19:50:30 DEBUG { 19:50:30 DEBUG "status": "success", 19:50:30 DEBUG "data": { 19:50:30 DEBUG "resultType": "vector", 19:50:30 DEBUG "result": [ 19:50:30 DEBUG { 19:50:30 DEBUG "metric": { 19:50:30 DEBUG "instance": "10.142.0.107:9100" 19:50:30 DEBUG }, 19:50:30 DEBUG "value": [ 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG [...] 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG "instance": "10.142.0.43:9100" 19:50:30 DEBUG }, 19:50:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552630" 19:50:30 DEBUG "value": [ 19:50:30 INFO [http_cmd]: http status code OK 19:50:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552630" 19:50:30 INFO [http_cmd]: http status code OK 19:50:30 DEBUG 1713552630, 19:50:30 DEBUG --- status code --- 19:50:30 DEBUG --- status code --- 19:50:30 DEBUG "0.06666666666667047" 19:50:30 INFO [http_cmd]: http status code OK 19:50:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:30 DEBUG ] 19:50:30 DEBUG --- status code --- 19:50:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552630" 19:50:30 DEBUG --- http response --- 19:50:30 DEBUG --- http response --- 19:50:30 DEBUG } 19:50:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:50:30 DEBUG ] 19:50:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:50:30 DEBUG --- http response --- 19:50:30 DEBUG ----- output ----- 19:50:30 DEBUG } 19:50:30 DEBUG ----- output ----- 19:50:30 DEBUG { 19:50:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:50:30 DEBUG } 19:50:30 DEBUG { 19:50:30 DEBUG "status": "success", 19:50:30 DEBUG ----- output ----- 19:50:30 DEBUG "status": "success", 19:50:30 DEBUG "data": { 19:50:30 DEBUG { 19:50:30 DEBUG "data": { 19:50:30 INFO [http_cmd]: http status code OK 19:50:30 DEBUG "resultType": "vector", 19:50:30 DEBUG "status": "success", 19:50:30 DEBUG "resultType": "vector", 19:50:30 DEBUG --- status code --- 19:50:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552630" 19:50:30 DEBUG "result": [ 19:50:30 DEBUG "data": { 19:50:30 DEBUG "result": [ 19:50:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:30 DEBUG { 19:50:30 DEBUG "resultType": "vector", 19:50:30 DEBUG { 19:50:30 INFO [http_cmd]: http status code OK 19:50:30 DEBUG --- http response --- 19:50:30 DEBUG "metric": { 19:50:30 DEBUG "result": [ 19:50:30 DEBUG "metric": { 19:50:30 DEBUG --- status code --- 19:50:30 DEBUG "instance": "10.142.0.107:9100" 19:50:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:50:30 DEBUG { 19:50:30 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:50:30 INFO [http_cmd]: http status code OK 19:50:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:30 INFO [http_cmd]: http status code OK 19:50:30 DEBUG }, 19:50:30 DEBUG ----- output ----- 19:50:30 DEBUG "metric": { 19:50:30 DEBUG }, 19:50:30 DEBUG --- status code --- 19:50:30 DEBUG --- http response --- 19:50:30 DEBUG --- status code --- 19:50:30 DEBUG "value": [ 19:50:30 DEBUG { 19:50:30 DEBUG "pod": "ds-cts-0" 19:50:30 DEBUG "value": [ 19:50:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:50:30 DEBUG 19:50:30 DEBUG "status": "success", 19:50:30 DEBUG }, 19:50:30 DEBUG 19:50:30 DEBUG --- http response --- 19:50:30 DEBUG --- http response --- 19:50:30 DEBUG ----- output ----- 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG "value": [ 19:50:30 DEBUG 19:50:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:50:30 DEBUG { 19:50:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:50:30 DEBUG [...] 19:50:30 DEBUG "data": { 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG ----- output ----- 19:50:30 DEBUG "status": "success", 19:50:30 DEBUG ----- output ----- 19:50:30 DEBUG 19:50:30 DEBUG "resultType": "vector", 19:50:30 DEBUG 19:50:30 DEBUG [...] 19:50:30 DEBUG { 19:50:30 DEBUG "data": { 19:50:30 DEBUG { 19:50:30 DEBUG 19:50:30 DEBUG "result": [ 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG "status": "success", 19:50:30 DEBUG "resultType": "vector", 19:50:30 DEBUG "status": "success", 19:50:30 DEBUG 19:50:30 DEBUG { 19:50:30 DEBUG [...] 19:50:30 DEBUG 19:50:30 DEBUG "data": { 19:50:30 DEBUG "result": [ 19:50:30 DEBUG "data": { 19:50:30 DEBUG "instance": "10.142.0.43:9100" 19:50:30 DEBUG "metric": { 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG "resultType": "vector", 19:50:30 DEBUG { 19:50:30 DEBUG "resultType": "vector", 19:50:30 DEBUG }, 19:50:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:50:30 DEBUG 19:50:30 DEBUG "pod": "idm-b85f9d978-txpwv" 19:50:30 DEBUG "result": [ 19:50:30 DEBUG "metric": { 19:50:30 DEBUG "result": [ 19:50:30 DEBUG "value": [ 19:50:30 DEBUG }, 19:50:30 DEBUG 19:50:30 DEBUG }, 19:50:30 DEBUG { 19:50:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:50:30 DEBUG { 19:50:30 DEBUG 1713552630, 19:50:30 DEBUG "value": [ 19:50:30 DEBUG "pod": "ds-cts-2" 19:50:30 DEBUG "value": [ 19:50:30 DEBUG "metric": { 19:50:30 DEBUG }, 19:50:30 DEBUG "metric": { 19:50:30 DEBUG "1961655725056" 19:50:30 DEBUG 19:50:30 DEBUG }, 19:50:30 DEBUG 1713552630, 19:50:30 DEBUG "instance": "10.142.0.107:9100" 19:50:30 DEBUG "value": [ 19:50:30 DEBUG "pod": "ds-idrepo-0" 19:50:30 DEBUG ] 19:50:30 DEBUG 19:50:30 DEBUG "value": [ 19:50:30 DEBUG "0" 19:50:30 DEBUG }, 19:50:30 DEBUG 19:50:30 DEBUG }, 19:50:30 DEBUG } 19:50:30 DEBUG 19:50:30 DEBUG 1713552630, 19:50:30 INFO [http_cmd]: http status code OK 19:50:30 DEBUG ] 19:50:30 DEBUG "value": [ 19:50:30 DEBUG 19:50:30 DEBUG "value": [ 19:50:30 DEBUG ] 19:50:30 DEBUG [...] 19:50:30 DEBUG "0" 19:50:30 DEBUG --- status code --- 19:50:30 DEBUG } 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG } 19:50:30 DEBUG 19:50:30 DEBUG ] 19:50:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:30 DEBUG ] 19:50:30 DEBUG 19:50:30 DEBUG [...] 19:50:30 DEBUG 19:50:30 DEBUG } 19:50:30 DEBUG 19:50:30 DEBUG } 19:50:30 DEBUG --- http response --- 19:50:30 DEBUG } 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG ] 19:50:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:50:30 DEBUG } 19:50:30 DEBUG [...] 19:50:30 DEBUG 19:50:30 DEBUG [...] 19:50:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:50:30 DEBUG } 19:50:30 DEBUG ----- output ----- 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG }, 19:50:30 DEBUG } 19:50:30 DEBUG { 19:50:30 DEBUG 19:50:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:50:30 DEBUG 19:50:30 DEBUG "value": [ 19:50:30 DEBUG "status": "success", 19:50:30 DEBUG 19:50:30 DEBUG }, 19:50:30 DEBUG 19:50:30 DEBUG 1713552630, 19:50:30 DEBUG "data": { 19:50:30 DEBUG "instance": "10.142.0.43:9100" 19:50:30 DEBUG "value": [ 19:50:30 DEBUG "pod": "ds-idrepo-2" 19:50:30 DEBUG "2507.7" 19:50:30 DEBUG "resultType": "vector", 19:50:30 DEBUG }, 19:50:30 DEBUG 1713552630, 19:50:30 DEBUG }, 19:50:30 DEBUG ] 19:50:30 DEBUG "result": [ 19:50:30 DEBUG "value": [ 19:50:30 DEBUG "2508.633333333333" 19:50:30 DEBUG "value": [ 19:50:30 DEBUG } 19:50:30 DEBUG { 19:50:30 DEBUG 1713552630, 19:50:30 DEBUG ] 19:50:30 DEBUG 1713552630, 19:50:30 DEBUG "0" 19:50:30 DEBUG "metric": { 19:50:30 DEBUG "75524464.26666667" 19:50:30 DEBUG } 19:50:30 DEBUG ] 19:50:30 DEBUG ] 19:50:30 DEBUG "instance": "10.142.0.107:9100" 19:50:30 DEBUG ] 19:50:30 DEBUG ] 19:50:30 DEBUG } 19:50:30 DEBUG } 19:50:30 DEBUG }, 19:50:30 DEBUG } 19:50:30 DEBUG } 19:50:30 DEBUG } 19:50:30 DEBUG ] 19:50:30 DEBUG "value": [ 19:50:30 DEBUG ] 19:50:30 DEBUG } 19:50:30 DEBUG } 19:50:30 DEBUG 19:50:30 DEBUG } 19:50:30 DEBUG } 19:50:30 DEBUG 19:50:30 DEBUG } 19:50:30 DEBUG 19:50:30 DEBUG [...] 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG "instance": "10.142.0.43:9100" 19:50:30 DEBUG }, 19:50:30 DEBUG "value": [ 19:50:30 DEBUG 1713552630, 19:50:30 DEBUG "75520678.86666667" 19:50:30 DEBUG ] 19:50:30 DEBUG } 19:50:30 DEBUG ] 19:50:30 DEBUG } 19:50:30 DEBUG } 19:50:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552630" 19:50:30 INFO [http_cmd]: http status code OK 19:50:30 DEBUG --- status code --- 19:50:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:30 DEBUG --- http response --- 19:50:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:50:30 DEBUG ----- output ----- 19:50:30 DEBUG { 19:50:30 DEBUG "status": "success", 19:50:30 DEBUG "data": { 19:50:30 DEBUG "resultType": "vector", 19:50:30 DEBUG "result": [ 19:50:30 DEBUG { 19:50:30 DEBUG "metric": { 19:50:30 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:50:30 DEBUG }, 19:50:30 DEBUG "value": [ 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG [...] 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:50:30 DEBUG }, 19:50:30 DEBUG "value": [ 19:50:30 DEBUG 1713552630, 19:50:30 DEBUG "0" 19:50:30 DEBUG ] 19:50:30 DEBUG } 19:50:30 DEBUG ] 19:50:30 DEBUG } 19:50:30 DEBUG } 19:50:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713552630" 19:50:30 INFO [http_cmd]: http status code OK 19:50:30 DEBUG --- status code --- 19:50:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:30 DEBUG --- http response --- 19:50:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:50:30 DEBUG ----- output ----- 19:50:30 DEBUG { 19:50:30 DEBUG "status": "success", 19:50:30 DEBUG "data": { 19:50:30 DEBUG "resultType": "vector", 19:50:30 DEBUG "result": [ 19:50:30 DEBUG { 19:50:30 DEBUG "metric": { 19:50:30 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:50:30 DEBUG }, 19:50:30 DEBUG "value": [ 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG [...] 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG 19:50:30 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:50:30 DEBUG }, 19:50:30 DEBUG "value": [ 19:50:30 DEBUG 1713552630, 19:50:30 DEBUG "3473633280" 19:50:30 DEBUG ] 19:50:30 DEBUG } 19:50:30 DEBUG ] 19:50:30 DEBUG } 19:50:30 DEBUG } 19:50:31 INFO 19:50:31 INFO [loop_until]: kubectl --namespace=xlou top pods 19:50:31 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:50:31 INFO [loop_until]: OK (rc = 0) 19:50:31 DEBUG --- stdout --- 19:50:31 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 16241m 9000Mi am-767f6b67b5-qcqz2 16298m 8747Mi am-767f6b67b5-vbltz 17608m 8990Mi ds-cts-0 17400m 10950Mi ds-cts-1 15766m 11194Mi ds-cts-2 16720m 10957Mi ds-idrepo-0 7255m 3666Mi ds-idrepo-1 7097m 3537Mi ds-idrepo-2 7318m 3493Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8052m 3220Mi 19:50:31 DEBUG --- stderr --- 19:50:31 DEBUG 19:50:32 INFO 19:50:32 INFO [loop_until]: kubectl --namespace=xlou top node 19:50:32 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:50:32 INFO [loop_until]: OK (rc = 0) 19:50:32 DEBUG --- stdout --- 19:50:32 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 69m 0% 6249Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7416m 24% 7268Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 15714m 52% 11697Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 89m 0% 3340Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 16923m 56% 13922Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 158m 0% 5201Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 16467m 55% 11628Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15733m 52% 2973Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 18232m 61% 11212Mi 10% gke-xlou-cdm-ds-0f601751-j488 7662m 25% 6735Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 15637m 52% 15960Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 7420m 24% 7434Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 17159m 57% 13402Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 8200m 51% 6982Mi 11% 19:50:32 DEBUG --- stderr --- 19:50:32 DEBUG 19:51:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713552690" 19:51:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552690" 19:51:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552690" 19:51:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552690" 19:51:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552690" 19:51:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552690" 19:51:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713552690" 19:51:30 INFO [http_cmd]: http status code OK 19:51:30 DEBUG --- status code --- 19:51:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:30 DEBUG --- http response --- 19:51:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552690" 19:51:30 INFO [http_cmd]: http status code OK 19:51:30 ERROR [http_cmd]: ERROR 19:51:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:51:30 DEBUG --- status code --- 19:51:30 ERROR --- status code --- 19:51:30 DEBUG ----- output ----- 19:51:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:30 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:51:30 DEBUG { 19:51:30 DEBUG --- http response --- 19:51:30 ERROR --- http response --- 19:51:30 DEBUG "status": "success", 19:51:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:51:30 DEBUG "data": { 19:51:30 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:51:30 DEBUG ----- output ----- 19:51:30 DEBUG "resultType": "vector", 19:51:30 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:51:30 DEBUG { 19:51:30 DEBUG "result": [ 19:51:30 DEBUG "status": "success", 19:51:30 DEBUG { 19:51:30 DEBUG "data": { 19:51:30 DEBUG "metric": { 19:51:30 DEBUG "resultType": "vector", 19:51:30 DEBUG "pod": "ds-idrepo-0" 19:51:30 DEBUG "result": [ 19:51:30 DEBUG }, 19:51:30 DEBUG { 19:51:30 DEBUG "value": [ 19:51:30 DEBUG "metric": { 19:51:30 DEBUG 19:51:30 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:51:30 DEBUG 19:51:30 DEBUG }, 19:51:30 DEBUG 19:51:30 DEBUG "value": [ 19:51:30 DEBUG [...] 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG [...] 19:51:30 DEBUG "pod": "ds-idrepo-2" 19:51:30 DEBUG 19:51:30 DEBUG }, 19:51:30 DEBUG 19:51:30 DEBUG "value": [ 19:51:30 DEBUG 19:51:30 DEBUG 1713552690, 19:51:30 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 19:51:30 DEBUG "0" 19:51:30 DEBUG }, 19:51:30 DEBUG ] 19:51:30 DEBUG "value": [ 19:51:30 DEBUG } 19:51:30 DEBUG 1713552690, 19:51:30 DEBUG ] 19:51:30 DEBUG "0" 19:51:30 DEBUG } 19:51:30 DEBUG ] 19:51:30 DEBUG } 19:51:30 DEBUG } 19:51:30 DEBUG ] 19:51:30 DEBUG } 19:51:30 DEBUG } 19:51:30 INFO [http_cmd]: http status code OK 19:51:30 DEBUG --- status code --- 19:51:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:30 DEBUG --- http response --- 19:51:30 INFO [http_cmd]: http status code OK 19:51:30 INFO [http_cmd]: http status code OK 19:51:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:51:30 DEBUG --- status code --- 19:51:30 DEBUG --- status code --- 19:51:30 DEBUG ----- output ----- 19:51:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:30 DEBUG { 19:51:30 DEBUG --- http response --- 19:51:30 DEBUG --- http response --- 19:51:30 DEBUG "status": "success", 19:51:30 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:51:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:51:30 DEBUG "data": { 19:51:30 INFO [http_cmd]: http status code OK 19:51:30 DEBUG ----- output ----- 19:51:30 DEBUG "resultType": "vector", 19:51:30 DEBUG --- status code --- 19:51:30 DEBUG { 19:51:30 DEBUG "result": [ 19:51:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:30 DEBUG "status": "success", 19:51:30 DEBUG { 19:51:30 DEBUG --- http response --- 19:51:30 DEBUG "data": { 19:51:30 DEBUG "metric": { 19:51:30 DEBUG "resultType": "vector", 19:51:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:51:30 DEBUG "pod": "ds-idrepo-0" 19:51:30 DEBUG "result": [ 19:51:30 DEBUG ----- output ----- 19:51:30 DEBUG }, 19:51:30 DEBUG { 19:51:30 DEBUG { 19:51:30 DEBUG "value": [ 19:51:30 DEBUG "metric": { 19:51:30 DEBUG "status": "success", 19:51:30 DEBUG 19:51:30 DEBUG "pod": "ds-cts-0" 19:51:30 DEBUG "data": { 19:51:30 DEBUG 19:51:30 DEBUG }, 19:51:30 DEBUG "resultType": "vector", 19:51:30 DEBUG 19:51:30 DEBUG "value": [ 19:51:30 DEBUG "result": [ 19:51:30 DEBUG [...] 19:51:30 DEBUG 19:51:30 DEBUG { 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG "metric": { 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:51:30 DEBUG 19:51:30 DEBUG [...] 19:51:30 DEBUG }, 19:51:30 DEBUG "pod": "ds-idrepo-2" 19:51:30 DEBUG 19:51:30 DEBUG "value": [ 19:51:30 DEBUG }, 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG "value": [ 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG 1713552690, 19:51:30 DEBUG "pod": "ds-cts-2" 19:51:30 DEBUG 19:51:30 DEBUG "0" 19:51:30 DEBUG }, 19:51:30 DEBUG [...] 19:51:30 DEBUG ] 19:51:30 DEBUG "value": [ 19:51:30 DEBUG 19:51:30 DEBUG } 19:51:30 DEBUG 1713552690, 19:51:30 DEBUG 19:51:30 DEBUG ] 19:51:30 DEBUG "0.0004908755555555555" 19:51:30 DEBUG 19:51:30 DEBUG } 19:51:30 DEBUG ] 19:51:30 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:51:30 DEBUG } 19:51:30 DEBUG } 19:51:30 INFO [http_cmd]: http status code OK 19:51:30 DEBUG }, 19:51:30 DEBUG ] 19:51:30 DEBUG --- status code --- 19:51:30 DEBUG "value": [ 19:51:30 DEBUG } 19:51:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:30 DEBUG 1713552690, 19:51:30 DEBUG } 19:51:30 DEBUG --- http response --- 19:51:30 DEBUG "8.033004480209733" 19:51:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:51:30 DEBUG ] 19:51:30 DEBUG ----- output ----- 19:51:30 DEBUG } 19:51:30 DEBUG { 19:51:30 DEBUG ] 19:51:30 DEBUG "status": "success", 19:51:30 DEBUG } 19:51:30 DEBUG "data": { 19:51:30 DEBUG } 19:51:30 DEBUG "resultType": "vector", 19:51:30 DEBUG "result": [ 19:51:30 DEBUG { 19:51:30 DEBUG "metric": { 19:51:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:51:30 DEBUG }, 19:51:30 DEBUG "value": [ 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG [...] 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:51:30 DEBUG }, 19:51:30 DEBUG "value": [ 19:51:30 DEBUG 1713552690, 19:51:30 DEBUG "450391.5" 19:51:30 DEBUG ] 19:51:30 DEBUG } 19:51:30 DEBUG ] 19:51:30 DEBUG } 19:51:30 DEBUG } 19:51:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552690" 19:51:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713552690" 19:51:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552690" 19:51:30 INFO [http_cmd]: http status code OK 19:51:30 DEBUG --- status code --- 19:51:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:30 DEBUG --- http response --- 19:51:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:51:30 DEBUG ----- output ----- 19:51:30 DEBUG { 19:51:30 DEBUG "status": "success", 19:51:30 DEBUG "data": { 19:51:30 DEBUG "resultType": "vector", 19:51:30 DEBUG "result": [ 19:51:30 DEBUG { 19:51:30 DEBUG "metric": { 19:51:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:51:30 DEBUG }, 19:51:30 DEBUG "value": [ 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG [...] 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:51:30 DEBUG }, 19:51:30 DEBUG "value": [ 19:51:30 DEBUG 1713552690, 19:51:30 DEBUG "0" 19:51:30 DEBUG ] 19:51:30 DEBUG } 19:51:30 DEBUG ] 19:51:30 DEBUG } 19:51:30 DEBUG } 19:51:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713552690" 19:51:30 INFO [http_cmd]: http status code OK 19:51:30 INFO [http_cmd]: http status code OK 19:51:30 DEBUG --- status code --- 19:51:30 DEBUG --- status code --- 19:51:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:30 DEBUG --- http response --- 19:51:30 DEBUG --- http response --- 19:51:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:51:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:51:30 DEBUG ----- output ----- 19:51:30 DEBUG ----- output ----- 19:51:30 DEBUG { 19:51:30 DEBUG { 19:51:30 DEBUG "status": "success", 19:51:30 DEBUG "status": "success", 19:51:30 DEBUG "data": { 19:51:30 DEBUG "data": { 19:51:30 DEBUG "resultType": "vector", 19:51:30 DEBUG "resultType": "vector", 19:51:30 DEBUG "result": [ 19:51:30 DEBUG "result": [ 19:51:30 DEBUG { 19:51:30 DEBUG { 19:51:30 DEBUG "metric": { 19:51:30 DEBUG "metric": { 19:51:30 DEBUG "pod": "ds-idrepo-0" 19:51:30 DEBUG "instance": "10.142.0.107:9100" 19:51:30 DEBUG }, 19:51:30 DEBUG }, 19:51:30 DEBUG "value": [ 19:51:30 DEBUG "value": [ 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG [...] 19:51:30 DEBUG [...] 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG "pod": "ds-cts-2" 19:51:30 DEBUG "instance": "10.142.0.43:9100" 19:51:30 DEBUG }, 19:51:30 DEBUG }, 19:51:30 DEBUG "value": [ 19:51:30 DEBUG "value": [ 19:51:30 DEBUG 1713552690, 19:51:30 DEBUG 1713552690, 19:51:30 DEBUG "0" 19:51:30 DEBUG "0.16969999999989605" 19:51:30 DEBUG ] 19:51:30 DEBUG ] 19:51:30 DEBUG } 19:51:30 DEBUG } 19:51:30 DEBUG ] 19:51:30 DEBUG ] 19:51:30 DEBUG } 19:51:30 DEBUG } 19:51:30 DEBUG } 19:51:30 DEBUG } 19:51:30 INFO [http_cmd]: http status code OK 19:51:30 DEBUG --- status code --- 19:51:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:30 DEBUG --- http response --- 19:51:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:51:30 DEBUG ----- output ----- 19:51:30 DEBUG { 19:51:30 DEBUG "status": "success", 19:51:30 DEBUG "data": { 19:51:30 DEBUG "resultType": "vector", 19:51:30 DEBUG "result": [ 19:51:30 DEBUG { 19:51:30 DEBUG "metric": { 19:51:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:51:30 DEBUG }, 19:51:30 DEBUG "value": [ 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG [...] 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:51:30 DEBUG }, 19:51:30 DEBUG "value": [ 19:51:30 DEBUG 1713552690, 19:51:30 DEBUG "0.014" 19:51:30 DEBUG ] 19:51:30 DEBUG } 19:51:30 DEBUG ] 19:51:30 DEBUG } 19:51:30 DEBUG } 19:51:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552690" 19:51:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552690" 19:51:30 INFO [http_cmd]: http status code OK 19:51:30 DEBUG --- status code --- 19:51:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:30 DEBUG --- http response --- 19:51:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:51:30 DEBUG ----- output ----- 19:51:30 DEBUG { 19:51:30 DEBUG "status": "success", 19:51:30 DEBUG "data": { 19:51:30 DEBUG "resultType": "vector", 19:51:30 DEBUG "result": [ 19:51:30 DEBUG { 19:51:30 DEBUG "metric": { 19:51:30 DEBUG "pod": "ds-cts-0" 19:51:30 DEBUG }, 19:51:30 DEBUG "value": [ 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG [...] 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG "pod": "ds-cts-2" 19:51:30 DEBUG }, 19:51:30 DEBUG "value": [ 19:51:30 DEBUG 1713552690, 19:51:30 DEBUG "29.833333333333332" 19:51:30 DEBUG ] 19:51:30 DEBUG } 19:51:30 DEBUG ] 19:51:30 DEBUG } 19:51:30 DEBUG } 19:51:30 INFO [http_cmd]: http status code OK 19:51:30 DEBUG --- status code --- 19:51:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:30 DEBUG --- http response --- 19:51:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:51:30 DEBUG ----- output ----- 19:51:30 DEBUG { 19:51:30 DEBUG "status": "success", 19:51:30 DEBUG "data": { 19:51:30 DEBUG "resultType": "vector", 19:51:30 DEBUG "result": [ 19:51:30 DEBUG { 19:51:30 DEBUG "metric": { 19:51:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:51:30 DEBUG }, 19:51:30 DEBUG "value": [ 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG [...] 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:51:30 DEBUG }, 19:51:30 DEBUG "value": [ 19:51:30 DEBUG 1713552690, 19:51:30 DEBUG "2506.8333333333335" 19:51:30 DEBUG ] 19:51:30 DEBUG } 19:51:30 DEBUG ] 19:51:30 DEBUG } 19:51:30 DEBUG } 19:51:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552690" 19:51:30 INFO [http_cmd]: http status code OK 19:51:30 DEBUG --- status code --- 19:51:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:30 DEBUG --- http response --- 19:51:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:51:30 DEBUG ----- output ----- 19:51:30 DEBUG { 19:51:30 DEBUG "status": "success", 19:51:30 DEBUG "data": { 19:51:30 DEBUG "resultType": "vector", 19:51:30 DEBUG "result": [ 19:51:30 DEBUG { 19:51:30 DEBUG "metric": { 19:51:30 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:51:30 DEBUG }, 19:51:30 DEBUG "value": [ 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG [...] 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 19:51:30 DEBUG }, 19:51:30 DEBUG "value": [ 19:51:30 DEBUG 1713552690, 19:51:30 DEBUG "0" 19:51:30 DEBUG ] 19:51:30 DEBUG } 19:51:30 DEBUG ] 19:51:30 DEBUG } 19:51:30 DEBUG } 19:51:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552690" 19:51:30 INFO [http_cmd]: http status code OK 19:51:30 DEBUG --- status code --- 19:51:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:30 DEBUG --- http response --- 19:51:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:51:30 DEBUG ----- output ----- 19:51:30 DEBUG { 19:51:30 DEBUG "status": "success", 19:51:30 DEBUG "data": { 19:51:30 DEBUG "resultType": "vector", 19:51:30 DEBUG "result": [ 19:51:30 DEBUG { 19:51:30 DEBUG "metric": { 19:51:30 DEBUG "pod": "ds-idrepo-0" 19:51:30 DEBUG }, 19:51:30 DEBUG "value": [ 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG [...] 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG "pod": "ds-idrepo-2" 19:51:30 DEBUG }, 19:51:30 DEBUG "value": [ 19:51:30 DEBUG 1713552690, 19:51:30 DEBUG "0" 19:51:30 DEBUG ] 19:51:30 DEBUG } 19:51:30 DEBUG ] 19:51:30 DEBUG } 19:51:30 DEBUG } 19:51:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552690" 19:51:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552690" 19:51:30 INFO [http_cmd]: http status code OK 19:51:30 DEBUG --- status code --- 19:51:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:30 DEBUG --- http response --- 19:51:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:51:30 DEBUG ----- output ----- 19:51:30 DEBUG { 19:51:30 DEBUG "status": "success", 19:51:30 DEBUG "data": { 19:51:30 DEBUG "resultType": "vector", 19:51:30 DEBUG "result": [ 19:51:30 DEBUG { 19:51:30 DEBUG "metric": { 19:51:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:51:30 DEBUG }, 19:51:30 DEBUG "value": [ 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG [...] 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:51:30 DEBUG }, 19:51:30 DEBUG "value": [ 19:51:30 DEBUG 1713552690, 19:51:30 DEBUG "2500.7" 19:51:30 DEBUG ] 19:51:30 DEBUG } 19:51:30 DEBUG ] 19:51:30 DEBUG } 19:51:30 DEBUG } 19:51:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552690" 19:51:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552690" 19:51:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552690" 19:51:30 INFO [http_cmd]: http status code OK 19:51:30 DEBUG --- status code --- 19:51:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:30 DEBUG --- http response --- 19:51:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552690" 19:51:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:51:30 DEBUG ----- output ----- 19:51:30 DEBUG { 19:51:30 DEBUG "status": "success", 19:51:30 DEBUG "data": { 19:51:30 DEBUG "resultType": "vector", 19:51:30 DEBUG "result": [ 19:51:30 DEBUG { 19:51:30 DEBUG "metric": { 19:51:30 DEBUG "instance": "10.142.0.107:9100" 19:51:30 DEBUG }, 19:51:30 DEBUG "value": [ 19:51:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552690" 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG [...] 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG "instance": "10.142.0.43:9100" 19:51:30 DEBUG }, 19:51:30 DEBUG "value": [ 19:51:30 DEBUG 1713552690, 19:51:30 DEBUG "0.0726666666666612" 19:51:30 DEBUG ] 19:51:30 DEBUG } 19:51:30 DEBUG ] 19:51:30 DEBUG } 19:51:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552690" 19:51:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552690" 19:51:30 DEBUG } 19:51:30 INFO [http_cmd]: http status code OK 19:51:30 DEBUG --- status code --- 19:51:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:30 DEBUG --- http response --- 19:51:30 INFO [http_cmd]: http status code OK 19:51:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:51:30 DEBUG --- status code --- 19:51:30 DEBUG ----- output ----- 19:51:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:30 DEBUG { 19:51:30 DEBUG --- http response --- 19:51:30 DEBUG "status": "success", 19:51:30 DEBUG "data": { 19:51:30 DEBUG "resultType": "vector", 19:51:30 DEBUG "result": [ 19:51:30 DEBUG { 19:51:30 DEBUG "metric": { 19:51:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:51:30 DEBUG }, 19:51:30 DEBUG "value": [ 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG [...] 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:51:30 DEBUG }, 19:51:30 DEBUG "value": [ 19:51:30 DEBUG 1713552690, 19:51:30 DEBUG "2506.5666666666666" 19:51:30 DEBUG ] 19:51:30 DEBUG } 19:51:30 DEBUG ] 19:51:30 DEBUG } 19:51:30 DEBUG } 19:51:30 INFO [http_cmd]: http status code OK 19:51:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:51:30 DEBUG --- status code --- 19:51:30 INFO [http_cmd]: http status code OK 19:51:30 DEBUG ----- output ----- 19:51:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:30 DEBUG --- status code --- 19:51:30 DEBUG { 19:51:30 DEBUG --- http response --- 19:51:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:30 DEBUG "status": "success", 19:51:30 DEBUG --- http response --- 19:51:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:51:30 DEBUG "data": { 19:51:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:51:30 INFO [http_cmd]: http status code OK 19:51:30 DEBUG ----- output ----- 19:51:30 INFO [http_cmd]: http status code OK 19:51:30 DEBUG "resultType": "vector", 19:51:30 DEBUG ----- output ----- 19:51:30 DEBUG --- status code --- 19:51:30 DEBUG { 19:51:30 DEBUG --- status code --- 19:51:30 DEBUG "result": [ 19:51:30 DEBUG { 19:51:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:30 DEBUG "status": "success", 19:51:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:30 DEBUG { 19:51:30 DEBUG "status": "success", 19:51:30 DEBUG --- http response --- 19:51:30 DEBUG "data": { 19:51:30 DEBUG --- http response --- 19:51:30 DEBUG "metric": { 19:51:30 DEBUG "data": { 19:51:30 DEBUG "resultType": "vector", 19:51:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:51:30 DEBUG "instance": "10.142.0.107:9100" 19:51:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:51:30 DEBUG "resultType": "vector", 19:51:30 INFO [http_cmd]: http status code OK 19:51:30 DEBUG "result": [ 19:51:30 DEBUG ----- output ----- 19:51:30 DEBUG }, 19:51:30 DEBUG ----- output ----- 19:51:30 DEBUG "result": [ 19:51:30 DEBUG --- status code --- 19:51:30 DEBUG { 19:51:30 DEBUG { 19:51:30 DEBUG "value": [ 19:51:30 DEBUG { 19:51:30 DEBUG { 19:51:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:30 DEBUG "metric": { 19:51:30 DEBUG "status": "success", 19:51:30 DEBUG 19:51:30 DEBUG "status": "success", 19:51:30 DEBUG "metric": { 19:51:30 DEBUG --- http response --- 19:51:30 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:51:30 DEBUG "data": { 19:51:30 DEBUG 19:51:30 DEBUG "data": { 19:51:30 DEBUG "pod": "ds-cts-0" 19:51:30 DEBUG }, 19:51:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:51:30 DEBUG "resultType": "vector", 19:51:30 DEBUG 19:51:30 DEBUG "resultType": "vector", 19:51:30 DEBUG }, 19:51:30 DEBUG "value": [ 19:51:30 DEBUG ----- output ----- 19:51:30 DEBUG "result": [ 19:51:30 DEBUG [...] 19:51:30 DEBUG "result": [ 19:51:30 DEBUG "value": [ 19:51:30 DEBUG 19:51:30 DEBUG { 19:51:30 DEBUG { 19:51:30 DEBUG 19:51:30 DEBUG { 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG "status": "success", 19:51:30 DEBUG "metric": { 19:51:30 DEBUG 19:51:30 DEBUG "metric": { 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG "data": { 19:51:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:51:30 DEBUG 19:51:30 DEBUG "instance": "10.142.0.107:9100" 19:51:30 DEBUG 19:51:30 DEBUG [...] 19:51:30 DEBUG "resultType": "vector", 19:51:30 DEBUG }, 19:51:30 DEBUG "instance": "10.142.0.43:9100" 19:51:30 DEBUG }, 19:51:30 DEBUG [...] 19:51:30 DEBUG 19:51:30 DEBUG "result": [ 19:51:30 DEBUG "value": [ 19:51:30 DEBUG }, 19:51:30 DEBUG "value": [ 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG { 19:51:30 DEBUG 19:51:30 DEBUG "value": [ 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG "metric": { 19:51:30 DEBUG 19:51:30 DEBUG 1713552690, 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 19:51:30 DEBUG "instance": "10.142.0.107:9100" 19:51:30 DEBUG 19:51:30 DEBUG "1964847250432" 19:51:30 DEBUG 19:51:30 DEBUG "pod": "ds-cts-2" 19:51:30 DEBUG }, 19:51:30 DEBUG }, 19:51:30 DEBUG [...] 19:51:30 DEBUG ] 19:51:30 DEBUG [...] 19:51:30 DEBUG }, 19:51:30 DEBUG "value": [ 19:51:30 DEBUG "value": [ 19:51:30 DEBUG 19:51:30 DEBUG } 19:51:30 DEBUG 19:51:30 DEBUG "value": [ 19:51:30 DEBUG 1713552690, 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG ] 19:51:30 DEBUG 19:51:30 DEBUG 1713552690, 19:51:30 DEBUG "0" 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG } 19:51:30 DEBUG 19:51:30 DEBUG "0" 19:51:30 DEBUG ] 19:51:30 DEBUG 19:51:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:51:30 DEBUG } 19:51:30 DEBUG "instance": "10.142.0.43:9100" 19:51:30 DEBUG ] 19:51:30 DEBUG } 19:51:30 DEBUG [...] 19:51:30 DEBUG }, 19:51:30 DEBUG }, 19:51:30 DEBUG } 19:51:30 DEBUG ] 19:51:30 DEBUG 19:51:30 DEBUG "value": [ 19:51:30 DEBUG "value": [ 19:51:30 DEBUG ] 19:51:30 DEBUG } 19:51:30 DEBUG 19:51:30 DEBUG 1713552690, 19:51:30 DEBUG 1713552690, 19:51:30 DEBUG } 19:51:30 DEBUG } 19:51:30 DEBUG 19:51:30 DEBUG "2506.6" 19:51:30 DEBUG "75727684.26666667" 19:51:30 DEBUG } 19:51:30 DEBUG "instance": "10.142.0.43:9100" 19:51:30 DEBUG ] 19:51:30 DEBUG ] 19:51:30 DEBUG }, 19:51:30 DEBUG } 19:51:30 DEBUG } 19:51:30 DEBUG "value": [ 19:51:30 DEBUG ] 19:51:30 DEBUG ] 19:51:30 DEBUG 1713552690, 19:51:30 DEBUG } 19:51:30 DEBUG } 19:51:30 DEBUG "75730084.8" 19:51:30 DEBUG } 19:51:30 DEBUG } 19:51:30 DEBUG ] 19:51:30 DEBUG } 19:51:30 DEBUG ] 19:51:30 DEBUG } 19:51:30 DEBUG } 19:51:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552690" 19:51:30 INFO [http_cmd]: http status code OK 19:51:30 DEBUG --- status code --- 19:51:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:30 DEBUG --- http response --- 19:51:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:51:30 DEBUG ----- output ----- 19:51:30 DEBUG { 19:51:30 DEBUG "status": "success", 19:51:30 DEBUG "data": { 19:51:30 DEBUG "resultType": "vector", 19:51:30 DEBUG "result": [ 19:51:30 DEBUG { 19:51:30 DEBUG "metric": { 19:51:30 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:51:30 DEBUG }, 19:51:30 DEBUG "value": [ 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG [...] 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 19:51:30 DEBUG }, 19:51:30 DEBUG "value": [ 19:51:30 DEBUG 1713552690, 19:51:30 DEBUG "0.00003480883947938134" 19:51:30 DEBUG ] 19:51:30 DEBUG } 19:51:30 DEBUG ] 19:51:30 DEBUG } 19:51:30 DEBUG } 19:51:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713552690" 19:51:30 INFO [http_cmd]: http status code OK 19:51:30 DEBUG --- status code --- 19:51:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:30 DEBUG --- http response --- 19:51:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:51:30 DEBUG ----- output ----- 19:51:30 DEBUG { 19:51:30 DEBUG "status": "success", 19:51:30 DEBUG "data": { 19:51:30 DEBUG "resultType": "vector", 19:51:30 DEBUG "result": [ 19:51:30 DEBUG { 19:51:30 DEBUG "metric": { 19:51:30 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:51:30 DEBUG }, 19:51:30 DEBUG "value": [ 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG [...] 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG 19:51:30 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:51:30 DEBUG }, 19:51:30 DEBUG "value": [ 19:51:30 DEBUG 1713552690, 19:51:30 DEBUG "3477553152" 19:51:30 DEBUG ] 19:51:30 DEBUG } 19:51:30 DEBUG ] 19:51:30 DEBUG } 19:51:30 DEBUG } 19:51:31 INFO 19:51:31 INFO [loop_until]: kubectl --namespace=xlou top pods 19:51:31 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:51:31 INFO [loop_until]: OK (rc = 0) 19:51:31 DEBUG --- stdout --- 19:51:31 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 15834m 9006Mi am-767f6b67b5-qcqz2 16524m 8746Mi am-767f6b67b5-vbltz 17655m 8996Mi ds-cts-0 17821m 10947Mi ds-cts-1 15658m 11206Mi ds-cts-2 17062m 10946Mi ds-idrepo-0 7199m 3666Mi ds-idrepo-1 7201m 3537Mi ds-idrepo-2 7430m 3492Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 4m 1217Mi lodemon-f8459c597-6tj6f 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8053m 3222Mi 19:51:31 DEBUG --- stderr --- 19:51:31 DEBUG 19:51:32 INFO 19:51:32 INFO [loop_until]: kubectl --namespace=xlou top node 19:51:32 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:51:32 INFO [loop_until]: OK (rc = 0) 19:51:32 DEBUG --- stdout --- 19:51:32 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 70m 0% 6250Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7418m 24% 7262Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16165m 54% 11708Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 82m 0% 3345Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 16577m 55% 13921Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 154m 0% 5220Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 16366m 54% 11638Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15599m 52% 2982Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 18121m 60% 11211Mi 10% gke-xlou-cdm-ds-0f601751-j488 7622m 25% 6739Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 15069m 50% 15952Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 7366m 24% 7437Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 17239m 57% 13399Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 8205m 51% 6983Mi 11% 19:51:32 DEBUG --- stderr --- 19:51:32 DEBUG 19:52:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552750" 19:52:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552750" 19:52:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713552750" 19:52:30 ERROR [http_cmd]: ERROR 19:52:30 INFO [http_cmd]: http status code OK 19:52:30 ERROR --- status code --- 19:52:30 DEBUG --- status code --- 19:52:30 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:52:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:30 ERROR --- http response --- 19:52:30 DEBUG --- http response --- 19:52:30 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:52:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:52:30 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:52:30 DEBUG ----- output ----- 19:52:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552750" 19:52:30 DEBUG { 19:52:30 DEBUG "status": "success", 19:52:30 DEBUG "data": { 19:52:30 DEBUG "resultType": "vector", 19:52:30 DEBUG "result": [ 19:52:30 DEBUG { 19:52:30 DEBUG "metric": { 19:52:30 DEBUG "pod": "ds-cts-0" 19:52:30 DEBUG }, 19:52:30 DEBUG "value": [ 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG [...] 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG "pod": "ds-cts-2" 19:52:30 DEBUG }, 19:52:30 DEBUG "value": [ 19:52:30 DEBUG 1713552750, 19:52:30 DEBUG "0.00065596" 19:52:30 DEBUG ] 19:52:30 DEBUG } 19:52:30 DEBUG ] 19:52:30 DEBUG } 19:52:30 DEBUG } 19:52:30 INFO [http_cmd]: http status code OK 19:52:30 DEBUG --- status code --- 19:52:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:30 DEBUG --- http response --- 19:52:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:52:30 DEBUG ----- output ----- 19:52:30 DEBUG { 19:52:30 DEBUG "status": "success", 19:52:30 DEBUG "data": { 19:52:30 DEBUG "resultType": "vector", 19:52:30 DEBUG "result": [ 19:52:30 DEBUG { 19:52:30 DEBUG "metric": { 19:52:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:52:30 DEBUG }, 19:52:30 DEBUG "value": [ 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG [...] 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:52:30 DEBUG }, 19:52:30 DEBUG "value": [ 19:52:30 DEBUG 1713552750, 19:52:30 DEBUG "450211.5" 19:52:30 DEBUG ] 19:52:30 DEBUG } 19:52:30 DEBUG ] 19:52:30 DEBUG } 19:52:30 DEBUG } 19:52:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552750" 19:52:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552750" 19:52:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552750" 19:52:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713552750" 19:52:30 INFO [http_cmd]: http status code OK 19:52:30 DEBUG --- status code --- 19:52:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:30 DEBUG --- http response --- 19:52:30 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:52:30 INFO [http_cmd]: http status code OK 19:52:30 DEBUG --- status code --- 19:52:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:30 DEBUG --- http response --- 19:52:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:52:30 DEBUG ----- output ----- 19:52:30 DEBUG { 19:52:30 DEBUG "status": "success", 19:52:30 DEBUG "data": { 19:52:30 DEBUG "resultType": "vector", 19:52:30 DEBUG "result": [ 19:52:30 DEBUG { 19:52:30 DEBUG "metric": { 19:52:30 DEBUG "pod": "ds-idrepo-0" 19:52:30 INFO [http_cmd]: http status code OK 19:52:30 DEBUG }, 19:52:30 DEBUG --- status code --- 19:52:30 DEBUG "value": [ 19:52:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:30 DEBUG 19:52:30 DEBUG --- http response --- 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG [...] 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG "pod": "ds-idrepo-2" 19:52:30 DEBUG }, 19:52:30 DEBUG "value": [ 19:52:30 DEBUG 1713552750, 19:52:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:52:30 DEBUG "0" 19:52:30 DEBUG ----- output ----- 19:52:30 DEBUG ] 19:52:30 DEBUG { 19:52:30 DEBUG } 19:52:30 DEBUG "status": "success", 19:52:30 DEBUG ] 19:52:30 DEBUG "data": { 19:52:30 DEBUG } 19:52:30 DEBUG "resultType": "vector", 19:52:30 DEBUG } 19:52:30 DEBUG "result": [ 19:52:30 DEBUG { 19:52:30 DEBUG "metric": { 19:52:30 DEBUG "pod": "ds-idrepo-0" 19:52:30 DEBUG }, 19:52:30 DEBUG "value": [ 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG [...] 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 19:52:30 DEBUG }, 19:52:30 DEBUG "value": [ 19:52:30 DEBUG 1713552750, 19:52:30 DEBUG "0" 19:52:30 DEBUG ] 19:52:30 DEBUG } 19:52:30 DEBUG ] 19:52:30 DEBUG } 19:52:30 DEBUG } 19:52:30 INFO [http_cmd]: http status code OK 19:52:30 DEBUG --- status code --- 19:52:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:30 DEBUG --- http response --- 19:52:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:52:30 DEBUG ----- output ----- 19:52:30 DEBUG { 19:52:30 DEBUG "status": "success", 19:52:30 DEBUG "data": { 19:52:30 DEBUG "resultType": "vector", 19:52:30 DEBUG "result": [ 19:52:30 DEBUG { 19:52:30 DEBUG "metric": { 19:52:30 DEBUG "pod": "ds-idrepo-0" 19:52:30 DEBUG }, 19:52:30 DEBUG "value": [ 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG [...] 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG "pod": "ds-idrepo-2" 19:52:30 DEBUG }, 19:52:30 DEBUG "value": [ 19:52:30 DEBUG 1713552750, 19:52:30 DEBUG "0" 19:52:30 DEBUG ] 19:52:30 DEBUG } 19:52:30 DEBUG ] 19:52:30 DEBUG } 19:52:30 DEBUG } 19:52:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713552750" 19:52:30 INFO [http_cmd]: http status code OK 19:52:30 DEBUG --- status code --- 19:52:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:30 DEBUG --- http response --- 19:52:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:52:30 DEBUG ----- output ----- 19:52:30 DEBUG { 19:52:30 DEBUG "status": "success", 19:52:30 DEBUG "data": { 19:52:30 DEBUG "resultType": "vector", 19:52:30 DEBUG "result": [ 19:52:30 DEBUG { 19:52:30 DEBUG "metric": { 19:52:30 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:52:30 DEBUG }, 19:52:30 DEBUG "value": [ 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG [...] 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:52:30 DEBUG }, 19:52:30 DEBUG "value": [ 19:52:30 DEBUG 1713552750, 19:52:30 DEBUG "8.035745111034332" 19:52:30 DEBUG ] 19:52:30 DEBUG } 19:52:30 DEBUG ] 19:52:30 DEBUG } 19:52:30 DEBUG } 19:52:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552750" 19:52:30 INFO [http_cmd]: http status code OK 19:52:30 DEBUG --- status code --- 19:52:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:30 DEBUG --- http response --- 19:52:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:52:30 DEBUG ----- output ----- 19:52:30 DEBUG { 19:52:30 DEBUG "status": "success", 19:52:30 DEBUG "data": { 19:52:30 DEBUG "resultType": "vector", 19:52:30 DEBUG "result": [ 19:52:30 DEBUG { 19:52:30 DEBUG "metric": { 19:52:30 DEBUG "pod": "ds-idrepo-0" 19:52:30 DEBUG }, 19:52:30 DEBUG "value": [ 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG [...] 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG "pod": "ds-cts-2" 19:52:30 DEBUG }, 19:52:30 DEBUG "value": [ 19:52:30 DEBUG 1713552750, 19:52:30 DEBUG "0" 19:52:30 DEBUG ] 19:52:30 DEBUG } 19:52:30 DEBUG ] 19:52:30 DEBUG } 19:52:30 DEBUG } 19:52:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552750" 19:52:30 INFO [http_cmd]: http status code OK 19:52:30 DEBUG --- status code --- 19:52:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:30 DEBUG --- http response --- 19:52:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:52:30 DEBUG ----- output ----- 19:52:30 DEBUG { 19:52:30 DEBUG "status": "success", 19:52:30 DEBUG "data": { 19:52:30 DEBUG "resultType": "vector", 19:52:30 DEBUG "result": [ 19:52:30 DEBUG { 19:52:30 DEBUG "metric": { 19:52:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:52:30 DEBUG }, 19:52:30 DEBUG "value": [ 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG [...] 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:52:30 DEBUG }, 19:52:30 DEBUG "value": [ 19:52:30 DEBUG 1713552750, 19:52:30 DEBUG "0" 19:52:30 DEBUG ] 19:52:30 DEBUG } 19:52:30 DEBUG ] 19:52:30 DEBUG } 19:52:30 DEBUG } 19:52:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713552750" 19:52:30 INFO [http_cmd]: http status code OK 19:52:30 DEBUG --- status code --- 19:52:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:30 DEBUG --- http response --- 19:52:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:52:30 DEBUG ----- output ----- 19:52:30 DEBUG { 19:52:30 DEBUG "status": "success", 19:52:30 DEBUG "data": { 19:52:30 DEBUG "resultType": "vector", 19:52:30 DEBUG "result": [ 19:52:30 DEBUG { 19:52:30 DEBUG "metric": { 19:52:30 DEBUG "instance": "10.142.0.107:9100" 19:52:30 DEBUG }, 19:52:30 DEBUG "value": [ 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG [...] 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG "instance": "10.142.0.43:9100" 19:52:30 DEBUG }, 19:52:30 DEBUG "value": [ 19:52:30 DEBUG 1713552750, 19:52:30 DEBUG "0.170433333333285" 19:52:30 DEBUG ] 19:52:30 DEBUG } 19:52:30 DEBUG ] 19:52:30 DEBUG } 19:52:30 DEBUG } 19:52:30 INFO [http_cmd]: http status code OK 19:52:30 DEBUG --- status code --- 19:52:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:30 DEBUG --- http response --- 19:52:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:52:30 DEBUG ----- output ----- 19:52:30 DEBUG { 19:52:30 DEBUG "status": "success", 19:52:30 DEBUG "data": { 19:52:30 DEBUG "resultType": "vector", 19:52:30 DEBUG "result": [ 19:52:30 DEBUG { 19:52:30 DEBUG "metric": { 19:52:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552750" 19:52:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:52:30 DEBUG }, 19:52:30 DEBUG "value": [ 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG [...] 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:52:30 DEBUG }, 19:52:30 DEBUG "value": [ 19:52:30 DEBUG 1713552750, 19:52:30 DEBUG "0.014" 19:52:30 DEBUG ] 19:52:30 DEBUG } 19:52:30 DEBUG ] 19:52:30 DEBUG } 19:52:30 DEBUG } 19:52:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552750" 19:52:30 INFO [http_cmd]: http status code OK 19:52:30 DEBUG --- status code --- 19:52:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:30 DEBUG --- http response --- 19:52:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552750" 19:52:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:52:30 DEBUG ----- output ----- 19:52:30 DEBUG { 19:52:30 DEBUG "status": "success", 19:52:30 DEBUG "data": { 19:52:30 DEBUG "resultType": "vector", 19:52:30 DEBUG "result": [ 19:52:30 DEBUG { 19:52:30 DEBUG "metric": { 19:52:30 DEBUG "pod": "ds-idrepo-0" 19:52:30 DEBUG }, 19:52:30 DEBUG "value": [ 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG [...] 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG "pod": "ds-idrepo-2" 19:52:30 DEBUG }, 19:52:30 DEBUG "value": [ 19:52:30 DEBUG 1713552750, 19:52:30 DEBUG "0" 19:52:30 DEBUG ] 19:52:30 DEBUG } 19:52:30 DEBUG ] 19:52:30 DEBUG } 19:52:30 DEBUG } 19:52:30 INFO [http_cmd]: http status code OK 19:52:30 DEBUG --- status code --- 19:52:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:30 DEBUG --- http response --- 19:52:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:52:30 DEBUG ----- output ----- 19:52:30 DEBUG { 19:52:30 DEBUG "status": "success", 19:52:30 DEBUG "data": { 19:52:30 DEBUG "resultType": "vector", 19:52:30 DEBUG "result": [ 19:52:30 DEBUG { 19:52:30 DEBUG "metric": { 19:52:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:52:30 DEBUG }, 19:52:30 DEBUG "value": [ 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG [...] 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:52:30 DEBUG }, 19:52:30 DEBUG "value": [ 19:52:30 DEBUG 1713552750, 19:52:30 DEBUG "2491.7666666666664" 19:52:30 DEBUG ] 19:52:30 DEBUG } 19:52:30 DEBUG ] 19:52:30 DEBUG } 19:52:30 DEBUG } 19:52:30 INFO [http_cmd]: http status code OK 19:52:30 DEBUG --- status code --- 19:52:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:30 DEBUG --- http response --- 19:52:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:52:30 DEBUG ----- output ----- 19:52:30 DEBUG { 19:52:30 DEBUG "status": "success", 19:52:30 DEBUG "data": { 19:52:30 DEBUG "resultType": "vector", 19:52:30 DEBUG "result": [ 19:52:30 DEBUG { 19:52:30 DEBUG "metric": { 19:52:30 DEBUG "pod": "ds-cts-0" 19:52:30 DEBUG }, 19:52:30 DEBUG "value": [ 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG [...] 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG "pod": "ds-cts-2" 19:52:30 DEBUG }, 19:52:30 DEBUG "value": [ 19:52:30 DEBUG 1713552750, 19:52:30 DEBUG "29.833333333333332" 19:52:30 DEBUG ] 19:52:30 DEBUG } 19:52:30 DEBUG ] 19:52:30 DEBUG } 19:52:30 DEBUG } 19:52:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552750" 19:52:30 INFO [http_cmd]: http status code OK 19:52:30 DEBUG --- status code --- 19:52:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:30 DEBUG --- http response --- 19:52:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:52:30 DEBUG ----- output ----- 19:52:30 DEBUG { 19:52:30 DEBUG "status": "success", 19:52:30 DEBUG "data": { 19:52:30 DEBUG "resultType": "vector", 19:52:30 DEBUG "result": [ 19:52:30 DEBUG { 19:52:30 DEBUG "metric": { 19:52:30 DEBUG "pod": "ds-idrepo-0" 19:52:30 DEBUG }, 19:52:30 DEBUG "value": [ 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG [...] 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 19:52:30 DEBUG }, 19:52:30 DEBUG "value": [ 19:52:30 DEBUG 1713552750, 19:52:30 DEBUG "0" 19:52:30 DEBUG ] 19:52:30 DEBUG } 19:52:30 DEBUG ] 19:52:30 DEBUG } 19:52:30 DEBUG } 19:52:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552750" 19:52:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552750" 19:52:30 INFO [http_cmd]: http status code OK 19:52:30 DEBUG --- status code --- 19:52:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:30 DEBUG --- http response --- 19:52:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:52:30 DEBUG ----- output ----- 19:52:30 DEBUG { 19:52:30 DEBUG "status": "success", 19:52:30 DEBUG "data": { 19:52:30 DEBUG "resultType": "vector", 19:52:30 DEBUG "result": [ 19:52:30 DEBUG { 19:52:30 DEBUG "metric": { 19:52:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:52:30 DEBUG }, 19:52:30 DEBUG "value": [ 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG [...] 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:52:30 DEBUG }, 19:52:30 DEBUG "value": [ 19:52:30 DEBUG 1713552750, 19:52:30 DEBUG "2498.4333333333334" 19:52:30 DEBUG ] 19:52:30 DEBUG } 19:52:30 DEBUG ] 19:52:30 DEBUG } 19:52:30 DEBUG } 19:52:30 INFO [http_cmd]: http status code OK 19:52:30 DEBUG --- status code --- 19:52:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:30 DEBUG --- http response --- 19:52:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:52:30 DEBUG ----- output ----- 19:52:30 DEBUG { 19:52:30 DEBUG "status": "success", 19:52:30 DEBUG "data": { 19:52:30 DEBUG "resultType": "vector", 19:52:30 DEBUG "result": [ 19:52:30 DEBUG { 19:52:30 DEBUG "metric": { 19:52:30 DEBUG "pod": "ds-idrepo-0" 19:52:30 DEBUG }, 19:52:30 DEBUG "value": [ 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG [...] 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 19:52:30 DEBUG }, 19:52:30 DEBUG "value": [ 19:52:30 DEBUG 1713552750, 19:52:30 DEBUG "0" 19:52:30 DEBUG ] 19:52:30 DEBUG } 19:52:30 DEBUG ] 19:52:30 DEBUG } 19:52:30 DEBUG } 19:52:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552750" 19:52:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552750" 19:52:30 INFO [http_cmd]: http status code OK 19:52:30 DEBUG --- status code --- 19:52:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:30 DEBUG --- http response --- 19:52:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:52:30 DEBUG ----- output ----- 19:52:30 DEBUG { 19:52:30 DEBUG "status": "success", 19:52:30 DEBUG "data": { 19:52:30 DEBUG "resultType": "vector", 19:52:30 DEBUG "result": [ 19:52:30 DEBUG { 19:52:30 DEBUG "metric": { 19:52:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:52:30 DEBUG }, 19:52:30 DEBUG "value": [ 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG [...] 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:52:30 DEBUG }, 19:52:30 DEBUG "value": [ 19:52:30 DEBUG 1713552750, 19:52:30 DEBUG "2491.6" 19:52:30 DEBUG ] 19:52:30 DEBUG } 19:52:30 DEBUG ] 19:52:30 DEBUG } 19:52:30 DEBUG } 19:52:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552750" 19:52:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552750" 19:52:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552750" 19:52:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552750" 19:52:30 INFO [http_cmd]: http status code OK 19:52:30 DEBUG --- status code --- 19:52:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552750" 19:52:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:30 DEBUG --- http response --- 19:52:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:52:30 DEBUG ----- output ----- 19:52:30 DEBUG { 19:52:30 DEBUG "status": "success", 19:52:30 DEBUG "data": { 19:52:30 DEBUG "resultType": "vector", 19:52:30 DEBUG "result": [ 19:52:30 DEBUG { 19:52:30 DEBUG "metric": { 19:52:30 DEBUG "instance": "10.142.0.107:9100" 19:52:30 DEBUG }, 19:52:30 DEBUG "value": [ 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG [...] 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG "instance": "10.142.0.43:9100" 19:52:30 DEBUG }, 19:52:30 DEBUG "value": [ 19:52:30 DEBUG 1713552750, 19:52:30 DEBUG "0.07366666666666409" 19:52:30 DEBUG ] 19:52:30 DEBUG } 19:52:30 DEBUG ] 19:52:30 DEBUG } 19:52:30 DEBUG } 19:52:30 INFO [http_cmd]: http status code OK 19:52:30 INFO [http_cmd]: http status code OK 19:52:30 DEBUG --- status code --- 19:52:30 DEBUG --- status code --- 19:52:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:30 DEBUG --- http response --- 19:52:30 DEBUG --- http response --- 19:52:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:52:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:52:30 DEBUG ----- output ----- 19:52:30 DEBUG ----- output ----- 19:52:30 DEBUG { 19:52:30 DEBUG { 19:52:30 INFO [http_cmd]: http status code OK 19:52:30 DEBUG "status": "success", 19:52:30 DEBUG "status": "success", 19:52:30 INFO [http_cmd]: http status code OK 19:52:30 DEBUG --- status code --- 19:52:30 DEBUG "data": { 19:52:30 DEBUG "data": { 19:52:30 DEBUG --- status code --- 19:52:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:30 DEBUG "resultType": "vector", 19:52:30 DEBUG "resultType": "vector", 19:52:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:30 DEBUG --- http response --- 19:52:30 DEBUG "result": [ 19:52:30 DEBUG "result": [ 19:52:30 DEBUG --- http response --- 19:52:30 DEBUG { 19:52:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:52:30 DEBUG { 19:52:30 DEBUG "metric": { 19:52:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:52:30 DEBUG ----- output ----- 19:52:30 DEBUG "metric": { 19:52:30 DEBUG "pod": "ds-cts-0" 19:52:30 DEBUG ----- output ----- 19:52:30 DEBUG { 19:52:30 DEBUG "instance": "10.142.0.107:9100" 19:52:30 DEBUG }, 19:52:30 DEBUG { 19:52:30 DEBUG "status": "success", 19:52:30 DEBUG }, 19:52:30 DEBUG "value": [ 19:52:30 DEBUG "status": "success", 19:52:30 DEBUG "data": { 19:52:30 DEBUG "value": [ 19:52:30 DEBUG 19:52:30 DEBUG "data": { 19:52:30 DEBUG "resultType": "vector", 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG "resultType": "vector", 19:52:30 DEBUG "result": [ 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG "result": [ 19:52:30 DEBUG { 19:52:30 DEBUG 19:52:30 DEBUG [...] 19:52:30 DEBUG { 19:52:30 DEBUG "metric": { 19:52:30 DEBUG [...] 19:52:30 DEBUG 19:52:30 DEBUG "metric": { 19:52:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:52:30 INFO [http_cmd]: http status code OK 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG "instance": "10.142.0.107:9100" 19:52:30 DEBUG }, 19:52:30 DEBUG --- status code --- 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG }, 19:52:30 DEBUG "value": [ 19:52:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:30 DEBUG 19:52:30 DEBUG "pod": "ds-cts-2" 19:52:30 DEBUG "value": [ 19:52:30 DEBUG 19:52:30 DEBUG --- http response --- 19:52:30 DEBUG "instance": "10.142.0.43:9100" 19:52:30 DEBUG }, 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG }, 19:52:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:52:30 DEBUG "value": [ 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG "value": [ 19:52:30 DEBUG ----- output ----- 19:52:30 DEBUG 1713552750, 19:52:30 DEBUG 19:52:30 DEBUG [...] 19:52:30 DEBUG 1713552750, 19:52:30 DEBUG { 19:52:30 DEBUG "0" 19:52:30 DEBUG [...] 19:52:30 DEBUG 19:52:30 DEBUG "1968072653824" 19:52:30 DEBUG "status": "success", 19:52:30 DEBUG ] 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG ] 19:52:30 DEBUG "data": { 19:52:30 DEBUG } 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG } 19:52:30 DEBUG "resultType": "vector", 19:52:30 DEBUG ] 19:52:30 DEBUG 19:52:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:52:30 DEBUG ] 19:52:30 DEBUG "result": [ 19:52:30 DEBUG } 19:52:30 DEBUG "instance": "10.142.0.43:9100" 19:52:30 DEBUG }, 19:52:30 DEBUG } 19:52:30 DEBUG { 19:52:30 DEBUG } 19:52:30 DEBUG }, 19:52:30 DEBUG "value": [ 19:52:30 DEBUG } 19:52:30 DEBUG "metric": { 19:52:30 DEBUG "value": [ 19:52:30 DEBUG 1713552750, 19:52:30 DEBUG "instance": "10.142.0.107:9100" 19:52:30 DEBUG 1713552750, 19:52:30 DEBUG "2492.4333333333334" 19:52:30 DEBUG }, 19:52:30 DEBUG "75204767.26666667" 19:52:30 DEBUG ] 19:52:30 DEBUG "value": [ 19:52:30 DEBUG ] 19:52:30 DEBUG } 19:52:30 DEBUG 19:52:30 DEBUG } 19:52:30 DEBUG ] 19:52:30 DEBUG 19:52:30 DEBUG ] 19:52:30 DEBUG } 19:52:30 DEBUG 19:52:30 DEBUG } 19:52:30 DEBUG } 19:52:30 DEBUG [...] 19:52:30 DEBUG } 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG "instance": "10.142.0.43:9100" 19:52:30 DEBUG }, 19:52:30 DEBUG "value": [ 19:52:30 DEBUG 1713552750, 19:52:30 DEBUG "75200395.13333333" 19:52:30 DEBUG ] 19:52:30 DEBUG } 19:52:30 DEBUG ] 19:52:30 DEBUG } 19:52:30 DEBUG } 19:52:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552750" 19:52:30 INFO [http_cmd]: http status code OK 19:52:30 DEBUG --- status code --- 19:52:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:30 DEBUG --- http response --- 19:52:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:52:30 DEBUG ----- output ----- 19:52:30 DEBUG { 19:52:30 DEBUG "status": "success", 19:52:30 DEBUG "data": { 19:52:30 DEBUG "resultType": "vector", 19:52:30 DEBUG "result": [ 19:52:30 DEBUG { 19:52:30 DEBUG "metric": { 19:52:30 DEBUG "pod": "ds-idrepo-0" 19:52:30 DEBUG }, 19:52:30 DEBUG "value": [ 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG [...] 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG 19:52:30 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 19:52:30 DEBUG }, 19:52:30 DEBUG "value": [ 19:52:30 DEBUG 1713552750, 19:52:30 DEBUG "0.000028043799954238253" 19:52:30 DEBUG ] 19:52:30 DEBUG } 19:52:30 DEBUG ] 19:52:30 DEBUG } 19:52:30 DEBUG } 19:52:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713552751" 19:52:31 INFO [http_cmd]: http status code OK 19:52:31 DEBUG --- status code --- 19:52:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:31 DEBUG --- http response --- 19:52:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:52:31 DEBUG ----- output ----- 19:52:31 DEBUG { 19:52:31 DEBUG "status": "success", 19:52:31 DEBUG "data": { 19:52:31 DEBUG "resultType": "vector", 19:52:31 DEBUG "result": [ 19:52:31 DEBUG { 19:52:31 DEBUG "metric": { 19:52:31 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:52:31 DEBUG }, 19:52:31 DEBUG "value": [ 19:52:31 DEBUG 19:52:31 DEBUG 19:52:31 DEBUG 19:52:31 DEBUG [...] 19:52:31 DEBUG 19:52:31 DEBUG 19:52:31 DEBUG 19:52:31 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:52:31 DEBUG }, 19:52:31 DEBUG "value": [ 19:52:31 DEBUG 1713552751, 19:52:31 DEBUG "3481788416" 19:52:31 DEBUG ] 19:52:31 DEBUG } 19:52:31 DEBUG ] 19:52:31 DEBUG } 19:52:31 DEBUG } 19:52:32 INFO 19:52:32 INFO [loop_until]: kubectl --namespace=xlou top pods 19:52:32 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:52:32 INFO [loop_until]: OK (rc = 0) 19:52:32 DEBUG --- stdout --- 19:52:32 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 16067m 9005Mi am-767f6b67b5-qcqz2 16096m 8824Mi am-767f6b67b5-vbltz 17981m 8996Mi ds-cts-0 17819m 10937Mi ds-cts-1 15410m 11195Mi ds-cts-2 17388m 10952Mi ds-idrepo-0 7010m 3672Mi ds-idrepo-1 7126m 3537Mi ds-idrepo-2 7323m 3492Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8067m 3267Mi 19:52:32 DEBUG --- stderr --- 19:52:32 DEBUG 19:52:32 INFO 19:52:32 INFO [loop_until]: kubectl --namespace=xlou top node 19:52:32 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:52:32 INFO [loop_until]: OK (rc = 0) 19:52:32 DEBUG --- stdout --- 19:52:32 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 71m 0% 6251Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7400m 24% 7261Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 15985m 53% 11705Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 85m 0% 3351Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 17565m 58% 13920Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 156m 0% 5198Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 16620m 55% 11695Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15557m 52% 2993Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 18150m 60% 11212Mi 10% gke-xlou-cdm-ds-0f601751-j488 7636m 25% 6747Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 15624m 52% 15949Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 7418m 24% 7436Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 17167m 57% 13401Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 8197m 51% 6978Mi 11% 19:52:32 DEBUG --- stderr --- 19:52:32 DEBUG 19:53:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713552810" 19:53:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552810" 19:53:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552810" 19:53:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552810" 19:53:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713552810" 19:53:30 INFO [http_cmd]: http status code OK 19:53:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552810" 19:53:30 INFO [http_cmd]: http status code OK 19:53:30 ERROR [http_cmd]: ERROR 19:53:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552810" 19:53:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552810" 19:53:30 DEBUG --- status code --- 19:53:30 DEBUG --- status code --- 19:53:30 ERROR --- status code --- 19:53:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:30 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:53:30 DEBUG --- http response --- 19:53:30 DEBUG --- http response --- 19:53:30 ERROR --- http response --- 19:53:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:53:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:53:30 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:53:30 DEBUG ----- output ----- 19:53:30 DEBUG ----- output ----- 19:53:30 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:53:30 DEBUG { 19:53:30 DEBUG { 19:53:30 DEBUG "status": "success", 19:53:30 DEBUG "status": "success", 19:53:30 DEBUG "data": { 19:53:30 DEBUG "data": { 19:53:30 DEBUG "resultType": "vector", 19:53:30 DEBUG "resultType": "vector", 19:53:30 DEBUG "result": [ 19:53:30 DEBUG "result": [ 19:53:30 DEBUG { 19:53:30 DEBUG { 19:53:30 DEBUG "metric": { 19:53:30 DEBUG "metric": { 19:53:30 DEBUG "pod": "ds-cts-0" 19:53:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:53:30 DEBUG }, 19:53:30 DEBUG }, 19:53:30 DEBUG "value": [ 19:53:30 DEBUG "value": [ 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG [...] 19:53:30 DEBUG [...] 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG "pod": "ds-cts-2" 19:53:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:53:30 DEBUG }, 19:53:30 DEBUG }, 19:53:30 DEBUG "value": [ 19:53:30 DEBUG "value": [ 19:53:30 DEBUG 1713552810, 19:53:30 DEBUG 1713552810, 19:53:30 DEBUG "0.0003999999999999999" 19:53:30 DEBUG "450281" 19:53:30 DEBUG ] 19:53:30 DEBUG ] 19:53:30 DEBUG } 19:53:30 DEBUG } 19:53:30 DEBUG ] 19:53:30 DEBUG ] 19:53:30 DEBUG } 19:53:30 DEBUG } 19:53:30 DEBUG } 19:53:30 DEBUG } 19:53:30 INFO [http_cmd]: http status code OK 19:53:30 DEBUG --- status code --- 19:53:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:30 DEBUG --- http response --- 19:53:30 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:53:30 INFO [http_cmd]: http status code OK 19:53:30 DEBUG --- status code --- 19:53:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:30 DEBUG --- http response --- 19:53:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:53:30 DEBUG ----- output ----- 19:53:30 DEBUG { 19:53:30 DEBUG "status": "success", 19:53:30 DEBUG "data": { 19:53:30 DEBUG "resultType": "vector", 19:53:30 DEBUG "result": [ 19:53:30 DEBUG { 19:53:30 DEBUG "metric": { 19:53:30 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:53:30 DEBUG }, 19:53:30 DEBUG "value": [ 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG [...] 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG "pod": "idm-b85f9d978-txpwv" 19:53:30 DEBUG }, 19:53:30 DEBUG "value": [ 19:53:30 DEBUG 1713552810, 19:53:30 DEBUG "495.1175311565493" 19:53:30 DEBUG ] 19:53:30 DEBUG } 19:53:30 DEBUG ] 19:53:30 DEBUG } 19:53:30 DEBUG } 19:53:30 INFO [http_cmd]: http status code OK 19:53:30 DEBUG --- status code --- 19:53:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:30 DEBUG --- http response --- 19:53:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:53:30 DEBUG ----- output ----- 19:53:30 DEBUG { 19:53:30 DEBUG "status": "success", 19:53:30 DEBUG "data": { 19:53:30 DEBUG "resultType": "vector", 19:53:30 DEBUG "result": [ 19:53:30 DEBUG { 19:53:30 DEBUG "metric": { 19:53:30 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:53:30 DEBUG }, 19:53:30 DEBUG "value": [ 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG [...] 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:53:30 DEBUG }, 19:53:30 DEBUG "value": [ 19:53:30 DEBUG 1713552810, 19:53:30 DEBUG "8.035625489624966" 19:53:30 DEBUG ] 19:53:30 DEBUG } 19:53:30 DEBUG ] 19:53:30 DEBUG } 19:53:30 DEBUG } 19:53:30 INFO [http_cmd]: http status code OK 19:53:30 INFO [http_cmd]: http status code OK 19:53:30 DEBUG --- status code --- 19:53:30 DEBUG --- status code --- 19:53:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:30 DEBUG --- http response --- 19:53:30 DEBUG --- http response --- 19:53:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:53:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:53:30 DEBUG ----- output ----- 19:53:30 DEBUG ----- output ----- 19:53:30 DEBUG { 19:53:30 DEBUG { 19:53:30 DEBUG "status": "success", 19:53:30 DEBUG "status": "success", 19:53:30 DEBUG "data": { 19:53:30 DEBUG "data": { 19:53:30 DEBUG "resultType": "vector", 19:53:30 DEBUG "resultType": "vector", 19:53:30 DEBUG "result": [ 19:53:30 DEBUG "result": [ 19:53:30 DEBUG { 19:53:30 DEBUG { 19:53:30 DEBUG "metric": { 19:53:30 DEBUG "metric": { 19:53:30 DEBUG "pod": "ds-idrepo-0" 19:53:30 DEBUG "pod": "ds-idrepo-0" 19:53:30 DEBUG }, 19:53:30 DEBUG }, 19:53:30 DEBUG "value": [ 19:53:30 DEBUG "value": [ 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG [...] 19:53:30 DEBUG [...] 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG "pod": "ds-idrepo-2" 19:53:30 DEBUG "pod": "ds-idrepo-2" 19:53:30 DEBUG }, 19:53:30 DEBUG }, 19:53:30 DEBUG "value": [ 19:53:30 DEBUG "value": [ 19:53:30 DEBUG 1713552810, 19:53:30 DEBUG 1713552810, 19:53:30 DEBUG "0" 19:53:30 DEBUG "0" 19:53:30 DEBUG ] 19:53:30 DEBUG ] 19:53:30 DEBUG } 19:53:30 DEBUG } 19:53:30 DEBUG ] 19:53:30 DEBUG ] 19:53:30 DEBUG } 19:53:30 DEBUG } 19:53:30 DEBUG } 19:53:30 DEBUG } 19:53:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713552810" 19:53:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552810" 19:53:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552810" 19:53:30 INFO [http_cmd]: http status code OK 19:53:30 DEBUG --- status code --- 19:53:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:30 DEBUG --- http response --- 19:53:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:53:30 DEBUG ----- output ----- 19:53:30 DEBUG { 19:53:30 DEBUG "status": "success", 19:53:30 DEBUG "data": { 19:53:30 DEBUG "resultType": "vector", 19:53:30 DEBUG "result": [ 19:53:30 DEBUG { 19:53:30 DEBUG "metric": { 19:53:30 DEBUG "pod": "ds-idrepo-0" 19:53:30 DEBUG }, 19:53:30 DEBUG "value": [ 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG [...] 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG "pod": "ds-cts-2" 19:53:30 DEBUG }, 19:53:30 DEBUG "value": [ 19:53:30 DEBUG 1713552810, 19:53:30 DEBUG "0" 19:53:30 DEBUG ] 19:53:30 DEBUG } 19:53:30 DEBUG ] 19:53:30 DEBUG } 19:53:30 DEBUG } 19:53:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552810" 19:53:30 INFO [http_cmd]: http status code OK 19:53:30 DEBUG --- status code --- 19:53:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:30 DEBUG --- http response --- 19:53:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:53:30 DEBUG ----- output ----- 19:53:30 DEBUG { 19:53:30 DEBUG "status": "success", 19:53:30 DEBUG "data": { 19:53:30 DEBUG "resultType": "vector", 19:53:30 DEBUG "result": [ 19:53:30 DEBUG { 19:53:30 DEBUG "metric": { 19:53:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:53:30 DEBUG }, 19:53:30 DEBUG "value": [ 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG [...] 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:53:30 DEBUG }, 19:53:30 DEBUG "value": [ 19:53:30 DEBUG 1713552810, 19:53:30 DEBUG "0" 19:53:30 DEBUG ] 19:53:30 DEBUG } 19:53:30 DEBUG ] 19:53:30 DEBUG } 19:53:30 DEBUG } 19:53:30 INFO [http_cmd]: http status code OK 19:53:30 DEBUG --- status code --- 19:53:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:30 DEBUG --- http response --- 19:53:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:53:30 DEBUG ----- output ----- 19:53:30 DEBUG { 19:53:30 DEBUG "status": "success", 19:53:30 DEBUG "data": { 19:53:30 DEBUG "resultType": "vector", 19:53:30 DEBUG "result": [ 19:53:30 DEBUG { 19:53:30 DEBUG "metric": { 19:53:30 DEBUG "pod": "ds-cts-0" 19:53:30 DEBUG }, 19:53:30 DEBUG "value": [ 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG [...] 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG "pod": "ds-cts-2" 19:53:30 DEBUG }, 19:53:30 DEBUG "value": [ 19:53:30 DEBUG 1713552810, 19:53:30 DEBUG "30" 19:53:30 DEBUG ] 19:53:30 DEBUG } 19:53:30 DEBUG ] 19:53:30 DEBUG } 19:53:30 DEBUG } 19:53:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713552810" 19:53:30 INFO [http_cmd]: http status code OK 19:53:30 DEBUG --- status code --- 19:53:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:30 DEBUG --- http response --- 19:53:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:53:30 DEBUG ----- output ----- 19:53:30 DEBUG { 19:53:30 DEBUG "status": "success", 19:53:30 DEBUG "data": { 19:53:30 DEBUG "resultType": "vector", 19:53:30 DEBUG "result": [ 19:53:30 DEBUG { 19:53:30 DEBUG "metric": { 19:53:30 DEBUG "instance": "10.142.0.107:9100" 19:53:30 DEBUG }, 19:53:30 DEBUG "value": [ 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG [...] 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG "instance": "10.142.0.43:9100" 19:53:30 DEBUG }, 19:53:30 DEBUG "value": [ 19:53:30 DEBUG 1713552810, 19:53:30 DEBUG "0.16419999999994841" 19:53:30 DEBUG ] 19:53:30 DEBUG } 19:53:30 DEBUG ] 19:53:30 DEBUG } 19:53:30 DEBUG } 19:53:30 INFO [http_cmd]: http status code OK 19:53:30 DEBUG --- status code --- 19:53:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:30 DEBUG --- http response --- 19:53:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:53:30 DEBUG ----- output ----- 19:53:30 DEBUG { 19:53:30 DEBUG "status": "success", 19:53:30 DEBUG "data": { 19:53:30 DEBUG "resultType": "vector", 19:53:30 DEBUG "result": [ 19:53:30 DEBUG { 19:53:30 DEBUG "metric": { 19:53:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:53:30 DEBUG }, 19:53:30 DEBUG "value": [ 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG [...] 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:53:30 DEBUG }, 19:53:30 DEBUG "value": [ 19:53:30 DEBUG 1713552810, 19:53:30 DEBUG "0.014" 19:53:30 DEBUG ] 19:53:30 DEBUG } 19:53:30 DEBUG ] 19:53:30 DEBUG } 19:53:30 DEBUG } 19:53:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552810" 19:53:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552810" 19:53:30 INFO [http_cmd]: http status code OK 19:53:30 DEBUG --- status code --- 19:53:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:30 DEBUG --- http response --- 19:53:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:53:30 DEBUG ----- output ----- 19:53:30 DEBUG { 19:53:30 DEBUG "status": "success", 19:53:30 DEBUG "data": { 19:53:30 DEBUG "resultType": "vector", 19:53:30 DEBUG "result": [ 19:53:30 DEBUG { 19:53:30 DEBUG "metric": { 19:53:30 DEBUG "pod": "ds-idrepo-0" 19:53:30 DEBUG }, 19:53:30 DEBUG "value": [ 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG [...] 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG "pod": "ds-idrepo-2" 19:53:30 DEBUG }, 19:53:30 DEBUG "value": [ 19:53:30 DEBUG 1713552810, 19:53:30 DEBUG "0" 19:53:30 DEBUG ] 19:53:30 DEBUG } 19:53:30 DEBUG ] 19:53:30 DEBUG } 19:53:30 DEBUG } 19:53:30 INFO [http_cmd]: http status code OK 19:53:30 DEBUG --- status code --- 19:53:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:30 DEBUG --- http response --- 19:53:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:53:30 DEBUG ----- output ----- 19:53:30 DEBUG { 19:53:30 DEBUG "status": "success", 19:53:30 DEBUG "data": { 19:53:30 DEBUG "resultType": "vector", 19:53:30 DEBUG "result": [ 19:53:30 DEBUG { 19:53:30 DEBUG "metric": { 19:53:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:53:30 DEBUG }, 19:53:30 DEBUG "value": [ 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG [...] 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:53:30 DEBUG }, 19:53:30 DEBUG "value": [ 19:53:30 DEBUG 1713552810, 19:53:30 DEBUG "2510.633333333333" 19:53:30 DEBUG ] 19:53:30 DEBUG } 19:53:30 DEBUG ] 19:53:30 DEBUG } 19:53:30 DEBUG } 19:53:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552810" 19:53:30 INFO [http_cmd]: http status code OK 19:53:30 DEBUG --- status code --- 19:53:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:30 DEBUG --- http response --- 19:53:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552810" 19:53:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:53:30 DEBUG ----- output ----- 19:53:30 DEBUG { 19:53:30 DEBUG "status": "success", 19:53:30 DEBUG "data": { 19:53:30 DEBUG "resultType": "vector", 19:53:30 DEBUG "result": [ 19:53:30 DEBUG { 19:53:30 DEBUG "metric": { 19:53:30 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:53:30 DEBUG }, 19:53:30 DEBUG "value": [ 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG [...] 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG "pod": "idm-b85f9d978-txpwv" 19:53:30 DEBUG }, 19:53:30 DEBUG "value": [ 19:53:30 DEBUG 1713552810, 19:53:30 DEBUG "37375927.30947675" 19:53:30 DEBUG ] 19:53:30 DEBUG } 19:53:30 DEBUG ] 19:53:30 DEBUG } 19:53:30 DEBUG } 19:53:30 INFO [http_cmd]: http status code OK 19:53:30 DEBUG --- status code --- 19:53:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:30 DEBUG --- http response --- 19:53:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:53:30 DEBUG ----- output ----- 19:53:30 DEBUG { 19:53:30 DEBUG "status": "success", 19:53:30 DEBUG "data": { 19:53:30 DEBUG "resultType": "vector", 19:53:30 DEBUG "result": [ 19:53:30 DEBUG { 19:53:30 DEBUG "metric": { 19:53:30 DEBUG "instance": "10.142.0.107:9100" 19:53:30 DEBUG }, 19:53:30 DEBUG "value": [ 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG [...] 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG "instance": "10.142.0.43:9100" 19:53:30 DEBUG }, 19:53:30 DEBUG "value": [ 19:53:30 DEBUG 1713552810, 19:53:30 DEBUG "75675809.86666667" 19:53:30 DEBUG ] 19:53:30 DEBUG } 19:53:30 DEBUG ] 19:53:30 DEBUG } 19:53:30 DEBUG } 19:53:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552810" 19:53:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552810" 19:53:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552810" 19:53:30 INFO [http_cmd]: http status code OK 19:53:30 DEBUG --- status code --- 19:53:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:30 DEBUG --- http response --- 19:53:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:53:30 DEBUG ----- output ----- 19:53:30 DEBUG { 19:53:30 DEBUG "status": "success", 19:53:30 DEBUG "data": { 19:53:30 DEBUG "resultType": "vector", 19:53:30 DEBUG "result": [ 19:53:30 DEBUG { 19:53:30 DEBUG "metric": { 19:53:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:53:30 DEBUG }, 19:53:30 DEBUG "value": [ 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG [...] 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:53:30 DEBUG }, 19:53:30 DEBUG "value": [ 19:53:30 DEBUG 1713552810, 19:53:30 DEBUG "2505.3" 19:53:30 DEBUG ] 19:53:30 DEBUG } 19:53:30 DEBUG ] 19:53:30 DEBUG } 19:53:30 DEBUG } 19:53:30 INFO [http_cmd]: http status code OK 19:53:30 DEBUG --- status code --- 19:53:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:30 DEBUG --- http response --- 19:53:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:53:30 DEBUG ----- output ----- 19:53:30 DEBUG { 19:53:30 DEBUG "status": "success", 19:53:30 DEBUG "data": { 19:53:30 DEBUG "resultType": "vector", 19:53:30 DEBUG "result": [ 19:53:30 DEBUG { 19:53:30 DEBUG "metric": { 19:53:30 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:53:30 DEBUG }, 19:53:30 DEBUG "value": [ 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG [...] 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG "pod": "idm-b85f9d978-txpwv" 19:53:30 DEBUG }, 19:53:30 DEBUG "value": [ 19:53:30 DEBUG 1713552810, 19:53:30 DEBUG "0" 19:53:30 DEBUG ] 19:53:30 DEBUG } 19:53:30 DEBUG ] 19:53:30 DEBUG } 19:53:30 DEBUG } 19:53:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552810" 19:53:30 INFO [http_cmd]: http status code OK 19:53:30 DEBUG --- status code --- 19:53:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:30 DEBUG --- http response --- 19:53:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:53:30 DEBUG ----- output ----- 19:53:30 DEBUG { 19:53:30 DEBUG "status": "success", 19:53:30 DEBUG "data": { 19:53:30 DEBUG "resultType": "vector", 19:53:30 DEBUG "result": [ 19:53:30 DEBUG { 19:53:30 DEBUG "metric": { 19:53:30 DEBUG "instance": "10.142.0.107:9100" 19:53:30 DEBUG }, 19:53:30 DEBUG "value": [ 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG [...] 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG "instance": "10.142.0.43:9100" 19:53:30 DEBUG }, 19:53:30 DEBUG "value": [ 19:53:30 DEBUG 1713552810, 19:53:30 DEBUG "1971302321152" 19:53:30 DEBUG ] 19:53:30 DEBUG } 19:53:30 DEBUG ] 19:53:30 DEBUG } 19:53:30 DEBUG } 19:53:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552810" 19:53:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552810" 19:53:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552810" 19:53:30 INFO [http_cmd]: http status code OK 19:53:30 DEBUG --- status code --- 19:53:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:30 DEBUG --- http response --- 19:53:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:53:30 DEBUG ----- output ----- 19:53:30 DEBUG { 19:53:30 DEBUG "status": "success", 19:53:30 DEBUG "data": { 19:53:30 DEBUG "resultType": "vector", 19:53:30 DEBUG "result": [ 19:53:30 DEBUG { 19:53:30 DEBUG "metric": { 19:53:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:53:30 DEBUG }, 19:53:30 DEBUG "value": [ 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG [...] 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:53:30 DEBUG }, 19:53:30 DEBUG "value": [ 19:53:30 DEBUG 1713552810, 19:53:30 DEBUG "2511.9666666666667" 19:53:30 DEBUG ] 19:53:30 DEBUG } 19:53:30 DEBUG ] 19:53:30 DEBUG } 19:53:30 DEBUG } 19:53:30 INFO [http_cmd]: http status code OK 19:53:30 DEBUG --- status code --- 19:53:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:30 DEBUG --- http response --- 19:53:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:53:30 DEBUG ----- output ----- 19:53:30 DEBUG { 19:53:30 DEBUG "status": "success", 19:53:30 DEBUG "data": { 19:53:30 DEBUG "resultType": "vector", 19:53:30 DEBUG "result": [ 19:53:30 DEBUG { 19:53:30 DEBUG "metric": { 19:53:30 DEBUG "pod": "ds-cts-0" 19:53:30 DEBUG }, 19:53:30 DEBUG "value": [ 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG [...] 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG "pod": "ds-cts-2" 19:53:30 DEBUG }, 19:53:30 DEBUG "value": [ 19:53:30 DEBUG 1713552810, 19:53:30 DEBUG "0" 19:53:30 DEBUG ] 19:53:30 DEBUG } 19:53:30 DEBUG ] 19:53:30 DEBUG } 19:53:30 DEBUG } 19:53:30 INFO [http_cmd]: http status code OK 19:53:30 DEBUG --- status code --- 19:53:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:30 DEBUG --- http response --- 19:53:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:53:30 DEBUG ----- output ----- 19:53:30 DEBUG { 19:53:30 DEBUG "status": "success", 19:53:30 DEBUG "data": { 19:53:30 DEBUG "resultType": "vector", 19:53:30 DEBUG "result": [ 19:53:30 DEBUG { 19:53:30 DEBUG "metric": { 19:53:30 DEBUG "instance": "10.142.0.107:9100" 19:53:30 DEBUG }, 19:53:30 DEBUG "value": [ 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG [...] 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG "instance": "10.142.0.43:9100" 19:53:30 DEBUG }, 19:53:30 DEBUG "value": [ 19:53:30 DEBUG 1713552810, 19:53:30 DEBUG "0.07299999999998666" 19:53:30 DEBUG ] 19:53:30 DEBUG } 19:53:30 DEBUG ] 19:53:30 DEBUG } 19:53:30 DEBUG } 19:53:30 INFO [http_cmd]: http status code OK 19:53:30 DEBUG --- status code --- 19:53:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:30 DEBUG --- http response --- 19:53:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:53:30 DEBUG ----- output ----- 19:53:30 DEBUG { 19:53:30 DEBUG "status": "success", 19:53:30 DEBUG "data": { 19:53:30 DEBUG "resultType": "vector", 19:53:30 DEBUG "result": [ 19:53:30 DEBUG { 19:53:30 DEBUG "metric": { 19:53:30 DEBUG "instance": "10.142.0.107:9100" 19:53:30 DEBUG }, 19:53:30 DEBUG "value": [ 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG [...] 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG 19:53:30 DEBUG "instance": "10.142.0.43:9100" 19:53:30 DEBUG }, 19:53:30 DEBUG "value": [ 19:53:30 DEBUG 1713552810, 19:53:30 DEBUG "75679847.56666666" 19:53:30 DEBUG ] 19:53:30 DEBUG } 19:53:30 DEBUG ] 19:53:30 DEBUG } 19:53:30 DEBUG } 19:53:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552810" 19:53:31 INFO [http_cmd]: http status code OK 19:53:31 DEBUG --- status code --- 19:53:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:31 DEBUG --- http response --- 19:53:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:53:31 DEBUG ----- output ----- 19:53:31 DEBUG { 19:53:31 DEBUG "status": "success", 19:53:31 DEBUG "data": { 19:53:31 DEBUG "resultType": "vector", 19:53:31 DEBUG "result": [ 19:53:31 DEBUG { 19:53:31 DEBUG "metric": { 19:53:31 DEBUG "pod": "am-767f6b67b5-9zw22" 19:53:31 DEBUG }, 19:53:31 DEBUG "value": [ 19:53:31 DEBUG 19:53:31 DEBUG 19:53:31 DEBUG 19:53:31 DEBUG [...] 19:53:31 DEBUG 19:53:31 DEBUG 19:53:31 DEBUG 19:53:31 DEBUG "pod": "am-767f6b67b5-vbltz" 19:53:31 DEBUG }, 19:53:31 DEBUG "value": [ 19:53:31 DEBUG 1713552810, 19:53:31 DEBUG "2510.0666666666666" 19:53:31 DEBUG ] 19:53:31 DEBUG } 19:53:31 DEBUG ] 19:53:31 DEBUG } 19:53:31 DEBUG } 19:53:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552811" 19:53:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713552811" 19:53:31 INFO [http_cmd]: http status code OK 19:53:31 DEBUG --- status code --- 19:53:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:31 DEBUG --- http response --- 19:53:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:53:31 DEBUG ----- output ----- 19:53:31 DEBUG { 19:53:31 DEBUG "status": "success", 19:53:31 DEBUG "data": { 19:53:31 DEBUG "resultType": "vector", 19:53:31 DEBUG "result": [ 19:53:31 DEBUG { 19:53:31 DEBUG "metric": { 19:53:31 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:53:31 DEBUG }, 19:53:31 DEBUG "value": [ 19:53:31 DEBUG 19:53:31 DEBUG 19:53:31 DEBUG 19:53:31 DEBUG [...] 19:53:31 DEBUG 19:53:31 DEBUG 19:53:31 DEBUG 19:53:31 DEBUG "pod": "idm-b85f9d978-txpwv" 19:53:31 DEBUG }, 19:53:31 DEBUG "value": [ 19:53:31 DEBUG 1713552811, 19:53:31 DEBUG "0.002259332312491021" 19:53:31 DEBUG ] 19:53:31 DEBUG } 19:53:31 DEBUG ] 19:53:31 DEBUG } 19:53:31 DEBUG } 19:53:31 INFO [http_cmd]: http status code OK 19:53:31 DEBUG --- status code --- 19:53:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:31 DEBUG --- http response --- 19:53:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:53:31 DEBUG ----- output ----- 19:53:31 DEBUG { 19:53:31 DEBUG "status": "success", 19:53:31 DEBUG "data": { 19:53:31 DEBUG "resultType": "vector", 19:53:31 DEBUG "result": [ 19:53:31 DEBUG { 19:53:31 DEBUG "metric": { 19:53:31 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:53:31 DEBUG }, 19:53:31 DEBUG "value": [ 19:53:31 DEBUG 19:53:31 DEBUG 19:53:31 DEBUG 19:53:31 DEBUG [...] 19:53:31 DEBUG 19:53:31 DEBUG 19:53:31 DEBUG 19:53:31 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:53:31 DEBUG }, 19:53:31 DEBUG "value": [ 19:53:31 DEBUG 1713552811, 19:53:31 DEBUG "3579887616" 19:53:31 DEBUG ] 19:53:31 DEBUG } 19:53:31 DEBUG ] 19:53:31 DEBUG } 19:53:31 DEBUG } 19:53:32 INFO 19:53:32 INFO [loop_until]: kubectl --namespace=xlou top pods 19:53:32 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:53:32 INFO [loop_until]: OK (rc = 0) 19:53:32 DEBUG --- stdout --- 19:53:32 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 15790m 9006Mi am-767f6b67b5-qcqz2 16612m 8813Mi am-767f6b67b5-vbltz 17965m 8994Mi ds-cts-0 17284m 10948Mi ds-cts-1 15947m 11193Mi ds-cts-2 18345m 10954Mi ds-idrepo-0 7248m 3677Mi ds-idrepo-1 7161m 3537Mi ds-idrepo-2 7264m 3493Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1174Mi idm-b85f9d978-wnhxb 4m 1217Mi lodemon-f8459c597-6tj6f 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8019m 3320Mi 19:53:32 DEBUG --- stderr --- 19:53:32 DEBUG 19:53:32 INFO 19:53:32 INFO [loop_until]: kubectl --namespace=xlou top node 19:53:32 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:53:33 INFO [loop_until]: OK (rc = 0) 19:53:33 DEBUG --- stdout --- 19:53:33 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 71m 0% 6250Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7421m 24% 7271Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16110m 53% 11706Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 87m 0% 3343Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 16854m 56% 13939Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 151m 0% 5206Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 16017m 53% 11697Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15681m 52% 2987Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 18126m 60% 11216Mi 10% gke-xlou-cdm-ds-0f601751-j488 7676m 25% 6734Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 15780m 52% 15949Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 7475m 25% 7444Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 17500m 58% 13396Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 8201m 51% 7071Mi 12% 19:53:33 DEBUG --- stderr --- 19:53:33 DEBUG 19:54:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552870" 19:54:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552870" 19:54:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552870" 19:54:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552870" 19:54:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713552870" 19:54:30 INFO [http_cmd]: http status code OK 19:54:30 INFO [http_cmd]: http status code OK 19:54:30 DEBUG --- status code --- 19:54:30 INFO [http_cmd]: http status code OK 19:54:30 DEBUG --- status code --- 19:54:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:30 DEBUG --- status code --- 19:54:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:30 DEBUG --- http response --- 19:54:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:30 DEBUG --- http response --- 19:54:30 DEBUG --- http response --- 19:54:30 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:54:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:54:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:54:30 DEBUG ----- output ----- 19:54:30 DEBUG ----- output ----- 19:54:30 DEBUG { 19:54:30 DEBUG { 19:54:30 DEBUG "status": "success", 19:54:30 DEBUG "status": "success", 19:54:30 DEBUG "data": { 19:54:30 DEBUG "data": { 19:54:30 DEBUG "resultType": "vector", 19:54:30 DEBUG "resultType": "vector", 19:54:30 DEBUG "result": [ 19:54:30 DEBUG "result": [ 19:54:30 DEBUG { 19:54:30 DEBUG { 19:54:30 DEBUG "metric": { 19:54:30 DEBUG "metric": { 19:54:30 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:54:30 DEBUG "pod": "ds-cts-0" 19:54:30 DEBUG }, 19:54:30 DEBUG }, 19:54:30 DEBUG "value": [ 19:54:30 DEBUG "value": [ 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG [...] 19:54:30 DEBUG [...] 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:54:30 DEBUG "pod": "ds-cts-2" 19:54:30 DEBUG }, 19:54:30 DEBUG }, 19:54:30 DEBUG "value": [ 19:54:30 DEBUG "value": [ 19:54:30 DEBUG 1713552870, 19:54:30 DEBUG 1713552870, 19:54:30 DEBUG "0" 19:54:30 DEBUG "0.0004666666666666667" 19:54:30 DEBUG ] 19:54:30 DEBUG ] 19:54:30 DEBUG } 19:54:30 DEBUG } 19:54:30 DEBUG ] 19:54:30 DEBUG ] 19:54:30 DEBUG } 19:54:30 DEBUG } 19:54:30 DEBUG } 19:54:30 DEBUG } 19:54:30 INFO [http_cmd]: http status code OK 19:54:30 DEBUG --- status code --- 19:54:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:30 DEBUG --- http response --- 19:54:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:54:30 DEBUG ----- output ----- 19:54:30 DEBUG { 19:54:30 DEBUG "status": "success", 19:54:30 DEBUG "data": { 19:54:30 DEBUG "resultType": "vector", 19:54:30 DEBUG "result": [ 19:54:30 DEBUG { 19:54:30 DEBUG "metric": { 19:54:30 DEBUG "pod": "ds-idrepo-0" 19:54:30 DEBUG }, 19:54:30 DEBUG "value": [ 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG [...] 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG "pod": "ds-idrepo-2" 19:54:30 DEBUG }, 19:54:30 DEBUG "value": [ 19:54:30 DEBUG 1713552870, 19:54:30 DEBUG "0" 19:54:30 DEBUG ] 19:54:30 DEBUG } 19:54:30 DEBUG ] 19:54:30 DEBUG } 19:54:30 DEBUG } 19:54:30 INFO [http_cmd]: http status code OK 19:54:30 DEBUG --- status code --- 19:54:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:30 DEBUG --- http response --- 19:54:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:54:30 DEBUG ----- output ----- 19:54:30 DEBUG { 19:54:30 DEBUG "status": "success", 19:54:30 DEBUG "data": { 19:54:30 DEBUG "resultType": "vector", 19:54:30 DEBUG "result": [ 19:54:30 DEBUG { 19:54:30 DEBUG "metric": { 19:54:30 DEBUG "pod": "ds-idrepo-0" 19:54:30 DEBUG }, 19:54:30 DEBUG "value": [ 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG [...] 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG "pod": "ds-cts-2" 19:54:30 DEBUG }, 19:54:30 DEBUG "value": [ 19:54:30 DEBUG 1713552870, 19:54:30 DEBUG "0" 19:54:30 DEBUG ] 19:54:30 DEBUG } 19:54:30 DEBUG ] 19:54:30 DEBUG } 19:54:30 DEBUG } 19:54:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552870" 19:54:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713552870" 19:54:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552870" 19:54:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552870" 19:54:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713552870" 19:54:30 INFO [http_cmd]: http status code OK 19:54:30 ERROR [http_cmd]: ERROR 19:54:30 DEBUG --- status code --- 19:54:30 INFO [http_cmd]: http status code OK 19:54:30 ERROR --- status code --- 19:54:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:30 DEBUG --- status code --- 19:54:30 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:54:30 DEBUG --- http response --- 19:54:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:30 ERROR --- http response --- 19:54:30 DEBUG --- http response --- 19:54:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:54:30 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:54:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:54:30 DEBUG ----- output ----- 19:54:30 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:54:30 DEBUG ----- output ----- 19:54:30 DEBUG { 19:54:30 DEBUG { 19:54:30 DEBUG "status": "success", 19:54:30 DEBUG "status": "success", 19:54:30 DEBUG "data": { 19:54:30 DEBUG "data": { 19:54:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552870" 19:54:30 DEBUG "resultType": "vector", 19:54:30 DEBUG "resultType": "vector", 19:54:30 DEBUG "result": [ 19:54:30 DEBUG "result": [ 19:54:30 DEBUG { 19:54:30 DEBUG { 19:54:30 DEBUG "metric": { 19:54:30 DEBUG "metric": { 19:54:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:54:30 DEBUG "instance": "10.142.0.107:9100" 19:54:30 DEBUG }, 19:54:30 DEBUG }, 19:54:30 DEBUG "value": [ 19:54:30 DEBUG "value": [ 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG [...] 19:54:30 DEBUG [...] 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:54:30 DEBUG "instance": "10.142.0.43:9100" 19:54:30 DEBUG }, 19:54:30 DEBUG }, 19:54:30 DEBUG "value": [ 19:54:30 DEBUG "value": [ 19:54:30 DEBUG 1713552870, 19:54:30 DEBUG 1713552870, 19:54:30 DEBUG "450153.5" 19:54:30 DEBUG "0.16703333333336728" 19:54:30 DEBUG ] 19:54:30 DEBUG ] 19:54:30 DEBUG } 19:54:30 INFO [http_cmd]: http status code OK 19:54:30 DEBUG } 19:54:30 DEBUG ] 19:54:30 DEBUG --- status code --- 19:54:30 DEBUG ] 19:54:30 DEBUG } 19:54:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:30 DEBUG } 19:54:30 DEBUG } 19:54:30 DEBUG --- http response --- 19:54:30 DEBUG } 19:54:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:54:30 DEBUG ----- output ----- 19:54:30 DEBUG { 19:54:30 DEBUG "status": "success", 19:54:30 DEBUG "data": { 19:54:30 DEBUG "resultType": "vector", 19:54:30 DEBUG "result": [ 19:54:30 DEBUG { 19:54:30 DEBUG "metric": { 19:54:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:54:30 DEBUG }, 19:54:30 DEBUG "value": [ 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG [...] 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:54:30 DEBUG }, 19:54:30 DEBUG "value": [ 19:54:30 DEBUG 1713552870, 19:54:30 DEBUG "0" 19:54:30 DEBUG ] 19:54:30 DEBUG } 19:54:30 DEBUG ] 19:54:30 DEBUG } 19:54:30 DEBUG } 19:54:30 INFO [http_cmd]: http status code OK 19:54:30 DEBUG --- status code --- 19:54:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:30 DEBUG --- http response --- 19:54:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:54:30 DEBUG ----- output ----- 19:54:30 DEBUG { 19:54:30 DEBUG "status": "success", 19:54:30 DEBUG "data": { 19:54:30 DEBUG "resultType": "vector", 19:54:30 DEBUG "result": [ 19:54:30 DEBUG { 19:54:30 DEBUG "metric": { 19:54:30 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:54:30 DEBUG }, 19:54:30 DEBUG "value": [ 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG [...] 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:54:30 DEBUG }, 19:54:30 DEBUG "value": [ 19:54:30 DEBUG 1713552870, 19:54:30 DEBUG "8.035185195785852" 19:54:30 DEBUG ] 19:54:30 DEBUG } 19:54:30 DEBUG ] 19:54:30 DEBUG } 19:54:30 DEBUG } 19:54:30 INFO [http_cmd]: http status code OK 19:54:30 DEBUG --- status code --- 19:54:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:30 DEBUG --- http response --- 19:54:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:54:30 DEBUG ----- output ----- 19:54:30 DEBUG { 19:54:30 DEBUG "status": "success", 19:54:30 DEBUG "data": { 19:54:30 DEBUG "resultType": "vector", 19:54:30 DEBUG "result": [ 19:54:30 DEBUG { 19:54:30 DEBUG "metric": { 19:54:30 DEBUG "pod": "ds-idrepo-0" 19:54:30 DEBUG }, 19:54:30 DEBUG "value": [ 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG [...] 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG "pod": "ds-idrepo-2" 19:54:30 DEBUG }, 19:54:30 DEBUG "value": [ 19:54:30 DEBUG 1713552870, 19:54:30 DEBUG "0" 19:54:30 DEBUG ] 19:54:30 DEBUG } 19:54:30 DEBUG ] 19:54:30 DEBUG } 19:54:30 DEBUG } 19:54:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552870" 19:54:30 INFO [http_cmd]: http status code OK 19:54:30 DEBUG --- status code --- 19:54:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:30 DEBUG --- http response --- 19:54:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:54:30 DEBUG ----- output ----- 19:54:30 DEBUG { 19:54:30 DEBUG "status": "success", 19:54:30 DEBUG "data": { 19:54:30 DEBUG "resultType": "vector", 19:54:30 DEBUG "result": [ 19:54:30 DEBUG { 19:54:30 DEBUG "metric": { 19:54:30 DEBUG "pod": "ds-cts-0" 19:54:30 DEBUG }, 19:54:30 DEBUG "value": [ 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG [...] 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG "pod": "ds-cts-2" 19:54:30 DEBUG }, 19:54:30 DEBUG "value": [ 19:54:30 DEBUG 1713552870, 19:54:30 DEBUG "29.9" 19:54:30 DEBUG ] 19:54:30 DEBUG } 19:54:30 DEBUG ] 19:54:30 DEBUG } 19:54:30 DEBUG } 19:54:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713552870" 19:54:30 INFO [http_cmd]: http status code OK 19:54:30 DEBUG --- status code --- 19:54:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:30 DEBUG --- http response --- 19:54:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:54:30 DEBUG ----- output ----- 19:54:30 DEBUG { 19:54:30 DEBUG "status": "success", 19:54:30 DEBUG "data": { 19:54:30 DEBUG "resultType": "vector", 19:54:30 DEBUG "result": [ 19:54:30 DEBUG { 19:54:30 DEBUG "metric": { 19:54:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:54:30 DEBUG }, 19:54:30 DEBUG "value": [ 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG [...] 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:54:30 DEBUG }, 19:54:30 DEBUG "value": [ 19:54:30 DEBUG 1713552870, 19:54:30 DEBUG "0.014" 19:54:30 DEBUG ] 19:54:30 DEBUG } 19:54:30 DEBUG ] 19:54:30 DEBUG } 19:54:30 DEBUG } 19:54:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552870" 19:54:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552870" 19:54:30 INFO [http_cmd]: http status code OK 19:54:30 DEBUG --- status code --- 19:54:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:30 DEBUG --- http response --- 19:54:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:54:30 DEBUG ----- output ----- 19:54:30 DEBUG { 19:54:30 DEBUG "status": "success", 19:54:30 DEBUG "data": { 19:54:30 DEBUG "resultType": "vector", 19:54:30 DEBUG "result": [ 19:54:30 DEBUG { 19:54:30 DEBUG "metric": { 19:54:30 DEBUG "pod": "ds-idrepo-0" 19:54:30 DEBUG }, 19:54:30 DEBUG "value": [ 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG [...] 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG "pod": "ds-idrepo-2" 19:54:30 DEBUG }, 19:54:30 DEBUG "value": [ 19:54:30 DEBUG 1713552870, 19:54:30 DEBUG "0" 19:54:30 DEBUG ] 19:54:30 DEBUG } 19:54:30 DEBUG ] 19:54:30 DEBUG } 19:54:30 DEBUG } 19:54:30 INFO [http_cmd]: http status code OK 19:54:30 DEBUG --- status code --- 19:54:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:30 DEBUG --- http response --- 19:54:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:54:30 DEBUG ----- output ----- 19:54:30 DEBUG { 19:54:30 DEBUG "status": "success", 19:54:30 DEBUG "data": { 19:54:30 DEBUG "resultType": "vector", 19:54:30 DEBUG "result": [ 19:54:30 DEBUG { 19:54:30 DEBUG "metric": { 19:54:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:54:30 DEBUG }, 19:54:30 DEBUG "value": [ 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG [...] 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG 19:54:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:54:30 DEBUG }, 19:54:30 DEBUG "value": [ 19:54:30 DEBUG 1713552870, 19:54:30 DEBUG "2496.8" 19:54:30 DEBUG ] 19:54:30 DEBUG } 19:54:30 DEBUG ] 19:54:30 DEBUG } 19:54:30 DEBUG } 19:54:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552870" 19:54:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552870" 19:54:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552870" 19:54:31 INFO [http_cmd]: http status code OK 19:54:31 DEBUG --- status code --- 19:54:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:31 DEBUG --- http response --- 19:54:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:54:31 DEBUG ----- output ----- 19:54:31 DEBUG { 19:54:31 DEBUG "status": "success", 19:54:31 DEBUG "data": { 19:54:31 DEBUG "resultType": "vector", 19:54:31 DEBUG "result": [ 19:54:31 DEBUG { 19:54:31 DEBUG "metric": { 19:54:31 DEBUG "pod": "am-767f6b67b5-9zw22" 19:54:31 DEBUG }, 19:54:31 DEBUG "value": [ 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG [...] 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG "pod": "am-767f6b67b5-vbltz" 19:54:31 DEBUG }, 19:54:31 DEBUG "value": [ 19:54:31 DEBUG 1713552870, 19:54:31 DEBUG "2506.8" 19:54:31 DEBUG ] 19:54:31 DEBUG } 19:54:31 DEBUG ] 19:54:31 DEBUG } 19:54:31 DEBUG } 19:54:31 INFO [http_cmd]: http status code OK 19:54:31 DEBUG --- status code --- 19:54:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:31 DEBUG --- http response --- 19:54:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:54:31 DEBUG ----- output ----- 19:54:31 DEBUG { 19:54:31 DEBUG "status": "success", 19:54:31 DEBUG "data": { 19:54:31 DEBUG "resultType": "vector", 19:54:31 DEBUG "result": [ 19:54:31 DEBUG { 19:54:31 DEBUG "metric": { 19:54:31 DEBUG "instance": "10.142.0.107:9100" 19:54:31 DEBUG }, 19:54:31 DEBUG "value": [ 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG [...] 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG "instance": "10.142.0.43:9100" 19:54:31 DEBUG }, 19:54:31 DEBUG "value": [ 19:54:31 DEBUG 1713552870, 19:54:31 DEBUG "0.07233333333332438" 19:54:31 DEBUG ] 19:54:31 DEBUG } 19:54:31 DEBUG ] 19:54:31 DEBUG } 19:54:31 DEBUG } 19:54:31 INFO [http_cmd]: http status code OK 19:54:31 DEBUG --- status code --- 19:54:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:31 DEBUG --- http response --- 19:54:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:54:31 DEBUG ----- output ----- 19:54:31 DEBUG { 19:54:31 DEBUG "status": "success", 19:54:31 DEBUG "data": { 19:54:31 DEBUG "resultType": "vector", 19:54:31 DEBUG "result": [ 19:54:31 DEBUG { 19:54:31 DEBUG "metric": { 19:54:31 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:54:31 DEBUG }, 19:54:31 DEBUG "value": [ 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG [...] 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:54:31 DEBUG }, 19:54:31 DEBUG "value": [ 19:54:31 DEBUG 1713552870, 19:54:31 DEBUG "7233.322585837574" 19:54:31 DEBUG ] 19:54:31 DEBUG } 19:54:31 DEBUG ] 19:54:31 DEBUG } 19:54:31 DEBUG } 19:54:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552871" 19:54:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552871" 19:54:31 INFO [http_cmd]: http status code OK 19:54:31 DEBUG --- status code --- 19:54:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:31 DEBUG --- http response --- 19:54:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:54:31 DEBUG ----- output ----- 19:54:31 DEBUG { 19:54:31 DEBUG "status": "success", 19:54:31 DEBUG "data": { 19:54:31 DEBUG "resultType": "vector", 19:54:31 DEBUG "result": [ 19:54:31 DEBUG { 19:54:31 DEBUG "metric": { 19:54:31 DEBUG "instance": "10.142.0.107:9100" 19:54:31 DEBUG }, 19:54:31 DEBUG "value": [ 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG [...] 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG "instance": "10.142.0.43:9100" 19:54:31 DEBUG }, 19:54:31 DEBUG "value": [ 19:54:31 DEBUG 1713552871, 19:54:31 DEBUG "75409736.66666666" 19:54:31 DEBUG ] 19:54:31 DEBUG } 19:54:31 DEBUG ] 19:54:31 DEBUG } 19:54:31 DEBUG } 19:54:31 INFO [http_cmd]: http status code OK 19:54:31 DEBUG --- status code --- 19:54:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:31 DEBUG --- http response --- 19:54:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:54:31 DEBUG ----- output ----- 19:54:31 DEBUG { 19:54:31 DEBUG "status": "success", 19:54:31 DEBUG "data": { 19:54:31 DEBUG "resultType": "vector", 19:54:31 DEBUG "result": [ 19:54:31 DEBUG { 19:54:31 DEBUG "metric": { 19:54:31 DEBUG "pod": "am-767f6b67b5-9zw22" 19:54:31 DEBUG }, 19:54:31 DEBUG "value": [ 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG [...] 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG "pod": "am-767f6b67b5-vbltz" 19:54:31 DEBUG }, 19:54:31 DEBUG "value": [ 19:54:31 DEBUG 1713552871, 19:54:31 DEBUG "2497.7" 19:54:31 DEBUG ] 19:54:31 DEBUG } 19:54:31 DEBUG ] 19:54:31 DEBUG } 19:54:31 DEBUG } 19:54:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552871" 19:54:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552871" 19:54:31 INFO [http_cmd]: http status code OK 19:54:31 DEBUG --- status code --- 19:54:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:31 DEBUG --- http response --- 19:54:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:54:31 DEBUG ----- output ----- 19:54:31 DEBUG { 19:54:31 DEBUG "status": "success", 19:54:31 DEBUG "data": { 19:54:31 DEBUG "resultType": "vector", 19:54:31 DEBUG "result": [ 19:54:31 DEBUG { 19:54:31 DEBUG "metric": { 19:54:31 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:54:31 DEBUG }, 19:54:31 DEBUG "value": [ 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG [...] 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG "pod": "idm-b85f9d978-wnhxb" 19:54:31 DEBUG }, 19:54:31 DEBUG "value": [ 19:54:31 DEBUG 1713552871, 19:54:31 DEBUG "0" 19:54:31 DEBUG ] 19:54:31 DEBUG } 19:54:31 DEBUG ] 19:54:31 DEBUG } 19:54:31 DEBUG } 19:54:31 INFO [http_cmd]: http status code OK 19:54:31 DEBUG --- status code --- 19:54:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:31 DEBUG --- http response --- 19:54:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:54:31 DEBUG ----- output ----- 19:54:31 DEBUG { 19:54:31 DEBUG "status": "success", 19:54:31 DEBUG "data": { 19:54:31 DEBUG "resultType": "vector", 19:54:31 DEBUG "result": [ 19:54:31 DEBUG { 19:54:31 DEBUG "metric": { 19:54:31 DEBUG "instance": "10.142.0.107:9100" 19:54:31 DEBUG }, 19:54:31 DEBUG "value": [ 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG [...] 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG "instance": "10.142.0.43:9100" 19:54:31 DEBUG }, 19:54:31 DEBUG "value": [ 19:54:31 DEBUG 1713552871, 19:54:31 DEBUG "1974608518144" 19:54:31 DEBUG ] 19:54:31 DEBUG } 19:54:31 DEBUG ] 19:54:31 DEBUG } 19:54:31 DEBUG } 19:54:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552871" 19:54:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552871" 19:54:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713552871" 19:54:31 INFO [http_cmd]: http status code OK 19:54:31 INFO [http_cmd]: http status code OK 19:54:31 DEBUG --- status code --- 19:54:31 DEBUG --- status code --- 19:54:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:31 DEBUG --- http response --- 19:54:31 DEBUG --- http response --- 19:54:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552871" 19:54:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:54:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:54:31 DEBUG ----- output ----- 19:54:31 DEBUG ----- output ----- 19:54:31 DEBUG { 19:54:31 DEBUG { 19:54:31 DEBUG "status": "success", 19:54:31 DEBUG "status": "success", 19:54:31 DEBUG "data": { 19:54:31 DEBUG "data": { 19:54:31 DEBUG "resultType": "vector", 19:54:31 DEBUG "resultType": "vector", 19:54:31 DEBUG "result": [ 19:54:31 DEBUG "result": [ 19:54:31 DEBUG { 19:54:31 DEBUG { 19:54:31 DEBUG "metric": { 19:54:31 DEBUG "metric": { 19:54:31 DEBUG "instance": "10.142.0.107:9100" 19:54:31 DEBUG "pod": "ds-cts-0" 19:54:31 DEBUG }, 19:54:31 DEBUG }, 19:54:31 DEBUG "value": [ 19:54:31 DEBUG "value": [ 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG [...] 19:54:31 DEBUG [...] 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG "instance": "10.142.0.43:9100" 19:54:31 DEBUG "pod": "ds-cts-2" 19:54:31 DEBUG }, 19:54:31 DEBUG }, 19:54:31 DEBUG "value": [ 19:54:31 DEBUG "value": [ 19:54:31 DEBUG 1713552871, 19:54:31 DEBUG 1713552871, 19:54:31 DEBUG "75412077.76666667" 19:54:31 DEBUG "0" 19:54:31 DEBUG ] 19:54:31 DEBUG ] 19:54:31 DEBUG } 19:54:31 DEBUG } 19:54:31 DEBUG ] 19:54:31 DEBUG ] 19:54:31 DEBUG } 19:54:31 DEBUG } 19:54:31 DEBUG } 19:54:31 DEBUG } 19:54:31 INFO [http_cmd]: http status code OK 19:54:31 INFO [http_cmd]: http status code OK 19:54:31 DEBUG --- status code --- 19:54:31 DEBUG --- status code --- 19:54:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:31 DEBUG --- http response --- 19:54:31 DEBUG --- http response --- 19:54:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:54:31 DEBUG ----- output ----- 19:54:31 DEBUG { 19:54:31 DEBUG "status": "success", 19:54:31 DEBUG "data": { 19:54:31 DEBUG "resultType": "vector", 19:54:31 DEBUG "result": [ 19:54:31 DEBUG { 19:54:31 DEBUG "metric": { 19:54:31 DEBUG "pod": "am-767f6b67b5-9zw22" 19:54:31 DEBUG }, 19:54:31 DEBUG "value": [ 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG [...] 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG "pod": "am-767f6b67b5-vbltz" 19:54:31 DEBUG }, 19:54:31 DEBUG "value": [ 19:54:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:54:31 DEBUG 1713552871, 19:54:31 DEBUG ----- output ----- 19:54:31 DEBUG "2498" 19:54:31 DEBUG { 19:54:31 DEBUG ] 19:54:31 DEBUG "status": "success", 19:54:31 DEBUG } 19:54:31 DEBUG "data": { 19:54:31 DEBUG ] 19:54:31 DEBUG "resultType": "vector", 19:54:31 DEBUG } 19:54:31 DEBUG "result": [ 19:54:31 DEBUG } 19:54:31 DEBUG { 19:54:31 DEBUG "metric": { 19:54:31 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:54:31 DEBUG }, 19:54:31 DEBUG "value": [ 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG [...] 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:54:31 DEBUG }, 19:54:31 DEBUG "value": [ 19:54:31 DEBUG 1713552871, 19:54:31 DEBUG "3581775872" 19:54:31 DEBUG ] 19:54:31 DEBUG } 19:54:31 DEBUG ] 19:54:31 DEBUG } 19:54:31 DEBUG } 19:54:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552871" 19:54:31 INFO [http_cmd]: http status code OK 19:54:31 DEBUG --- status code --- 19:54:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:31 DEBUG --- http response --- 19:54:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:54:31 DEBUG ----- output ----- 19:54:31 DEBUG { 19:54:31 DEBUG "status": "success", 19:54:31 DEBUG "data": { 19:54:31 DEBUG "resultType": "vector", 19:54:31 DEBUG "result": [ 19:54:31 DEBUG { 19:54:31 DEBUG "metric": { 19:54:31 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:54:31 DEBUG }, 19:54:31 DEBUG "value": [ 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG [...] 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG 19:54:31 DEBUG "pod": "idm-b85f9d978-wnhxb" 19:54:31 DEBUG }, 19:54:31 DEBUG "value": [ 19:54:31 DEBUG 1713552871, 19:54:31 DEBUG "0.003232011524755532" 19:54:31 DEBUG ] 19:54:31 DEBUG } 19:54:31 DEBUG ] 19:54:31 DEBUG } 19:54:31 DEBUG } 19:54:32 INFO 19:54:32 INFO [loop_until]: kubectl --namespace=xlou top pods 19:54:32 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:54:32 INFO [loop_until]: OK (rc = 0) 19:54:32 DEBUG --- stdout --- 19:54:32 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 16345m 8997Mi am-767f6b67b5-qcqz2 16656m 8811Mi am-767f6b67b5-vbltz 17324m 8994Mi ds-cts-0 16685m 10952Mi ds-cts-1 15350m 11200Mi ds-cts-2 17990m 10958Mi ds-idrepo-0 7242m 3677Mi ds-idrepo-1 6994m 3537Mi ds-idrepo-2 7445m 3493Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 4m 1217Mi lodemon-f8459c597-6tj6f 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8077m 3321Mi 19:54:32 DEBUG --- stderr --- 19:54:32 DEBUG 19:54:33 INFO 19:54:33 INFO [loop_until]: kubectl --namespace=xlou top node 19:54:33 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:54:33 INFO [loop_until]: OK (rc = 0) 19:54:33 DEBUG --- stdout --- 19:54:33 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 70m 0% 6257Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7399m 24% 7275Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 15949m 53% 11706Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 3350Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 16969m 56% 13938Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 141m 0% 5217Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 16374m 54% 11704Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15208m 50% 2986Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 18160m 60% 11209Mi 10% gke-xlou-cdm-ds-0f601751-j488 7639m 25% 6739Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 15706m 52% 15948Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 7441m 24% 7438Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 17540m 58% 13415Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 7939m 49% 7073Mi 12% 19:54:33 DEBUG --- stderr --- 19:54:33 DEBUG 19:55:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552930" 19:55:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552930" 19:55:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552930" 19:55:30 INFO [http_cmd]: http status code OK 19:55:30 DEBUG --- status code --- 19:55:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:30 DEBUG --- http response --- 19:55:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:55:30 DEBUG ----- output ----- 19:55:30 DEBUG { 19:55:30 DEBUG "status": "success", 19:55:30 DEBUG "data": { 19:55:30 DEBUG "resultType": "vector", 19:55:30 DEBUG "result": [ 19:55:30 DEBUG { 19:55:30 DEBUG "metric": { 19:55:30 DEBUG "pod": "ds-cts-0" 19:55:30 DEBUG }, 19:55:30 DEBUG "value": [ 19:55:30 DEBUG 19:55:30 DEBUG 19:55:30 DEBUG 19:55:30 DEBUG [...] 19:55:30 DEBUG 19:55:30 DEBUG 19:55:30 DEBUG 19:55:30 DEBUG "pod": "ds-cts-2" 19:55:30 DEBUG }, 19:55:30 DEBUG "value": [ 19:55:30 DEBUG 1713552930, 19:55:30 DEBUG "0.00003333333333333336" 19:55:30 DEBUG ] 19:55:30 DEBUG } 19:55:30 DEBUG ] 19:55:30 DEBUG } 19:55:30 DEBUG } 19:55:30 INFO [http_cmd]: http status code OK 19:55:30 DEBUG --- status code --- 19:55:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:30 DEBUG --- http response --- 19:55:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:55:30 DEBUG ----- output ----- 19:55:30 DEBUG { 19:55:30 DEBUG "status": "success", 19:55:30 DEBUG "data": { 19:55:30 DEBUG "resultType": "vector", 19:55:30 DEBUG "result": [ 19:55:30 DEBUG { 19:55:30 DEBUG "metric": { 19:55:30 DEBUG "pod": "ds-idrepo-0" 19:55:30 DEBUG }, 19:55:30 DEBUG "value": [ 19:55:30 DEBUG 19:55:30 DEBUG 19:55:30 DEBUG 19:55:30 DEBUG [...] 19:55:30 DEBUG 19:55:30 DEBUG 19:55:30 DEBUG 19:55:30 DEBUG "pod": "ds-idrepo-2" 19:55:30 DEBUG }, 19:55:30 DEBUG "value": [ 19:55:30 DEBUG 1713552930, 19:55:30 DEBUG "0" 19:55:30 DEBUG ] 19:55:30 DEBUG } 19:55:30 DEBUG ] 19:55:30 DEBUG } 19:55:30 DEBUG } 19:55:30 INFO [http_cmd]: http status code OK 19:55:30 DEBUG --- status code --- 19:55:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:30 DEBUG --- http response --- 19:55:30 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:55:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552930" 19:55:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713552930" 19:55:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713552930" 19:55:30 INFO [http_cmd]: http status code OK 19:55:30 DEBUG --- status code --- 19:55:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:30 DEBUG --- http response --- 19:55:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:55:30 DEBUG ----- output ----- 19:55:30 DEBUG { 19:55:30 DEBUG "status": "success", 19:55:30 DEBUG "data": { 19:55:30 DEBUG "resultType": "vector", 19:55:30 DEBUG "result": [ 19:55:30 DEBUG { 19:55:30 DEBUG "metric": { 19:55:30 DEBUG "pod": "ds-idrepo-0" 19:55:30 DEBUG }, 19:55:30 DEBUG "value": [ 19:55:30 DEBUG 19:55:30 DEBUG 19:55:30 DEBUG 19:55:30 DEBUG [...] 19:55:30 DEBUG 19:55:30 DEBUG 19:55:30 DEBUG 19:55:30 DEBUG "pod": "ds-cts-2" 19:55:30 DEBUG }, 19:55:30 DEBUG "value": [ 19:55:30 DEBUG 1713552930, 19:55:30 DEBUG "0" 19:55:30 DEBUG ] 19:55:30 DEBUG } 19:55:30 DEBUG ] 19:55:30 DEBUG } 19:55:30 DEBUG } 19:55:30 INFO [http_cmd]: http status code OK 19:55:30 DEBUG --- status code --- 19:55:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:30 DEBUG --- http response --- 19:55:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 19:55:30 DEBUG ----- output ----- 19:55:30 DEBUG { 19:55:30 DEBUG "status": "success", 19:55:30 DEBUG "data": { 19:55:30 DEBUG "resultType": "vector", 19:55:30 DEBUG "result": [ 19:55:30 DEBUG { 19:55:30 DEBUG "metric": { 19:55:30 DEBUG "pod": "ds-idrepo-0" 19:55:30 DEBUG }, 19:55:30 DEBUG "value": [ 19:55:30 DEBUG 19:55:30 DEBUG 19:55:30 DEBUG 19:55:30 DEBUG [...] 19:55:30 DEBUG 19:55:30 DEBUG 19:55:30 DEBUG 19:55:30 DEBUG "pod": "ds-idrepo-2" 19:55:30 DEBUG }, 19:55:30 DEBUG "value": [ 19:55:30 DEBUG 1713552930, 19:55:30 DEBUG "4003.4208918753816" 19:55:30 DEBUG ] 19:55:30 DEBUG } 19:55:30 DEBUG ] 19:55:30 DEBUG } 19:55:30 DEBUG } 19:55:30 ERROR [http_cmd]: ERROR 19:55:30 ERROR --- status code --- 19:55:30 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:55:30 ERROR --- http response --- 19:55:30 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:55:30 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:55:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552930" 19:55:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552930" 19:55:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713552930" 19:55:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552930" 19:55:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552930" 19:55:30 INFO [http_cmd]: http status code OK 19:55:30 INFO [http_cmd]: http status code OK 19:55:30 DEBUG --- status code --- 19:55:30 DEBUG --- status code --- 19:55:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:30 DEBUG --- http response --- 19:55:30 DEBUG --- http response --- 19:55:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:55:30 DEBUG ----- output ----- 19:55:30 DEBUG { 19:55:30 DEBUG "status": "success", 19:55:30 DEBUG "data": { 19:55:30 DEBUG "resultType": "vector", 19:55:30 DEBUG "result": [ 19:55:30 DEBUG { 19:55:30 DEBUG "metric": { 19:55:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:55:30 DEBUG }, 19:55:30 DEBUG "value": [ 19:55:30 DEBUG 19:55:30 DEBUG 19:55:30 DEBUG 19:55:30 DEBUG [...] 19:55:30 DEBUG 19:55:30 DEBUG 19:55:30 DEBUG 19:55:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:55:30 DEBUG }, 19:55:30 DEBUG "value": [ 19:55:30 DEBUG 1713552930, 19:55:30 DEBUG "450256.5" 19:55:30 DEBUG ] 19:55:30 DEBUG } 19:55:30 DEBUG ] 19:55:30 DEBUG } 19:55:30 DEBUG } 19:55:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:55:30 DEBUG ----- output ----- 19:55:30 DEBUG { 19:55:30 DEBUG "status": "success", 19:55:30 DEBUG "data": { 19:55:30 DEBUG "resultType": "vector", 19:55:30 DEBUG "result": [ 19:55:30 DEBUG { 19:55:30 DEBUG "metric": { 19:55:30 DEBUG "pod": "am-767f6b67b5-9zw22" 19:55:30 DEBUG }, 19:55:30 DEBUG "value": [ 19:55:30 DEBUG 19:55:30 DEBUG 19:55:30 DEBUG 19:55:30 DEBUG [...] 19:55:30 DEBUG 19:55:30 DEBUG 19:55:30 DEBUG 19:55:30 DEBUG "pod": "am-767f6b67b5-vbltz" 19:55:30 DEBUG }, 19:55:30 DEBUG "value": [ 19:55:30 DEBUG 1713552930, 19:55:30 DEBUG "0" 19:55:30 DEBUG ] 19:55:30 DEBUG } 19:55:30 DEBUG ] 19:55:30 DEBUG } 19:55:30 DEBUG } 19:55:30 INFO [http_cmd]: http status code OK 19:55:30 DEBUG --- status code --- 19:55:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:30 DEBUG --- http response --- 19:55:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:55:30 DEBUG ----- output ----- 19:55:30 DEBUG { 19:55:30 DEBUG "status": "success", 19:55:30 DEBUG "data": { 19:55:30 DEBUG "resultType": "vector", 19:55:30 DEBUG "result": [ 19:55:30 DEBUG { 19:55:30 DEBUG "metric": { 19:55:30 DEBUG "instance": "10.142.0.107:9100" 19:55:30 DEBUG }, 19:55:30 DEBUG "value": [ 19:55:30 DEBUG 19:55:30 DEBUG 19:55:30 DEBUG 19:55:30 DEBUG [...] 19:55:30 DEBUG 19:55:30 DEBUG 19:55:30 DEBUG 19:55:30 DEBUG "instance": "10.142.0.43:9100" 19:55:30 DEBUG }, 19:55:30 DEBUG "value": [ 19:55:30 DEBUG 1713552930, 19:55:30 INFO [http_cmd]: http status code OK 19:55:30 DEBUG "0.2317999999999775" 19:55:30 DEBUG --- status code --- 19:55:30 DEBUG ] 19:55:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:30 DEBUG } 19:55:30 DEBUG --- http response --- 19:55:30 DEBUG ] 19:55:30 DEBUG } 19:55:30 DEBUG } 19:55:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:55:30 DEBUG ----- output ----- 19:55:30 DEBUG { 19:55:30 DEBUG "status": "success", 19:55:30 DEBUG "data": { 19:55:30 DEBUG "resultType": "vector", 19:55:30 DEBUG "result": [ 19:55:30 DEBUG { 19:55:30 DEBUG "metric": { 19:55:30 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:55:30 DEBUG }, 19:55:30 DEBUG "value": [ 19:55:30 DEBUG 19:55:30 DEBUG 19:55:30 DEBUG 19:55:30 DEBUG [...] 19:55:30 DEBUG 19:55:30 DEBUG 19:55:30 DEBUG 19:55:30 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:55:30 DEBUG }, 19:55:30 DEBUG "value": [ 19:55:30 DEBUG 1713552930, 19:55:30 DEBUG "8.045930122900328" 19:55:30 DEBUG ] 19:55:30 DEBUG } 19:55:30 DEBUG ] 19:55:30 DEBUG } 19:55:30 DEBUG } 19:55:31 INFO [http_cmd]: http status code OK 19:55:31 DEBUG --- status code --- 19:55:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:31 DEBUG --- http response --- 19:55:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:55:31 DEBUG ----- output ----- 19:55:31 DEBUG { 19:55:31 DEBUG "status": "success", 19:55:31 DEBUG "data": { 19:55:31 DEBUG "resultType": "vector", 19:55:31 DEBUG "result": [ 19:55:31 DEBUG { 19:55:31 DEBUG "metric": { 19:55:31 DEBUG "pod": "ds-idrepo-0" 19:55:31 DEBUG }, 19:55:31 DEBUG "value": [ 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG [...] 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG "pod": "ds-idrepo-2" 19:55:31 DEBUG }, 19:55:31 DEBUG "value": [ 19:55:31 DEBUG 1713552930, 19:55:31 DEBUG "0" 19:55:31 DEBUG ] 19:55:31 DEBUG } 19:55:31 DEBUG ] 19:55:31 DEBUG } 19:55:31 DEBUG } 19:55:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552931" 19:55:31 INFO [http_cmd]: http status code OK 19:55:31 DEBUG --- status code --- 19:55:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:31 DEBUG --- http response --- 19:55:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:55:31 DEBUG ----- output ----- 19:55:31 DEBUG { 19:55:31 DEBUG "status": "success", 19:55:31 DEBUG "data": { 19:55:31 DEBUG "resultType": "vector", 19:55:31 DEBUG "result": [ 19:55:31 DEBUG { 19:55:31 DEBUG "metric": { 19:55:31 DEBUG "pod": "ds-cts-0" 19:55:31 DEBUG }, 19:55:31 DEBUG "value": [ 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG [...] 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG "pod": "ds-cts-2" 19:55:31 DEBUG }, 19:55:31 DEBUG "value": [ 19:55:31 DEBUG 1713552931, 19:55:31 DEBUG "30" 19:55:31 DEBUG ] 19:55:31 DEBUG } 19:55:31 DEBUG ] 19:55:31 DEBUG } 19:55:31 DEBUG } 19:55:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713552931" 19:55:31 INFO [http_cmd]: http status code OK 19:55:31 DEBUG --- status code --- 19:55:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:31 DEBUG --- http response --- 19:55:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:55:31 DEBUG ----- output ----- 19:55:31 DEBUG { 19:55:31 DEBUG "status": "success", 19:55:31 DEBUG "data": { 19:55:31 DEBUG "resultType": "vector", 19:55:31 DEBUG "result": [ 19:55:31 DEBUG { 19:55:31 DEBUG "metric": { 19:55:31 DEBUG "pod": "am-767f6b67b5-9zw22" 19:55:31 DEBUG }, 19:55:31 DEBUG "value": [ 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG [...] 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG "pod": "am-767f6b67b5-vbltz" 19:55:31 DEBUG }, 19:55:31 DEBUG "value": [ 19:55:31 DEBUG 1713552931, 19:55:31 DEBUG "0.014" 19:55:31 DEBUG ] 19:55:31 DEBUG } 19:55:31 DEBUG ] 19:55:31 DEBUG } 19:55:31 DEBUG } 19:55:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552931" 19:55:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552931" 19:55:31 INFO [http_cmd]: http status code OK 19:55:31 DEBUG --- status code --- 19:55:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:31 DEBUG --- http response --- 19:55:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:55:31 DEBUG ----- output ----- 19:55:31 DEBUG { 19:55:31 DEBUG "status": "success", 19:55:31 DEBUG "data": { 19:55:31 DEBUG "resultType": "vector", 19:55:31 DEBUG "result": [ 19:55:31 DEBUG { 19:55:31 DEBUG "metric": { 19:55:31 DEBUG "pod": "ds-idrepo-0" 19:55:31 DEBUG }, 19:55:31 DEBUG "value": [ 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG [...] 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG "pod": "ds-idrepo-2" 19:55:31 DEBUG }, 19:55:31 DEBUG "value": [ 19:55:31 DEBUG 1713552931, 19:55:31 DEBUG "0" 19:55:31 DEBUG ] 19:55:31 DEBUG } 19:55:31 DEBUG ] 19:55:31 DEBUG } 19:55:31 DEBUG } 19:55:31 INFO [http_cmd]: http status code OK 19:55:31 DEBUG --- status code --- 19:55:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:31 DEBUG --- http response --- 19:55:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:55:31 DEBUG ----- output ----- 19:55:31 DEBUG { 19:55:31 DEBUG "status": "success", 19:55:31 DEBUG "data": { 19:55:31 DEBUG "resultType": "vector", 19:55:31 DEBUG "result": [ 19:55:31 DEBUG { 19:55:31 DEBUG "metric": { 19:55:31 DEBUG "pod": "am-767f6b67b5-9zw22" 19:55:31 DEBUG }, 19:55:31 DEBUG "value": [ 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG [...] 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG "pod": "am-767f6b67b5-vbltz" 19:55:31 DEBUG }, 19:55:31 DEBUG "value": [ 19:55:31 DEBUG 1713552931, 19:55:31 DEBUG "2499.233333333333" 19:55:31 DEBUG ] 19:55:31 DEBUG } 19:55:31 DEBUG ] 19:55:31 DEBUG } 19:55:31 DEBUG } 19:55:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552931" 19:55:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552931" 19:55:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552931" 19:55:31 INFO [http_cmd]: http status code OK 19:55:31 DEBUG --- status code --- 19:55:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:31 DEBUG --- http response --- 19:55:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:55:31 DEBUG ----- output ----- 19:55:31 DEBUG { 19:55:31 DEBUG "status": "success", 19:55:31 DEBUG "data": { 19:55:31 DEBUG "resultType": "vector", 19:55:31 DEBUG "result": [ 19:55:31 DEBUG { 19:55:31 DEBUG "metric": { 19:55:31 DEBUG "pod": "am-767f6b67b5-9zw22" 19:55:31 DEBUG }, 19:55:31 DEBUG "value": [ 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG [...] 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG "pod": "am-767f6b67b5-vbltz" 19:55:31 DEBUG }, 19:55:31 DEBUG "value": [ 19:55:31 DEBUG 1713552931, 19:55:31 DEBUG "2492.633333333333" 19:55:31 DEBUG ] 19:55:31 DEBUG } 19:55:31 DEBUG ] 19:55:31 DEBUG } 19:55:31 DEBUG } 19:55:31 INFO [http_cmd]: http status code OK 19:55:31 DEBUG --- status code --- 19:55:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:31 DEBUG --- http response --- 19:55:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:55:31 DEBUG ----- output ----- 19:55:31 DEBUG { 19:55:31 DEBUG "status": "success", 19:55:31 DEBUG "data": { 19:55:31 DEBUG "resultType": "vector", 19:55:31 DEBUG "result": [ 19:55:31 DEBUG { 19:55:31 DEBUG "metric": { 19:55:31 DEBUG "pod": "ds-idrepo-0" 19:55:31 DEBUG }, 19:55:31 DEBUG "value": [ 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG [...] 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG "pod": "ds-idrepo-2" 19:55:31 DEBUG }, 19:55:31 DEBUG "value": [ 19:55:31 DEBUG 1713552931, 19:55:31 DEBUG "135387187.29383016" 19:55:31 DEBUG ] 19:55:31 DEBUG } 19:55:31 DEBUG ] 19:55:31 DEBUG } 19:55:31 DEBUG } 19:55:31 INFO [http_cmd]: http status code OK 19:55:31 DEBUG --- status code --- 19:55:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:31 DEBUG --- http response --- 19:55:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:55:31 DEBUG ----- output ----- 19:55:31 DEBUG { 19:55:31 DEBUG "status": "success", 19:55:31 DEBUG "data": { 19:55:31 DEBUG "resultType": "vector", 19:55:31 DEBUG "result": [ 19:55:31 DEBUG { 19:55:31 DEBUG "metric": { 19:55:31 DEBUG "instance": "10.142.0.107:9100" 19:55:31 DEBUG }, 19:55:31 DEBUG "value": [ 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG [...] 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG "instance": "10.142.0.43:9100" 19:55:31 DEBUG }, 19:55:31 DEBUG "value": [ 19:55:31 DEBUG 1713552931, 19:55:31 DEBUG "0.10666666666667196" 19:55:31 DEBUG ] 19:55:31 DEBUG } 19:55:31 DEBUG ] 19:55:31 DEBUG } 19:55:31 DEBUG } 19:55:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552931" 19:55:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552931" 19:55:31 INFO [http_cmd]: http status code OK 19:55:31 DEBUG --- status code --- 19:55:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:31 DEBUG --- http response --- 19:55:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:55:31 DEBUG ----- output ----- 19:55:31 DEBUG { 19:55:31 DEBUG "status": "success", 19:55:31 DEBUG "data": { 19:55:31 DEBUG "resultType": "vector", 19:55:31 DEBUG "result": [ 19:55:31 DEBUG { 19:55:31 DEBUG "metric": { 19:55:31 DEBUG "pod": "am-767f6b67b5-9zw22" 19:55:31 DEBUG }, 19:55:31 DEBUG "value": [ 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG [...] 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG "pod": "am-767f6b67b5-vbltz" 19:55:31 DEBUG }, 19:55:31 DEBUG "value": [ 19:55:31 DEBUG 1713552931, 19:55:31 DEBUG "2499.133333333333" 19:55:31 DEBUG ] 19:55:31 DEBUG } 19:55:31 DEBUG ] 19:55:31 DEBUG } 19:55:31 DEBUG } 19:55:31 INFO [http_cmd]: http status code OK 19:55:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552931" 19:55:31 DEBUG --- status code --- 19:55:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:31 DEBUG --- http response --- 19:55:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:55:31 DEBUG ----- output ----- 19:55:31 DEBUG { 19:55:31 DEBUG "status": "success", 19:55:31 DEBUG "data": { 19:55:31 DEBUG "resultType": "vector", 19:55:31 DEBUG "result": [ 19:55:31 DEBUG { 19:55:31 DEBUG "metric": { 19:55:31 DEBUG "instance": "10.142.0.107:9100" 19:55:31 DEBUG }, 19:55:31 DEBUG "value": [ 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG [...] 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG "instance": "10.142.0.43:9100" 19:55:31 DEBUG }, 19:55:31 DEBUG "value": [ 19:55:31 DEBUG 1713552931, 19:55:31 DEBUG "75421260.80000001" 19:55:31 DEBUG ] 19:55:31 DEBUG } 19:55:31 DEBUG ] 19:55:31 DEBUG } 19:55:31 DEBUG } 19:55:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552931" 19:55:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713552931" 19:55:31 INFO [http_cmd]: http status code OK 19:55:31 DEBUG --- status code --- 19:55:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:31 DEBUG --- http response --- 19:55:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 19:55:31 DEBUG ----- output ----- 19:55:31 DEBUG { 19:55:31 DEBUG "status": "success", 19:55:31 DEBUG "data": { 19:55:31 DEBUG "resultType": "vector", 19:55:31 DEBUG "result": [ 19:55:31 DEBUG { 19:55:31 DEBUG "metric": { 19:55:31 DEBUG "pod": "ds-idrepo-0" 19:55:31 DEBUG }, 19:55:31 DEBUG "value": [ 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG [...] 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG "pod": "ds-idrepo-2" 19:55:31 DEBUG }, 19:55:31 DEBUG "value": [ 19:55:31 DEBUG 1713552931, 19:55:31 DEBUG "0" 19:55:31 DEBUG ] 19:55:31 DEBUG } 19:55:31 DEBUG ] 19:55:31 DEBUG } 19:55:31 DEBUG } 19:55:31 INFO [http_cmd]: http status code OK 19:55:31 DEBUG --- status code --- 19:55:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:31 DEBUG --- http response --- 19:55:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:55:31 DEBUG ----- output ----- 19:55:31 DEBUG { 19:55:31 DEBUG "status": "success", 19:55:31 DEBUG "data": { 19:55:31 DEBUG "resultType": "vector", 19:55:31 DEBUG "result": [ 19:55:31 DEBUG { 19:55:31 DEBUG "metric": { 19:55:31 DEBUG "instance": "10.142.0.107:9100" 19:55:31 DEBUG }, 19:55:31 DEBUG "value": [ 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG [...] 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG "instance": "10.142.0.43:9100" 19:55:31 DEBUG }, 19:55:31 DEBUG "value": [ 19:55:31 DEBUG 1713552931, 19:55:31 DEBUG "1978400244736" 19:55:31 DEBUG ] 19:55:31 DEBUG } 19:55:31 DEBUG ] 19:55:31 DEBUG } 19:55:31 DEBUG } 19:55:31 INFO [http_cmd]: http status code OK 19:55:31 DEBUG --- status code --- 19:55:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:31 DEBUG --- http response --- 19:55:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:55:31 DEBUG ----- output ----- 19:55:31 DEBUG { 19:55:31 DEBUG "status": "success", 19:55:31 DEBUG "data": { 19:55:31 DEBUG "resultType": "vector", 19:55:31 DEBUG "result": [ 19:55:31 DEBUG { 19:55:31 DEBUG "metric": { 19:55:31 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:55:31 DEBUG }, 19:55:31 DEBUG "value": [ 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG [...] 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:55:31 DEBUG }, 19:55:31 DEBUG "value": [ 19:55:31 DEBUG 1713552931, 19:55:31 DEBUG "3583799296" 19:55:31 DEBUG ] 19:55:31 DEBUG } 19:55:31 DEBUG ] 19:55:31 DEBUG } 19:55:31 DEBUG } 19:55:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552931" 19:55:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552931" 19:55:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552931" 19:55:31 INFO [http_cmd]: http status code OK 19:55:31 INFO [http_cmd]: http status code OK 19:55:31 DEBUG --- status code --- 19:55:31 DEBUG --- status code --- 19:55:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:31 DEBUG --- http response --- 19:55:31 DEBUG --- http response --- 19:55:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:55:31 DEBUG ----- output ----- 19:55:31 DEBUG { 19:55:31 DEBUG "status": "success", 19:55:31 DEBUG "data": { 19:55:31 DEBUG "resultType": "vector", 19:55:31 DEBUG "result": [ 19:55:31 DEBUG { 19:55:31 DEBUG "metric": { 19:55:31 DEBUG "pod": "ds-cts-0" 19:55:31 DEBUG }, 19:55:31 DEBUG "value": [ 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG [...] 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG "pod": "ds-cts-2" 19:55:31 DEBUG }, 19:55:31 DEBUG "value": [ 19:55:31 DEBUG 1713552931, 19:55:31 DEBUG "0" 19:55:31 DEBUG ] 19:55:31 DEBUG } 19:55:31 DEBUG ] 19:55:31 DEBUG } 19:55:31 DEBUG } 19:55:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:55:31 DEBUG ----- output ----- 19:55:31 DEBUG { 19:55:31 DEBUG "status": "success", 19:55:31 DEBUG "data": { 19:55:31 DEBUG "resultType": "vector", 19:55:31 DEBUG "result": [ 19:55:31 DEBUG { 19:55:31 DEBUG "metric": { 19:55:31 DEBUG "instance": "10.142.0.107:9100" 19:55:31 DEBUG }, 19:55:31 DEBUG "value": [ 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG [...] 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG "instance": "10.142.0.43:9100" 19:55:31 DEBUG }, 19:55:31 DEBUG "value": [ 19:55:31 DEBUG 1713552931, 19:55:31 DEBUG "75424023.93333334" 19:55:31 DEBUG ] 19:55:31 DEBUG } 19:55:31 DEBUG ] 19:55:31 DEBUG } 19:55:31 DEBUG } 19:55:31 INFO [http_cmd]: http status code OK 19:55:31 DEBUG --- status code --- 19:55:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:31 DEBUG --- http response --- 19:55:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:55:31 DEBUG ----- output ----- 19:55:31 DEBUG { 19:55:31 DEBUG "status": "success", 19:55:31 DEBUG "data": { 19:55:31 DEBUG "resultType": "vector", 19:55:31 DEBUG "result": [ 19:55:31 DEBUG { 19:55:31 DEBUG "metric": { 19:55:31 DEBUG "pod": "am-767f6b67b5-9zw22" 19:55:31 DEBUG }, 19:55:31 DEBUG "value": [ 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG [...] 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG "pod": "am-767f6b67b5-vbltz" 19:55:31 DEBUG }, 19:55:31 DEBUG "value": [ 19:55:31 DEBUG 1713552931, 19:55:31 DEBUG "2499.2" 19:55:31 DEBUG ] 19:55:31 DEBUG } 19:55:31 DEBUG ] 19:55:31 DEBUG } 19:55:31 DEBUG } 19:55:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552931" 19:55:31 INFO [http_cmd]: http status code OK 19:55:31 DEBUG --- status code --- 19:55:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:31 DEBUG --- http response --- 19:55:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:55:31 DEBUG ----- output ----- 19:55:31 DEBUG { 19:55:31 DEBUG "status": "success", 19:55:31 DEBUG "data": { 19:55:31 DEBUG "resultType": "vector", 19:55:31 DEBUG "result": [ 19:55:31 DEBUG { 19:55:31 DEBUG "metric": { 19:55:31 DEBUG "pod": "ds-idrepo-0" 19:55:31 DEBUG }, 19:55:31 DEBUG "value": [ 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG [...] 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG 19:55:31 DEBUG "pod": "ds-idrepo-2" 19:55:31 DEBUG }, 19:55:31 DEBUG "value": [ 19:55:31 DEBUG 1713552931, 19:55:31 DEBUG "7.3375529627372655" 19:55:31 DEBUG ] 19:55:31 DEBUG } 19:55:31 DEBUG ] 19:55:31 DEBUG } 19:55:31 DEBUG } 19:55:32 INFO 19:55:32 INFO [loop_until]: kubectl --namespace=xlou top pods 19:55:32 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:55:32 INFO [loop_until]: OK (rc = 0) 19:55:32 DEBUG --- stdout --- 19:55:32 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 16366m 9002Mi am-767f6b67b5-qcqz2 16690m 8813Mi am-767f6b67b5-vbltz 17663m 8996Mi ds-cts-0 16678m 10954Mi ds-cts-1 15973m 11186Mi ds-cts-2 17641m 10964Mi ds-idrepo-0 7059m 3677Mi ds-idrepo-1 6986m 3537Mi ds-idrepo-2 7382m 3492Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8150m 3324Mi 19:55:32 DEBUG --- stderr --- 19:55:32 DEBUG 19:55:33 INFO 19:55:33 INFO [loop_until]: kubectl --namespace=xlou top node 19:55:33 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:55:33 INFO [loop_until]: OK (rc = 0) 19:55:33 DEBUG --- stdout --- 19:55:33 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 74m 0% 6261Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7413m 24% 7276Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16113m 53% 11702Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 88m 0% 3346Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 16888m 56% 13938Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 161m 0% 5196Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 16368m 54% 11694Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15498m 51% 2990Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 18187m 60% 11215Mi 10% gke-xlou-cdm-ds-0f601751-j488 7598m 25% 6736Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 15863m 53% 15958Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 7441m 24% 7434Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 17588m 58% 13384Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 8194m 51% 7074Mi 12% 19:55:33 DEBUG --- stderr --- 19:55:33 DEBUG 19:56:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552990" 19:56:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552990" 19:56:30 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552990" 19:56:30 INFO [http_cmd]: http status code OK 19:56:30 DEBUG --- status code --- 19:56:30 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:30 DEBUG --- http response --- 19:56:30 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:56:30 DEBUG ----- output ----- 19:56:30 DEBUG { 19:56:30 DEBUG "status": "success", 19:56:30 DEBUG "data": { 19:56:30 DEBUG "resultType": "vector", 19:56:30 DEBUG "result": [ 19:56:30 DEBUG { 19:56:30 DEBUG "metric": { 19:56:30 DEBUG "pod": "ds-cts-0" 19:56:30 DEBUG }, 19:56:30 DEBUG "value": [ 19:56:30 DEBUG 19:56:30 DEBUG 19:56:30 DEBUG 19:56:30 DEBUG [...] 19:56:30 DEBUG 19:56:30 DEBUG 19:56:30 DEBUG 19:56:30 DEBUG "pod": "ds-cts-2" 19:56:30 DEBUG }, 19:56:30 DEBUG "value": [ 19:56:30 DEBUG 1713552990, 19:56:30 DEBUG "0.0003666666666666666" 19:56:30 DEBUG ] 19:56:30 DEBUG } 19:56:30 DEBUG ] 19:56:30 DEBUG } 19:56:30 DEBUG } 19:56:31 INFO [http_cmd]: http status code OK 19:56:31 INFO [http_cmd]: http status code OK 19:56:31 DEBUG --- status code --- 19:56:31 DEBUG --- status code --- 19:56:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:31 DEBUG --- http response --- 19:56:31 DEBUG --- http response --- 19:56:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:56:31 DEBUG ----- output ----- 19:56:31 DEBUG { 19:56:31 DEBUG "status": "success", 19:56:31 DEBUG "data": { 19:56:31 DEBUG "resultType": "vector", 19:56:31 DEBUG "result": [ 19:56:31 DEBUG { 19:56:31 DEBUG "metric": { 19:56:31 DEBUG "pod": "ds-idrepo-0" 19:56:31 DEBUG }, 19:56:31 DEBUG "value": [ 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG [...] 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG "pod": "ds-idrepo-2" 19:56:31 DEBUG }, 19:56:31 DEBUG "value": [ 19:56:31 DEBUG 1713552990, 19:56:31 DEBUG "0" 19:56:31 DEBUG ] 19:56:31 DEBUG } 19:56:31 DEBUG ] 19:56:31 DEBUG } 19:56:31 DEBUG } 19:56:31 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:56:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713552991" 19:56:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552991" 19:56:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552991" 19:56:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713552991" 19:56:31 INFO [http_cmd]: http status code OK 19:56:31 DEBUG --- status code --- 19:56:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:31 DEBUG --- http response --- 19:56:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:56:31 DEBUG ----- output ----- 19:56:31 DEBUG { 19:56:31 DEBUG "status": "success", 19:56:31 DEBUG "data": { 19:56:31 DEBUG "resultType": "vector", 19:56:31 DEBUG "result": [ 19:56:31 DEBUG { 19:56:31 DEBUG "metric": { 19:56:31 DEBUG "pod": "ds-idrepo-0" 19:56:31 DEBUG }, 19:56:31 DEBUG "value": [ 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG [...] 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG "pod": "ds-cts-2" 19:56:31 DEBUG }, 19:56:31 DEBUG "value": [ 19:56:31 DEBUG 1713552991, 19:56:31 DEBUG "0" 19:56:31 DEBUG ] 19:56:31 DEBUG } 19:56:31 DEBUG ] 19:56:31 DEBUG } 19:56:31 DEBUG } 19:56:31 INFO [http_cmd]: http status code OK 19:56:31 DEBUG --- status code --- 19:56:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:31 DEBUG --- http response --- 19:56:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:56:31 DEBUG ----- output ----- 19:56:31 DEBUG { 19:56:31 DEBUG "status": "success", 19:56:31 DEBUG "data": { 19:56:31 DEBUG "resultType": "vector", 19:56:31 DEBUG "result": [ 19:56:31 DEBUG { 19:56:31 DEBUG "metric": { 19:56:31 DEBUG "pod": "am-767f6b67b5-9zw22" 19:56:31 DEBUG }, 19:56:31 DEBUG "value": [ 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG [...] 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG "pod": "am-767f6b67b5-vbltz" 19:56:31 DEBUG }, 19:56:31 DEBUG "value": [ 19:56:31 DEBUG 1713552991, 19:56:31 DEBUG "450239" 19:56:31 DEBUG ] 19:56:31 DEBUG } 19:56:31 DEBUG ] 19:56:31 DEBUG } 19:56:31 DEBUG } 19:56:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713552991" 19:56:31 INFO [http_cmd]: http status code OK 19:56:31 DEBUG --- status code --- 19:56:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:31 DEBUG --- http response --- 19:56:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:56:31 DEBUG ----- output ----- 19:56:31 DEBUG { 19:56:31 DEBUG "status": "success", 19:56:31 DEBUG "data": { 19:56:31 DEBUG "resultType": "vector", 19:56:31 DEBUG "result": [ 19:56:31 DEBUG { 19:56:31 DEBUG "metric": { 19:56:31 DEBUG "pod": "ds-idrepo-1" 19:56:31 DEBUG }, 19:56:31 DEBUG "value": [ 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG [...] 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG "pod": "am-767f6b67b5-qcqz2" 19:56:31 DEBUG }, 19:56:31 DEBUG "value": [ 19:56:31 DEBUG 1713552991, 19:56:31 DEBUG "490.36274392433853" 19:56:31 DEBUG ] 19:56:31 DEBUG } 19:56:31 DEBUG ] 19:56:31 DEBUG } 19:56:31 DEBUG } 19:56:31 ERROR [http_cmd]: ERROR 19:56:31 ERROR --- status code --- 19:56:31 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:56:31 ERROR --- http response --- 19:56:31 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:56:31 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:56:31 INFO [http_cmd]: http status code OK 19:56:31 DEBUG --- status code --- 19:56:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:31 DEBUG --- http response --- 19:56:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:56:31 DEBUG ----- output ----- 19:56:31 DEBUG { 19:56:31 DEBUG "status": "success", 19:56:31 DEBUG "data": { 19:56:31 DEBUG "resultType": "vector", 19:56:31 DEBUG "result": [ 19:56:31 DEBUG { 19:56:31 DEBUG "metric": { 19:56:31 DEBUG "pod": "am-767f6b67b5-9zw22" 19:56:31 DEBUG }, 19:56:31 DEBUG "value": [ 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG [...] 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG "pod": "am-767f6b67b5-vbltz" 19:56:31 DEBUG }, 19:56:31 DEBUG "value": [ 19:56:31 DEBUG 1713552991, 19:56:31 DEBUG "0.014" 19:56:31 DEBUG ] 19:56:31 DEBUG } 19:56:31 DEBUG ] 19:56:31 DEBUG } 19:56:31 DEBUG } 19:56:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552991" 19:56:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552991" 19:56:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552991" 19:56:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713552991" 19:56:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552991" 19:56:31 INFO [http_cmd]: http status code OK 19:56:31 INFO [http_cmd]: http status code OK 19:56:31 DEBUG --- status code --- 19:56:31 DEBUG --- status code --- 19:56:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:31 DEBUG --- http response --- 19:56:31 DEBUG --- http response --- 19:56:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:56:31 DEBUG ----- output ----- 19:56:31 DEBUG { 19:56:31 DEBUG "status": "success", 19:56:31 DEBUG "data": { 19:56:31 DEBUG "resultType": "vector", 19:56:31 DEBUG "result": [ 19:56:31 DEBUG { 19:56:31 DEBUG "metric": { 19:56:31 DEBUG "pod": "am-767f6b67b5-9zw22" 19:56:31 DEBUG }, 19:56:31 DEBUG "value": [ 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG [...] 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG "pod": "am-767f6b67b5-vbltz" 19:56:31 DEBUG }, 19:56:31 DEBUG "value": [ 19:56:31 DEBUG 1713552991, 19:56:31 DEBUG "0" 19:56:31 DEBUG ] 19:56:31 DEBUG } 19:56:31 DEBUG ] 19:56:31 DEBUG } 19:56:31 DEBUG } 19:56:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:56:31 DEBUG ----- output ----- 19:56:31 DEBUG { 19:56:31 DEBUG "status": "success", 19:56:31 DEBUG "data": { 19:56:31 DEBUG "resultType": "vector", 19:56:31 DEBUG "result": [ 19:56:31 DEBUG { 19:56:31 DEBUG "metric": { 19:56:31 DEBUG "pod": "ds-cts-0" 19:56:31 DEBUG }, 19:56:31 DEBUG "value": [ 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG [...] 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG "pod": "ds-cts-2" 19:56:31 DEBUG }, 19:56:31 DEBUG "value": [ 19:56:31 DEBUG 1713552991, 19:56:31 DEBUG "29.933333333333334" 19:56:31 DEBUG ] 19:56:31 DEBUG } 19:56:31 DEBUG ] 19:56:31 DEBUG } 19:56:31 DEBUG } 19:56:31 INFO [http_cmd]: http status code OK 19:56:31 INFO [http_cmd]: http status code OK 19:56:31 DEBUG --- status code --- 19:56:31 DEBUG --- status code --- 19:56:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:31 DEBUG --- http response --- 19:56:31 DEBUG --- http response --- 19:56:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:56:31 DEBUG ----- output ----- 19:56:31 DEBUG { 19:56:31 DEBUG "status": "success", 19:56:31 DEBUG "data": { 19:56:31 DEBUG "resultType": "vector", 19:56:31 DEBUG "result": [ 19:56:31 DEBUG { 19:56:31 DEBUG "metric": { 19:56:31 DEBUG "pod": "ds-idrepo-0" 19:56:31 DEBUG }, 19:56:31 DEBUG "value": [ 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG [...] 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG "pod": "ds-idrepo-2" 19:56:31 DEBUG }, 19:56:31 DEBUG "value": [ 19:56:31 DEBUG 1713552991, 19:56:31 DEBUG "0" 19:56:31 DEBUG ] 19:56:31 DEBUG } 19:56:31 DEBUG ] 19:56:31 DEBUG } 19:56:31 DEBUG } 19:56:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:56:31 DEBUG ----- output ----- 19:56:31 DEBUG { 19:56:31 DEBUG "status": "success", 19:56:31 DEBUG "data": { 19:56:31 DEBUG "resultType": "vector", 19:56:31 DEBUG "result": [ 19:56:31 DEBUG { 19:56:31 DEBUG "metric": { 19:56:31 DEBUG "instance": "10.142.0.107:9100" 19:56:31 DEBUG }, 19:56:31 DEBUG "value": [ 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG [...] 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG "instance": "10.142.0.43:9100" 19:56:31 DEBUG }, 19:56:31 DEBUG "value": [ 19:56:31 DEBUG 1713552991, 19:56:31 DEBUG "0.16780000000005468" 19:56:31 DEBUG ] 19:56:31 DEBUG } 19:56:31 DEBUG ] 19:56:31 DEBUG } 19:56:31 DEBUG } 19:56:31 INFO [http_cmd]: http status code OK 19:56:31 DEBUG --- status code --- 19:56:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:31 DEBUG --- http response --- 19:56:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:56:31 DEBUG ----- output ----- 19:56:31 DEBUG { 19:56:31 DEBUG "status": "success", 19:56:31 DEBUG "data": { 19:56:31 DEBUG "resultType": "vector", 19:56:31 DEBUG "result": [ 19:56:31 DEBUG { 19:56:31 DEBUG "metric": { 19:56:31 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:56:31 DEBUG }, 19:56:31 DEBUG "value": [ 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG [...] 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:56:31 DEBUG }, 19:56:31 DEBUG "value": [ 19:56:31 DEBUG 1713552991, 19:56:31 DEBUG "8.041382213811088" 19:56:31 DEBUG ] 19:56:31 DEBUG } 19:56:31 DEBUG ] 19:56:31 DEBUG } 19:56:31 DEBUG } 19:56:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552991" 19:56:31 INFO [http_cmd]: http status code OK 19:56:31 DEBUG --- status code --- 19:56:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:31 DEBUG --- http response --- 19:56:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:56:31 DEBUG ----- output ----- 19:56:31 DEBUG { 19:56:31 DEBUG "status": "success", 19:56:31 DEBUG "data": { 19:56:31 DEBUG "resultType": "vector", 19:56:31 DEBUG "result": [ 19:56:31 DEBUG { 19:56:31 DEBUG "metric": { 19:56:31 DEBUG "pod": "am-767f6b67b5-9zw22" 19:56:31 DEBUG }, 19:56:31 DEBUG "value": [ 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG [...] 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG "pod": "am-767f6b67b5-vbltz" 19:56:31 DEBUG }, 19:56:31 DEBUG "value": [ 19:56:31 DEBUG 1713552991, 19:56:31 DEBUG "2505.5666666666666" 19:56:31 DEBUG ] 19:56:31 DEBUG } 19:56:31 DEBUG ] 19:56:31 DEBUG } 19:56:31 DEBUG } 19:56:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552991" 19:56:31 INFO [http_cmd]: http status code OK 19:56:31 DEBUG --- status code --- 19:56:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:31 DEBUG --- http response --- 19:56:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:56:31 DEBUG ----- output ----- 19:56:31 DEBUG { 19:56:31 DEBUG "status": "success", 19:56:31 DEBUG "data": { 19:56:31 DEBUG "resultType": "vector", 19:56:31 DEBUG "result": [ 19:56:31 DEBUG { 19:56:31 DEBUG "metric": { 19:56:31 DEBUG "pod": "ds-idrepo-0" 19:56:31 DEBUG }, 19:56:31 DEBUG "value": [ 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG [...] 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG "pod": "ds-idrepo-2" 19:56:31 DEBUG }, 19:56:31 DEBUG "value": [ 19:56:31 DEBUG 1713552991, 19:56:31 DEBUG "0" 19:56:31 DEBUG ] 19:56:31 DEBUG } 19:56:31 DEBUG ] 19:56:31 DEBUG } 19:56:31 DEBUG } 19:56:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552991" 19:56:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552991" 19:56:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552991" 19:56:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552991" 19:56:31 INFO [http_cmd]: http status code OK 19:56:31 DEBUG --- status code --- 19:56:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:31 DEBUG --- http response --- 19:56:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:56:31 DEBUG ----- output ----- 19:56:31 DEBUG { 19:56:31 DEBUG "status": "success", 19:56:31 DEBUG "data": { 19:56:31 DEBUG "resultType": "vector", 19:56:31 DEBUG "result": [ 19:56:31 DEBUG { 19:56:31 DEBUG "metric": { 19:56:31 DEBUG "instance": "10.142.0.107:9100" 19:56:31 DEBUG }, 19:56:31 DEBUG "value": [ 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG [...] 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG "instance": "10.142.0.43:9100" 19:56:31 DEBUG }, 19:56:31 DEBUG "value": [ 19:56:31 DEBUG 1713552991, 19:56:31 DEBUG "1982305762304" 19:56:31 DEBUG ] 19:56:31 DEBUG } 19:56:31 DEBUG ] 19:56:31 DEBUG } 19:56:31 DEBUG } 19:56:31 INFO [http_cmd]: http status code OK 19:56:31 DEBUG --- status code --- 19:56:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:31 DEBUG --- http response --- 19:56:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:56:31 DEBUG ----- output ----- 19:56:31 DEBUG { 19:56:31 DEBUG "status": "success", 19:56:31 DEBUG "data": { 19:56:31 DEBUG "resultType": "vector", 19:56:31 DEBUG "result": [ 19:56:31 DEBUG { 19:56:31 DEBUG "metric": { 19:56:31 DEBUG "pod": "am-767f6b67b5-9zw22" 19:56:31 DEBUG }, 19:56:31 DEBUG "value": [ 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG [...] 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG "pod": "am-767f6b67b5-vbltz" 19:56:31 DEBUG }, 19:56:31 DEBUG "value": [ 19:56:31 DEBUG 1713552991, 19:56:31 DEBUG "2510.3" 19:56:31 DEBUG ] 19:56:31 DEBUG } 19:56:31 DEBUG ] 19:56:31 DEBUG } 19:56:31 DEBUG } 19:56:31 INFO [http_cmd]: http status code OK 19:56:31 DEBUG --- status code --- 19:56:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:31 DEBUG --- http response --- 19:56:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:56:31 DEBUG ----- output ----- 19:56:31 DEBUG { 19:56:31 DEBUG "status": "success", 19:56:31 DEBUG "data": { 19:56:31 DEBUG "resultType": "vector", 19:56:31 DEBUG "result": [ 19:56:31 DEBUG { 19:56:31 DEBUG "metric": { 19:56:31 DEBUG "pod": "ds-idrepo-1" 19:56:31 DEBUG }, 19:56:31 DEBUG "value": [ 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG [...] 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG "pod": "am-767f6b67b5-qcqz2" 19:56:31 DEBUG }, 19:56:31 DEBUG "value": [ 19:56:31 DEBUG 1713552991, 19:56:31 DEBUG "276562871.3037232" 19:56:31 DEBUG ] 19:56:31 DEBUG } 19:56:31 DEBUG ] 19:56:31 DEBUG } 19:56:31 DEBUG } 19:56:31 INFO [http_cmd]: http status code OK 19:56:31 DEBUG --- status code --- 19:56:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:31 DEBUG --- http response --- 19:56:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:56:31 DEBUG ----- output ----- 19:56:31 DEBUG { 19:56:31 DEBUG "status": "success", 19:56:31 DEBUG "data": { 19:56:31 DEBUG "resultType": "vector", 19:56:31 DEBUG "result": [ 19:56:31 DEBUG { 19:56:31 DEBUG "metric": { 19:56:31 DEBUG "instance": "10.142.0.107:9100" 19:56:31 DEBUG }, 19:56:31 DEBUG "value": [ 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG [...] 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG "instance": "10.142.0.43:9100" 19:56:31 DEBUG }, 19:56:31 DEBUG "value": [ 19:56:31 DEBUG 1713552991, 19:56:31 DEBUG "0.07466666666665937" 19:56:31 DEBUG ] 19:56:31 DEBUG } 19:56:31 DEBUG ] 19:56:31 DEBUG } 19:56:31 DEBUG } 19:56:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552991" 19:56:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552991" 19:56:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713552991" 19:56:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552991" 19:56:31 INFO [http_cmd]: http status code OK 19:56:31 INFO [http_cmd]: http status code OK 19:56:31 DEBUG --- status code --- 19:56:31 DEBUG --- status code --- 19:56:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:31 DEBUG --- http response --- 19:56:31 DEBUG --- http response --- 19:56:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:56:31 DEBUG ----- output ----- 19:56:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:56:31 DEBUG { 19:56:31 DEBUG ----- output ----- 19:56:31 DEBUG "status": "success", 19:56:31 DEBUG { 19:56:31 DEBUG "data": { 19:56:31 DEBUG "status": "success", 19:56:31 DEBUG "resultType": "vector", 19:56:31 DEBUG "data": { 19:56:31 DEBUG "result": [ 19:56:31 DEBUG "resultType": "vector", 19:56:31 DEBUG { 19:56:31 DEBUG "result": [ 19:56:31 DEBUG "metric": { 19:56:31 DEBUG { 19:56:31 DEBUG "pod": "am-767f6b67b5-9zw22" 19:56:31 DEBUG "metric": { 19:56:31 DEBUG }, 19:56:31 DEBUG "instance": "10.142.0.107:9100" 19:56:31 DEBUG "value": [ 19:56:31 DEBUG }, 19:56:31 DEBUG 19:56:31 DEBUG "value": [ 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG [...] 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG [...] 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG "pod": "am-767f6b67b5-vbltz" 19:56:31 DEBUG 19:56:31 DEBUG }, 19:56:31 DEBUG "instance": "10.142.0.43:9100" 19:56:31 DEBUG "value": [ 19:56:31 DEBUG }, 19:56:31 DEBUG 1713552991, 19:56:31 DEBUG "value": [ 19:56:31 DEBUG "2506.8" 19:56:31 DEBUG 1713552991, 19:56:31 DEBUG ] 19:56:31 DEBUG "75721732.66666667" 19:56:31 DEBUG } 19:56:31 DEBUG ] 19:56:31 DEBUG ] 19:56:31 DEBUG } 19:56:31 DEBUG } 19:56:31 DEBUG ] 19:56:31 DEBUG } 19:56:31 DEBUG } 19:56:31 DEBUG } 19:56:31 INFO [http_cmd]: http status code OK 19:56:31 DEBUG --- status code --- 19:56:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:31 DEBUG --- http response --- 19:56:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:56:31 DEBUG ----- output ----- 19:56:31 DEBUG { 19:56:31 DEBUG "status": "success", 19:56:31 DEBUG "data": { 19:56:31 DEBUG "resultType": "vector", 19:56:31 DEBUG "result": [ 19:56:31 DEBUG { 19:56:31 DEBUG "metric": { 19:56:31 DEBUG "instance": "10.142.0.107:9100" 19:56:31 DEBUG }, 19:56:31 DEBUG "value": [ 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG [...] 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG "instance": "10.142.0.43:9100" 19:56:31 DEBUG }, 19:56:31 DEBUG "value": [ 19:56:31 DEBUG 1713552991, 19:56:31 DEBUG "75717829.76666667" 19:56:31 DEBUG ] 19:56:31 DEBUG } 19:56:31 DEBUG ] 19:56:31 DEBUG } 19:56:31 DEBUG } 19:56:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713552991" 19:56:31 INFO [http_cmd]: http status code OK 19:56:31 DEBUG --- status code --- 19:56:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:31 DEBUG --- http response --- 19:56:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:56:31 DEBUG ----- output ----- 19:56:31 DEBUG { 19:56:31 DEBUG "status": "success", 19:56:31 DEBUG "data": { 19:56:31 DEBUG "resultType": "vector", 19:56:31 DEBUG "result": [ 19:56:31 DEBUG { 19:56:31 DEBUG "metric": { 19:56:31 DEBUG "pod": "ds-idrepo-1" 19:56:31 DEBUG }, 19:56:31 DEBUG "value": [ 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG [...] 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG "pod": "am-767f6b67b5-qcqz2" 19:56:31 DEBUG }, 19:56:31 DEBUG "value": [ 19:56:31 DEBUG 1713552991, 19:56:31 DEBUG "0" 19:56:31 DEBUG ] 19:56:31 DEBUG } 19:56:31 DEBUG ] 19:56:31 DEBUG } 19:56:31 DEBUG } 19:56:31 INFO [http_cmd]: http status code OK 19:56:31 DEBUG --- status code --- 19:56:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:31 DEBUG --- http response --- 19:56:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:56:31 DEBUG ----- output ----- 19:56:31 DEBUG { 19:56:31 DEBUG "status": "success", 19:56:31 DEBUG "data": { 19:56:31 DEBUG "resultType": "vector", 19:56:31 DEBUG "result": [ 19:56:31 DEBUG { 19:56:31 DEBUG "metric": { 19:56:31 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:56:31 DEBUG }, 19:56:31 DEBUG "value": [ 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG [...] 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:56:31 DEBUG }, 19:56:31 DEBUG "value": [ 19:56:31 DEBUG 1713552991, 19:56:31 DEBUG "3587043328" 19:56:31 DEBUG ] 19:56:31 DEBUG } 19:56:31 DEBUG ] 19:56:31 DEBUG } 19:56:31 DEBUG } 19:56:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552991" 19:56:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552991" 19:56:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713552991" 19:56:31 INFO [http_cmd]: http status code OK 19:56:31 INFO [http_cmd]: http status code OK 19:56:31 DEBUG --- status code --- 19:56:31 DEBUG --- status code --- 19:56:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:31 DEBUG --- http response --- 19:56:31 DEBUG --- http response --- 19:56:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:56:31 DEBUG ----- output ----- 19:56:31 DEBUG { 19:56:31 DEBUG "status": "success", 19:56:31 DEBUG "data": { 19:56:31 DEBUG "resultType": "vector", 19:56:31 DEBUG "result": [ 19:56:31 DEBUG { 19:56:31 DEBUG "metric": { 19:56:31 DEBUG "pod": "am-767f6b67b5-9zw22" 19:56:31 DEBUG }, 19:56:31 DEBUG "value": [ 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG [...] 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG "pod": "am-767f6b67b5-vbltz" 19:56:31 DEBUG }, 19:56:31 DEBUG "value": [ 19:56:31 DEBUG 1713552991, 19:56:31 DEBUG "2505.633333333333" 19:56:31 DEBUG ] 19:56:31 DEBUG } 19:56:31 DEBUG ] 19:56:31 DEBUG } 19:56:31 DEBUG } 19:56:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:56:31 DEBUG ----- output ----- 19:56:31 DEBUG { 19:56:31 DEBUG "status": "success", 19:56:31 DEBUG "data": { 19:56:31 DEBUG "resultType": "vector", 19:56:31 DEBUG "result": [ 19:56:31 DEBUG { 19:56:31 DEBUG "metric": { 19:56:31 DEBUG "pod": "ds-cts-0" 19:56:31 DEBUG }, 19:56:31 DEBUG "value": [ 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG [...] 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG "pod": "ds-cts-2" 19:56:31 DEBUG }, 19:56:31 DEBUG "value": [ 19:56:31 DEBUG 1713552991, 19:56:31 DEBUG "0" 19:56:31 DEBUG ] 19:56:31 DEBUG } 19:56:31 DEBUG ] 19:56:31 DEBUG } 19:56:31 DEBUG } 19:56:31 INFO [http_cmd]: http status code OK 19:56:31 DEBUG --- status code --- 19:56:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:31 DEBUG --- http response --- 19:56:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:56:31 DEBUG ----- output ----- 19:56:31 DEBUG { 19:56:31 DEBUG "status": "success", 19:56:31 DEBUG "data": { 19:56:31 DEBUG "resultType": "vector", 19:56:31 DEBUG "result": [ 19:56:31 DEBUG { 19:56:31 DEBUG "metric": { 19:56:31 DEBUG "pod": "ds-idrepo-1" 19:56:31 DEBUG }, 19:56:31 DEBUG "value": [ 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG [...] 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG 19:56:31 DEBUG "pod": "am-767f6b67b5-qcqz2" 19:56:31 DEBUG }, 19:56:31 DEBUG "value": [ 19:56:31 DEBUG 1713552991, 19:56:31 DEBUG "16.26374392433838" 19:56:31 DEBUG ] 19:56:31 DEBUG } 19:56:31 DEBUG ] 19:56:31 DEBUG } 19:56:31 DEBUG } 19:56:32 INFO 19:56:32 INFO [loop_until]: kubectl --namespace=xlou top pods 19:56:32 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:56:32 INFO [loop_until]: OK (rc = 0) 19:56:32 DEBUG --- stdout --- 19:56:32 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 16545m 9002Mi am-767f6b67b5-qcqz2 17006m 8812Mi am-767f6b67b5-vbltz 17652m 8995Mi ds-cts-0 16820m 10959Mi ds-cts-1 15949m 11213Mi ds-cts-2 16885m 10978Mi ds-idrepo-0 7310m 3695Mi ds-idrepo-1 7112m 3536Mi ds-idrepo-2 7287m 3492Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8081m 3326Mi 19:56:32 DEBUG --- stderr --- 19:56:32 DEBUG 19:56:33 INFO 19:56:33 INFO [loop_until]: kubectl --namespace=xlou top node 19:56:33 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:56:33 INFO [loop_until]: OK (rc = 0) 19:56:33 DEBUG --- stdout --- 19:56:33 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 70m 0% 6250Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7450m 24% 7296Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16146m 54% 11701Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 81m 0% 3343Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 16898m 56% 13945Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 140m 0% 5218Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 16485m 55% 11690Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15535m 52% 2989Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 17579m 58% 11207Mi 10% gke-xlou-cdm-ds-0f601751-j488 7670m 25% 6745Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 16077m 53% 15954Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 7424m 24% 7437Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 17388m 58% 13405Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 8196m 51% 7075Mi 12% 19:56:33 DEBUG --- stderr --- 19:56:33 DEBUG 19:57:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553051" 19:57:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553051" 19:57:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553051" 19:57:31 INFO [http_cmd]: http status code OK 19:57:31 INFO [http_cmd]: http status code OK 19:57:31 DEBUG --- status code --- 19:57:31 DEBUG --- status code --- 19:57:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:31 DEBUG --- http response --- 19:57:31 DEBUG --- http response --- 19:57:31 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:57:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:57:31 DEBUG ----- output ----- 19:57:31 DEBUG { 19:57:31 DEBUG "status": "success", 19:57:31 DEBUG "data": { 19:57:31 DEBUG "resultType": "vector", 19:57:31 DEBUG "result": [ 19:57:31 DEBUG { 19:57:31 DEBUG "metric": { 19:57:31 DEBUG "pod": "ds-cts-0" 19:57:31 DEBUG }, 19:57:31 DEBUG "value": [ 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG [...] 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG "pod": "ds-cts-2" 19:57:31 DEBUG }, 19:57:31 DEBUG "value": [ 19:57:31 DEBUG 1713553051, 19:57:31 DEBUG "0.00030000000000000003" 19:57:31 DEBUG ] 19:57:31 DEBUG } 19:57:31 DEBUG ] 19:57:31 DEBUG } 19:57:31 DEBUG } 19:57:31 INFO [http_cmd]: http status code OK 19:57:31 DEBUG --- status code --- 19:57:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:31 DEBUG --- http response --- 19:57:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:57:31 DEBUG ----- output ----- 19:57:31 DEBUG { 19:57:31 DEBUG "status": "success", 19:57:31 DEBUG "data": { 19:57:31 DEBUG "resultType": "vector", 19:57:31 DEBUG "result": [ 19:57:31 DEBUG { 19:57:31 DEBUG "metric": { 19:57:31 DEBUG "pod": "ds-idrepo-0" 19:57:31 DEBUG }, 19:57:31 DEBUG "value": [ 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG [...] 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG "pod": "ds-idrepo-2" 19:57:31 DEBUG }, 19:57:31 DEBUG "value": [ 19:57:31 DEBUG 1713553051, 19:57:31 DEBUG "0" 19:57:31 DEBUG ] 19:57:31 DEBUG } 19:57:31 DEBUG ] 19:57:31 DEBUG } 19:57:31 DEBUG } 19:57:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713553051" 19:57:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553051" 19:57:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713553051" 19:57:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553051" 19:57:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713553051" 19:57:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713553051" 19:57:31 INFO [http_cmd]: http status code OK 19:57:31 INFO [http_cmd]: http status code OK 19:57:31 DEBUG --- status code --- 19:57:31 DEBUG --- status code --- 19:57:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:31 DEBUG --- http response --- 19:57:31 DEBUG --- http response --- 19:57:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:57:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:57:31 DEBUG ----- output ----- 19:57:31 DEBUG ----- output ----- 19:57:31 DEBUG { 19:57:31 DEBUG { 19:57:31 DEBUG "status": "success", 19:57:31 DEBUG "status": "success", 19:57:31 DEBUG "data": { 19:57:31 DEBUG "data": { 19:57:31 DEBUG "resultType": "vector", 19:57:31 DEBUG "resultType": "vector", 19:57:31 DEBUG "result": [ 19:57:31 DEBUG "result": [ 19:57:31 DEBUG { 19:57:31 DEBUG { 19:57:31 DEBUG "metric": { 19:57:31 DEBUG "metric": { 19:57:31 DEBUG "pod": "am-767f6b67b5-9zw22" 19:57:31 DEBUG "pod": "ds-idrepo-0" 19:57:31 DEBUG }, 19:57:31 DEBUG }, 19:57:31 DEBUG "value": [ 19:57:31 DEBUG "value": [ 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG [...] 19:57:31 DEBUG [...] 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG "pod": "am-767f6b67b5-vbltz" 19:57:31 DEBUG "pod": "ds-cts-2" 19:57:31 INFO [http_cmd]: http status code OK 19:57:31 DEBUG }, 19:57:31 DEBUG }, 19:57:31 DEBUG --- status code --- 19:57:31 DEBUG "value": [ 19:57:31 DEBUG "value": [ 19:57:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:31 DEBUG 1713553051, 19:57:31 DEBUG 1713553051, 19:57:31 DEBUG --- http response --- 19:57:31 DEBUG "449859.5" 19:57:31 DEBUG "0" 19:57:31 DEBUG ] 19:57:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:57:31 DEBUG ] 19:57:31 DEBUG } 19:57:31 DEBUG ----- output ----- 19:57:31 DEBUG } 19:57:31 DEBUG ] 19:57:31 DEBUG { 19:57:31 DEBUG ] 19:57:31 DEBUG } 19:57:31 DEBUG "status": "success", 19:57:31 DEBUG } 19:57:31 DEBUG } 19:57:31 INFO [http_cmd]: http status code OK 19:57:31 DEBUG "data": { 19:57:31 DEBUG } 19:57:31 DEBUG --- status code --- 19:57:31 DEBUG "resultType": "vector", 19:57:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:31 DEBUG "result": [ 19:57:31 DEBUG --- http response --- 19:57:31 DEBUG { 19:57:31 DEBUG "metric": { 19:57:31 DEBUG "pod": "am-767f6b67b5-9zw22" 19:57:31 DEBUG }, 19:57:31 DEBUG "value": [ 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG [...] 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG "pod": "am-767f6b67b5-vbltz" 19:57:31 DEBUG }, 19:57:31 DEBUG "value": [ 19:57:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:57:31 DEBUG 1713553051, 19:57:31 DEBUG ----- output ----- 19:57:31 DEBUG "0.014" 19:57:31 DEBUG { 19:57:31 DEBUG ] 19:57:31 DEBUG "status": "success", 19:57:31 DEBUG } 19:57:31 DEBUG "data": { 19:57:31 DEBUG ] 19:57:31 DEBUG "resultType": "vector", 19:57:31 DEBUG } 19:57:31 DEBUG "result": [ 19:57:31 DEBUG } 19:57:31 DEBUG { 19:57:31 DEBUG "metric": { 19:57:31 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:57:31 DEBUG }, 19:57:31 DEBUG "value": [ 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG [...] 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:57:31 DEBUG }, 19:57:31 DEBUG "value": [ 19:57:31 DEBUG 1713553051, 19:57:31 DEBUG "8.037980904870189" 19:57:31 DEBUG ] 19:57:31 DEBUG } 19:57:31 DEBUG ] 19:57:31 DEBUG } 19:57:31 DEBUG } 19:57:31 ERROR [http_cmd]: ERROR 19:57:31 ERROR --- status code --- 19:57:31 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:57:31 ERROR --- http response --- 19:57:31 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:57:31 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:57:31 INFO [http_cmd]: http status code OK 19:57:31 DEBUG --- status code --- 19:57:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:31 DEBUG --- http response --- 19:57:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 19:57:31 DEBUG ----- output ----- 19:57:31 DEBUG { 19:57:31 DEBUG "status": "success", 19:57:31 DEBUG "data": { 19:57:31 DEBUG "resultType": "vector", 19:57:31 DEBUG "result": [ 19:57:31 DEBUG { 19:57:31 DEBUG "metric": { 19:57:31 DEBUG "pod": "ds-idrepo-0" 19:57:31 DEBUG }, 19:57:31 DEBUG "value": [ 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG [...] 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:57:31 DEBUG }, 19:57:31 DEBUG "value": [ 19:57:31 DEBUG 1713553051, 19:57:31 DEBUG "0" 19:57:31 DEBUG ] 19:57:31 DEBUG } 19:57:31 DEBUG ] 19:57:31 DEBUG } 19:57:31 DEBUG } 19:57:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553051" 19:57:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553051" 19:57:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553051" 19:57:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553051" 19:57:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553051" 19:57:31 INFO [http_cmd]: http status code OK 19:57:31 INFO [http_cmd]: http status code OK 19:57:31 DEBUG --- status code --- 19:57:31 DEBUG --- status code --- 19:57:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:31 DEBUG --- http response --- 19:57:31 DEBUG --- http response --- 19:57:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:57:31 DEBUG ----- output ----- 19:57:31 DEBUG { 19:57:31 DEBUG "status": "success", 19:57:31 DEBUG "data": { 19:57:31 DEBUG "resultType": "vector", 19:57:31 DEBUG "result": [ 19:57:31 DEBUG { 19:57:31 DEBUG "metric": { 19:57:31 DEBUG "pod": "am-767f6b67b5-9zw22" 19:57:31 DEBUG }, 19:57:31 DEBUG "value": [ 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG [...] 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG "pod": "am-767f6b67b5-vbltz" 19:57:31 DEBUG }, 19:57:31 DEBUG "value": [ 19:57:31 DEBUG 1713553051, 19:57:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:57:31 DEBUG "0" 19:57:31 DEBUG ----- output ----- 19:57:31 DEBUG ] 19:57:31 DEBUG { 19:57:31 DEBUG } 19:57:31 DEBUG "status": "success", 19:57:31 DEBUG ] 19:57:31 DEBUG "data": { 19:57:31 DEBUG } 19:57:31 DEBUG "resultType": "vector", 19:57:31 DEBUG } 19:57:31 DEBUG "result": [ 19:57:31 DEBUG { 19:57:31 DEBUG "metric": { 19:57:31 DEBUG "pod": "ds-cts-0" 19:57:31 DEBUG }, 19:57:31 DEBUG "value": [ 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG [...] 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG "pod": "ds-cts-2" 19:57:31 DEBUG }, 19:57:31 DEBUG "value": [ 19:57:31 DEBUG 1713553051, 19:57:31 DEBUG "30" 19:57:31 DEBUG ] 19:57:31 DEBUG } 19:57:31 DEBUG ] 19:57:31 DEBUG } 19:57:31 DEBUG } 19:57:31 INFO [http_cmd]: http status code OK 19:57:31 INFO [http_cmd]: http status code OK 19:57:31 DEBUG --- status code --- 19:57:31 DEBUG --- status code --- 19:57:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:31 DEBUG --- http response --- 19:57:31 DEBUG --- http response --- 19:57:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:57:31 DEBUG ----- output ----- 19:57:31 DEBUG { 19:57:31 DEBUG "status": "success", 19:57:31 DEBUG "data": { 19:57:31 DEBUG "resultType": "vector", 19:57:31 DEBUG "result": [ 19:57:31 DEBUG { 19:57:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:57:31 DEBUG "metric": { 19:57:31 DEBUG ----- output ----- 19:57:31 DEBUG "pod": "ds-idrepo-0" 19:57:31 DEBUG { 19:57:31 DEBUG }, 19:57:31 DEBUG "status": "success", 19:57:31 DEBUG "value": [ 19:57:31 DEBUG "data": { 19:57:31 DEBUG 19:57:31 DEBUG "resultType": "vector", 19:57:31 DEBUG 19:57:31 DEBUG "result": [ 19:57:31 DEBUG 19:57:31 DEBUG { 19:57:31 DEBUG [...] 19:57:31 DEBUG "metric": { 19:57:31 DEBUG 19:57:31 DEBUG "instance": "10.142.0.107:9100" 19:57:31 DEBUG 19:57:31 DEBUG }, 19:57:31 DEBUG 19:57:31 DEBUG "value": [ 19:57:31 DEBUG "pod": "ds-idrepo-2" 19:57:31 DEBUG 19:57:31 DEBUG }, 19:57:31 DEBUG 19:57:31 DEBUG "value": [ 19:57:31 DEBUG 19:57:31 DEBUG 1713553051, 19:57:31 DEBUG [...] 19:57:31 DEBUG "0" 19:57:31 DEBUG 19:57:31 DEBUG ] 19:57:31 DEBUG 19:57:31 DEBUG } 19:57:31 DEBUG 19:57:31 DEBUG ] 19:57:31 DEBUG "instance": "10.142.0.43:9100" 19:57:31 DEBUG } 19:57:31 DEBUG }, 19:57:31 DEBUG } 19:57:31 DEBUG "value": [ 19:57:31 DEBUG 1713553051, 19:57:31 DEBUG "0.17846666666669173" 19:57:31 DEBUG ] 19:57:31 DEBUG } 19:57:31 DEBUG ] 19:57:31 DEBUG } 19:57:31 DEBUG } 19:57:31 INFO [http_cmd]: http status code OK 19:57:31 DEBUG --- status code --- 19:57:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:31 DEBUG --- http response --- 19:57:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:57:31 DEBUG ----- output ----- 19:57:31 DEBUG { 19:57:31 DEBUG "status": "success", 19:57:31 DEBUG "data": { 19:57:31 DEBUG "resultType": "vector", 19:57:31 DEBUG "result": [ 19:57:31 DEBUG { 19:57:31 DEBUG "metric": { 19:57:31 DEBUG "pod": "ds-idrepo-0" 19:57:31 DEBUG }, 19:57:31 DEBUG "value": [ 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG [...] 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG "pod": "ds-idrepo-2" 19:57:31 DEBUG }, 19:57:31 DEBUG "value": [ 19:57:31 DEBUG 1713553051, 19:57:31 DEBUG "0" 19:57:31 DEBUG ] 19:57:31 DEBUG } 19:57:31 DEBUG ] 19:57:31 DEBUG } 19:57:31 DEBUG } 19:57:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553051" 19:57:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553051" 19:57:31 INFO [http_cmd]: http status code OK 19:57:31 DEBUG --- status code --- 19:57:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:31 DEBUG --- http response --- 19:57:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:57:31 DEBUG ----- output ----- 19:57:31 DEBUG { 19:57:31 DEBUG "status": "success", 19:57:31 DEBUG "data": { 19:57:31 DEBUG "resultType": "vector", 19:57:31 DEBUG "result": [ 19:57:31 DEBUG { 19:57:31 DEBUG "metric": { 19:57:31 DEBUG "instance": "10.142.0.107:9100" 19:57:31 DEBUG }, 19:57:31 DEBUG "value": [ 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG [...] 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG "instance": "10.142.0.43:9100" 19:57:31 DEBUG }, 19:57:31 DEBUG "value": [ 19:57:31 DEBUG 1713553051, 19:57:31 DEBUG "0.07799999999999728" 19:57:31 DEBUG ] 19:57:31 DEBUG } 19:57:31 DEBUG ] 19:57:31 DEBUG } 19:57:31 DEBUG } 19:57:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553051" 19:57:31 INFO [http_cmd]: http status code OK 19:57:31 DEBUG --- status code --- 19:57:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:31 DEBUG --- http response --- 19:57:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553051" 19:57:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:57:31 DEBUG ----- output ----- 19:57:31 DEBUG { 19:57:31 DEBUG "status": "success", 19:57:31 DEBUG "data": { 19:57:31 DEBUG "resultType": "vector", 19:57:31 DEBUG "result": [ 19:57:31 DEBUG { 19:57:31 DEBUG "metric": { 19:57:31 DEBUG "pod": "am-767f6b67b5-9zw22" 19:57:31 DEBUG }, 19:57:31 DEBUG "value": [ 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG [...] 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG "pod": "am-767f6b67b5-vbltz" 19:57:31 DEBUG }, 19:57:31 DEBUG "value": [ 19:57:31 DEBUG 1713553051, 19:57:31 DEBUG "2501.3" 19:57:31 DEBUG ] 19:57:31 DEBUG } 19:57:31 DEBUG ] 19:57:31 DEBUG } 19:57:31 DEBUG } 19:57:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553051" 19:57:31 INFO [http_cmd]: http status code OK 19:57:31 DEBUG --- status code --- 19:57:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:31 DEBUG --- http response --- 19:57:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:57:31 DEBUG ----- output ----- 19:57:31 DEBUG { 19:57:31 DEBUG "status": "success", 19:57:31 DEBUG "data": { 19:57:31 DEBUG "resultType": "vector", 19:57:31 DEBUG "result": [ 19:57:31 DEBUG { 19:57:31 DEBUG "metric": { 19:57:31 DEBUG "pod": "am-767f6b67b5-9zw22" 19:57:31 DEBUG }, 19:57:31 DEBUG "value": [ 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG [...] 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG "pod": "am-767f6b67b5-vbltz" 19:57:31 DEBUG }, 19:57:31 DEBUG "value": [ 19:57:31 DEBUG 1713553051, 19:57:31 DEBUG "2499.633333333333" 19:57:31 INFO [http_cmd]: http status code OK 19:57:31 DEBUG ] 19:57:31 DEBUG --- status code --- 19:57:31 DEBUG } 19:57:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:31 DEBUG ] 19:57:31 DEBUG --- http response --- 19:57:31 DEBUG } 19:57:31 DEBUG } 19:57:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:57:31 DEBUG ----- output ----- 19:57:31 DEBUG { 19:57:31 DEBUG "status": "success", 19:57:31 DEBUG "data": { 19:57:31 DEBUG "resultType": "vector", 19:57:31 DEBUG "result": [ 19:57:31 DEBUG { 19:57:31 DEBUG "metric": { 19:57:31 DEBUG "pod": "ds-idrepo-0" 19:57:31 DEBUG }, 19:57:31 DEBUG "value": [ 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG [...] 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:57:31 DEBUG }, 19:57:31 DEBUG "value": [ 19:57:31 DEBUG 1713553051, 19:57:31 DEBUG "0" 19:57:31 DEBUG ] 19:57:31 DEBUG } 19:57:31 DEBUG ] 19:57:31 DEBUG } 19:57:31 DEBUG } 19:57:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553051" 19:57:31 INFO [http_cmd]: http status code OK 19:57:31 DEBUG --- status code --- 19:57:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:31 DEBUG --- http response --- 19:57:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:57:31 DEBUG ----- output ----- 19:57:31 DEBUG { 19:57:31 DEBUG "status": "success", 19:57:31 DEBUG "data": { 19:57:31 DEBUG "resultType": "vector", 19:57:31 DEBUG "result": [ 19:57:31 DEBUG { 19:57:31 DEBUG "metric": { 19:57:31 DEBUG "instance": "10.142.0.107:9100" 19:57:31 DEBUG }, 19:57:31 DEBUG "value": [ 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG [...] 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG "instance": "10.142.0.43:9100" 19:57:31 DEBUG }, 19:57:31 DEBUG "value": [ 19:57:31 DEBUG 1713553051, 19:57:31 DEBUG "75451598.26666667" 19:57:31 DEBUG ] 19:57:31 DEBUG } 19:57:31 DEBUG ] 19:57:31 DEBUG } 19:57:31 DEBUG } 19:57:31 INFO [http_cmd]: http status code OK 19:57:31 DEBUG --- status code --- 19:57:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:31 DEBUG --- http response --- 19:57:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:57:31 DEBUG ----- output ----- 19:57:31 DEBUG { 19:57:31 DEBUG "status": "success", 19:57:31 DEBUG "data": { 19:57:31 DEBUG "resultType": "vector", 19:57:31 DEBUG "result": [ 19:57:31 DEBUG { 19:57:31 DEBUG "metric": { 19:57:31 DEBUG "pod": "am-767f6b67b5-9zw22" 19:57:31 DEBUG }, 19:57:31 DEBUG "value": [ 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG [...] 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG "pod": "am-767f6b67b5-vbltz" 19:57:31 DEBUG }, 19:57:31 DEBUG "value": [ 19:57:31 DEBUG 1713553051, 19:57:31 DEBUG "2500.133333333333" 19:57:31 DEBUG ] 19:57:31 DEBUG } 19:57:31 DEBUG ] 19:57:31 DEBUG } 19:57:31 DEBUG } 19:57:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553051" 19:57:31 INFO [http_cmd]: http status code OK 19:57:31 DEBUG --- status code --- 19:57:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:31 DEBUG --- http response --- 19:57:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:57:31 DEBUG ----- output ----- 19:57:31 DEBUG { 19:57:31 DEBUG "status": "success", 19:57:31 DEBUG "data": { 19:57:31 DEBUG "resultType": "vector", 19:57:31 DEBUG "result": [ 19:57:31 DEBUG { 19:57:31 DEBUG "metric": { 19:57:31 DEBUG "instance": "10.142.0.107:9100" 19:57:31 DEBUG }, 19:57:31 DEBUG "value": [ 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG [...] 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG "instance": "10.142.0.43:9100" 19:57:31 DEBUG }, 19:57:31 DEBUG "value": [ 19:57:31 DEBUG 1713553051, 19:57:31 DEBUG "1986606111744" 19:57:31 DEBUG ] 19:57:31 DEBUG } 19:57:31 DEBUG ] 19:57:31 DEBUG } 19:57:31 DEBUG } 19:57:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713553051" 19:57:31 INFO [http_cmd]: http status code OK 19:57:31 DEBUG --- status code --- 19:57:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:31 DEBUG --- http response --- 19:57:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:57:31 DEBUG ----- output ----- 19:57:31 DEBUG { 19:57:31 DEBUG "status": "success", 19:57:31 DEBUG "data": { 19:57:31 DEBUG "resultType": "vector", 19:57:31 DEBUG "result": [ 19:57:31 DEBUG { 19:57:31 DEBUG "metric": { 19:57:31 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:57:31 DEBUG }, 19:57:31 DEBUG "value": [ 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG [...] 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:57:31 DEBUG }, 19:57:31 DEBUG "value": [ 19:57:31 DEBUG 1713553051, 19:57:31 DEBUG "3585810432" 19:57:31 DEBUG ] 19:57:31 DEBUG } 19:57:31 DEBUG ] 19:57:31 DEBUG } 19:57:31 DEBUG } 19:57:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553051" 19:57:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553051" 19:57:31 INFO [http_cmd]: http status code OK 19:57:31 DEBUG --- status code --- 19:57:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:31 DEBUG --- http response --- 19:57:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:57:31 DEBUG ----- output ----- 19:57:31 DEBUG { 19:57:31 DEBUG "status": "success", 19:57:31 DEBUG "data": { 19:57:31 DEBUG "resultType": "vector", 19:57:31 DEBUG "result": [ 19:57:31 DEBUG { 19:57:31 DEBUG "metric": { 19:57:31 DEBUG "instance": "10.142.0.107:9100" 19:57:31 DEBUG }, 19:57:31 DEBUG "value": [ 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG [...] 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG "instance": "10.142.0.43:9100" 19:57:31 DEBUG }, 19:57:31 DEBUG "value": [ 19:57:31 DEBUG 1713553051, 19:57:31 DEBUG "75449550.30000001" 19:57:31 DEBUG ] 19:57:31 DEBUG } 19:57:31 DEBUG ] 19:57:31 DEBUG } 19:57:31 DEBUG } 19:57:31 INFO [http_cmd]: http status code OK 19:57:31 DEBUG --- status code --- 19:57:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:31 DEBUG --- http response --- 19:57:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 19:57:31 DEBUG ----- output ----- 19:57:31 DEBUG { 19:57:31 DEBUG "status": "success", 19:57:31 DEBUG "data": { 19:57:31 DEBUG "resultType": "vector", 19:57:31 DEBUG "result": [ 19:57:31 DEBUG { 19:57:31 DEBUG "metric": { 19:57:31 DEBUG "pod": "ds-idrepo-0" 19:57:31 DEBUG }, 19:57:31 DEBUG "value": [ 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG [...] 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:57:31 DEBUG }, 19:57:31 DEBUG "value": [ 19:57:31 DEBUG 1713553051, 19:57:31 DEBUG "0" 19:57:31 DEBUG ] 19:57:31 DEBUG } 19:57:31 DEBUG ] 19:57:31 DEBUG } 19:57:31 DEBUG } 19:57:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553051" 19:57:31 INFO [http_cmd]: http status code OK 19:57:31 DEBUG --- status code --- 19:57:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:31 DEBUG --- http response --- 19:57:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:57:31 DEBUG ----- output ----- 19:57:31 DEBUG { 19:57:31 DEBUG "status": "success", 19:57:31 DEBUG "data": { 19:57:31 DEBUG "resultType": "vector", 19:57:31 DEBUG "result": [ 19:57:31 DEBUG { 19:57:31 DEBUG "metric": { 19:57:31 DEBUG "pod": "ds-cts-0" 19:57:31 DEBUG }, 19:57:31 DEBUG "value": [ 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG [...] 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG "pod": "ds-cts-2" 19:57:31 DEBUG }, 19:57:31 DEBUG "value": [ 19:57:31 DEBUG 1713553051, 19:57:31 DEBUG "0" 19:57:31 DEBUG ] 19:57:31 DEBUG } 19:57:31 DEBUG ] 19:57:31 DEBUG } 19:57:31 DEBUG } 19:57:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553051" 19:57:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553051" 19:57:31 INFO [http_cmd]: http status code OK 19:57:31 DEBUG --- status code --- 19:57:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:31 DEBUG --- http response --- 19:57:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:57:31 DEBUG ----- output ----- 19:57:31 DEBUG { 19:57:31 DEBUG "status": "success", 19:57:31 DEBUG "data": { 19:57:31 DEBUG "resultType": "vector", 19:57:31 DEBUG "result": [ 19:57:31 DEBUG { 19:57:31 DEBUG "metric": { 19:57:31 DEBUG "pod": "am-767f6b67b5-9zw22" 19:57:31 DEBUG }, 19:57:31 DEBUG "value": [ 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG [...] 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG "pod": "am-767f6b67b5-vbltz" 19:57:31 DEBUG }, 19:57:31 DEBUG "value": [ 19:57:31 DEBUG 1713553051, 19:57:31 DEBUG "2501.0333333333333" 19:57:31 DEBUG ] 19:57:31 DEBUG } 19:57:31 DEBUG ] 19:57:31 DEBUG } 19:57:31 DEBUG } 19:57:31 INFO [http_cmd]: http status code OK 19:57:31 DEBUG --- status code --- 19:57:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:31 DEBUG --- http response --- 19:57:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:57:31 DEBUG ----- output ----- 19:57:31 DEBUG { 19:57:31 DEBUG "status": "success", 19:57:31 DEBUG "data": { 19:57:31 DEBUG "resultType": "vector", 19:57:31 DEBUG "result": [ 19:57:31 DEBUG { 19:57:31 DEBUG "metric": { 19:57:31 DEBUG "pod": "ds-idrepo-0" 19:57:31 DEBUG }, 19:57:31 DEBUG "value": [ 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG [...] 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG 19:57:31 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:57:31 DEBUG }, 19:57:31 DEBUG "value": [ 19:57:31 DEBUG 1713553051, 19:57:31 DEBUG "0.00000843441511261707" 19:57:31 DEBUG ] 19:57:31 DEBUG } 19:57:31 DEBUG ] 19:57:31 DEBUG } 19:57:31 DEBUG } 19:57:32 INFO 19:57:32 INFO [loop_until]: kubectl --namespace=xlou top pods 19:57:32 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:57:32 INFO [loop_until]: OK (rc = 0) 19:57:32 DEBUG --- stdout --- 19:57:32 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 15941m 9004Mi am-767f6b67b5-qcqz2 16407m 8808Mi am-767f6b67b5-vbltz 17701m 8998Mi ds-cts-0 16921m 10965Mi ds-cts-1 15472m 11219Mi ds-cts-2 17041m 10968Mi ds-idrepo-0 7186m 3695Mi ds-idrepo-1 6935m 3537Mi ds-idrepo-2 7311m 3492Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8278m 3327Mi 19:57:32 DEBUG --- stderr --- 19:57:32 DEBUG 19:57:33 INFO 19:57:33 INFO [loop_until]: kubectl --namespace=xlou top node 19:57:33 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:57:33 INFO [loop_until]: OK (rc = 0) 19:57:33 DEBUG --- stdout --- 19:57:33 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 69m 0% 6255Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7403m 24% 7293Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 15988m 53% 11702Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 80m 0% 3345Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 17031m 57% 13943Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 160m 0% 5219Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 16419m 54% 11684Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15394m 51% 2993Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 18163m 60% 11199Mi 10% gke-xlou-cdm-ds-0f601751-j488 7588m 25% 6743Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 15854m 53% 15956Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 7319m 24% 7441Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 17337m 58% 13416Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 8199m 51% 7073Mi 12% 19:57:33 DEBUG --- stderr --- 19:57:33 DEBUG 19:58:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553111" 19:58:31 INFO [http_cmd]: http status code OK 19:58:31 DEBUG --- status code --- 19:58:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:31 DEBUG --- http response --- 19:58:31 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:58:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553111" 19:58:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553111" 19:58:31 INFO [http_cmd]: http status code OK 19:58:31 DEBUG --- status code --- 19:58:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:31 DEBUG --- http response --- 19:58:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:58:31 DEBUG ----- output ----- 19:58:31 DEBUG { 19:58:31 DEBUG "status": "success", 19:58:31 DEBUG "data": { 19:58:31 DEBUG "resultType": "vector", 19:58:31 DEBUG "result": [ 19:58:31 DEBUG { 19:58:31 DEBUG "metric": { 19:58:31 DEBUG "pod": "ds-idrepo-0" 19:58:31 DEBUG }, 19:58:31 DEBUG "value": [ 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG [...] 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG "pod": "ds-idrepo-2" 19:58:31 DEBUG }, 19:58:31 DEBUG "value": [ 19:58:31 DEBUG 1713553111, 19:58:31 DEBUG "0" 19:58:31 DEBUG ] 19:58:31 DEBUG } 19:58:31 DEBUG ] 19:58:31 DEBUG } 19:58:31 DEBUG } 19:58:31 INFO [http_cmd]: http status code OK 19:58:31 DEBUG --- status code --- 19:58:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:31 DEBUG --- http response --- 19:58:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:58:31 DEBUG ----- output ----- 19:58:31 DEBUG { 19:58:31 DEBUG "status": "success", 19:58:31 DEBUG "data": { 19:58:31 DEBUG "resultType": "vector", 19:58:31 DEBUG "result": [ 19:58:31 DEBUG { 19:58:31 DEBUG "metric": { 19:58:31 DEBUG "pod": "ds-cts-0" 19:58:31 DEBUG }, 19:58:31 DEBUG "value": [ 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG [...] 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG "pod": "ds-cts-2" 19:58:31 DEBUG }, 19:58:31 DEBUG "value": [ 19:58:31 DEBUG 1713553111, 19:58:31 DEBUG "0.00019999999999999998" 19:58:31 DEBUG ] 19:58:31 DEBUG } 19:58:31 DEBUG ] 19:58:31 DEBUG } 19:58:31 DEBUG } 19:58:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553111" 19:58:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553111" 19:58:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553111" 19:58:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713553111" 19:58:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553111" 19:58:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713553111" 19:58:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713553111" 19:58:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553111" 19:58:31 INFO [http_cmd]: http status code OK 19:58:31 DEBUG --- status code --- 19:58:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:31 DEBUG --- http response --- 19:58:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:58:31 DEBUG ----- output ----- 19:58:31 DEBUG { 19:58:31 DEBUG "status": "success", 19:58:31 DEBUG "data": { 19:58:31 DEBUG "resultType": "vector", 19:58:31 DEBUG "result": [ 19:58:31 DEBUG { 19:58:31 DEBUG "metric": { 19:58:31 DEBUG "pod": "am-767f6b67b5-9zw22" 19:58:31 DEBUG }, 19:58:31 DEBUG "value": [ 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG [...] 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG "pod": "am-767f6b67b5-vbltz" 19:58:31 DEBUG }, 19:58:31 DEBUG "value": [ 19:58:31 DEBUG 1713553111, 19:58:31 DEBUG "0" 19:58:31 DEBUG ] 19:58:31 DEBUG } 19:58:31 DEBUG ] 19:58:31 DEBUG } 19:58:31 DEBUG } 19:58:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713553111" 19:58:31 INFO [http_cmd]: http status code OK 19:58:31 INFO [http_cmd]: http status code OK 19:58:31 DEBUG --- status code --- 19:58:31 DEBUG --- status code --- 19:58:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:31 DEBUG --- http response --- 19:58:31 DEBUG --- http response --- 19:58:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:58:31 DEBUG ----- output ----- 19:58:31 DEBUG { 19:58:31 DEBUG "status": "success", 19:58:31 DEBUG "data": { 19:58:31 DEBUG "resultType": "vector", 19:58:31 DEBUG "result": [ 19:58:31 DEBUG { 19:58:31 DEBUG "metric": { 19:58:31 DEBUG "pod": "am-767f6b67b5-9zw22" 19:58:31 DEBUG }, 19:58:31 DEBUG "value": [ 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:58:31 DEBUG 19:58:31 INFO [http_cmd]: http status code OK 19:58:31 INFO [http_cmd]: http status code OK 19:58:31 INFO [http_cmd]: http status code OK 19:58:31 DEBUG ----- output ----- 19:58:31 DEBUG [...] 19:58:31 DEBUG --- status code --- 19:58:31 DEBUG --- status code --- 19:58:31 DEBUG --- status code --- 19:58:31 DEBUG { 19:58:31 DEBUG 19:58:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:31 INFO [http_cmd]: http status code OK 19:58:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:31 INFO [http_cmd]: http status code OK 19:58:31 DEBUG "status": "success", 19:58:31 DEBUG 19:58:31 DEBUG --- http response --- 19:58:31 DEBUG --- http response --- 19:58:31 DEBUG --- status code --- 19:58:31 DEBUG --- http response --- 19:58:31 DEBUG --- status code --- 19:58:31 DEBUG "data": { 19:58:31 DEBUG 19:58:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:58:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:58:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:58:31 DEBUG "resultType": "vector", 19:58:31 DEBUG "pod": "am-767f6b67b5-vbltz" 19:58:31 DEBUG ----- output ----- 19:58:31 DEBUG --- http response --- 19:58:31 DEBUG ----- output ----- 19:58:31 DEBUG --- http response --- 19:58:31 DEBUG ----- output ----- 19:58:31 DEBUG "result": [ 19:58:31 DEBUG }, 19:58:31 DEBUG { 19:58:31 DEBUG { 19:58:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:58:31 DEBUG { 19:58:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:58:31 DEBUG { 19:58:31 DEBUG "value": [ 19:58:31 DEBUG "status": "success", 19:58:31 DEBUG "status": "success", 19:58:31 DEBUG ----- output ----- 19:58:31 DEBUG "status": "success", 19:58:31 DEBUG ----- output ----- 19:58:31 DEBUG "metric": { 19:58:31 DEBUG 1713553111, 19:58:31 DEBUG "data": { 19:58:31 DEBUG "data": { 19:58:31 DEBUG { 19:58:31 DEBUG "data": { 19:58:31 DEBUG { 19:58:31 DEBUG "pod": "ds-idrepo-0" 19:58:31 DEBUG "450017" 19:58:31 DEBUG "resultType": "vector", 19:58:31 DEBUG "resultType": "vector", 19:58:31 DEBUG "status": "success", 19:58:31 DEBUG "resultType": "vector", 19:58:31 DEBUG "status": "success", 19:58:31 DEBUG }, 19:58:31 DEBUG ] 19:58:31 DEBUG "result": [ 19:58:31 DEBUG "result": [ 19:58:31 DEBUG "data": { 19:58:31 DEBUG "result": [ 19:58:31 DEBUG "data": { 19:58:31 DEBUG "value": [ 19:58:31 DEBUG } 19:58:31 DEBUG { 19:58:31 DEBUG { 19:58:31 DEBUG "resultType": "vector", 19:58:31 DEBUG { 19:58:31 DEBUG "resultType": "vector", 19:58:31 DEBUG 19:58:31 DEBUG ] 19:58:31 DEBUG "metric": { 19:58:31 DEBUG "metric": { 19:58:31 DEBUG "result": [ 19:58:31 DEBUG "metric": { 19:58:31 DEBUG "result": [ 19:58:31 DEBUG 19:58:31 DEBUG } 19:58:31 DEBUG "pod": "am-767f6b67b5-9zw22" 19:58:31 DEBUG "pod": "ds-idrepo-0" 19:58:31 DEBUG { 19:58:31 DEBUG "pod": "ds-idrepo-1" 19:58:31 DEBUG { 19:58:31 DEBUG 19:58:31 DEBUG } 19:58:31 DEBUG }, 19:58:31 DEBUG }, 19:58:31 DEBUG "metric": { 19:58:31 DEBUG }, 19:58:31 DEBUG "metric": { 19:58:31 DEBUG [...] 19:58:31 DEBUG "value": [ 19:58:31 DEBUG "value": [ 19:58:31 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:58:31 DEBUG "value": [ 19:58:31 DEBUG "instance": "10.142.0.107:9100" 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 ERROR [http_cmd]: ERROR 19:58:31 DEBUG 19:58:31 DEBUG }, 19:58:31 DEBUG 19:58:31 DEBUG }, 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 ERROR --- status code --- 19:58:31 DEBUG 19:58:31 DEBUG "value": [ 19:58:31 DEBUG 19:58:31 DEBUG "value": [ 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG "pod": "ds-cts-2" 19:58:31 DEBUG [...] 19:58:31 ERROR --- http response --- 19:58:31 DEBUG [...] 19:58:31 DEBUG 19:58:31 DEBUG [...] 19:58:31 DEBUG 19:58:31 DEBUG }, 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG "value": [ 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:58:31 DEBUG [...] 19:58:31 DEBUG 19:58:31 DEBUG [...] 19:58:31 DEBUG 1713553111, 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG "0" 19:58:31 DEBUG "pod": "am-767f6b67b5-vbltz" 19:58:31 DEBUG "pod": "ds-idrepo-2" 19:58:31 DEBUG 19:58:31 DEBUG "pod": "ds-cts-1" 19:58:31 DEBUG 19:58:31 DEBUG ] 19:58:31 DEBUG }, 19:58:31 DEBUG }, 19:58:31 DEBUG 19:58:31 DEBUG }, 19:58:31 DEBUG 19:58:31 DEBUG } 19:58:31 DEBUG "value": [ 19:58:31 DEBUG "value": [ 19:58:31 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:58:31 DEBUG "value": [ 19:58:31 DEBUG "instance": "10.142.0.43:9100" 19:58:31 DEBUG ] 19:58:31 DEBUG 1713553111, 19:58:31 DEBUG 1713553111, 19:58:31 DEBUG }, 19:58:31 DEBUG 1713553111, 19:58:31 DEBUG }, 19:58:31 DEBUG } 19:58:31 DEBUG "0.014" 19:58:31 DEBUG "0" 19:58:31 DEBUG "value": [ 19:58:31 DEBUG "62453310.92654236" 19:58:31 DEBUG "value": [ 19:58:31 DEBUG } 19:58:31 DEBUG ] 19:58:31 DEBUG ] 19:58:31 DEBUG 1713553111, 19:58:31 DEBUG ] 19:58:31 DEBUG 1713553111, 19:58:31 DEBUG } 19:58:31 DEBUG } 19:58:31 DEBUG "8.035876078135155" 19:58:31 DEBUG } 19:58:31 DEBUG "0.16830000000001538" 19:58:31 DEBUG ] 19:58:31 DEBUG ] 19:58:31 DEBUG ] 19:58:31 DEBUG ] 19:58:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553111" 19:58:31 DEBUG ] 19:58:31 DEBUG } 19:58:31 DEBUG } 19:58:31 DEBUG } 19:58:31 DEBUG } 19:58:31 DEBUG } 19:58:31 DEBUG } 19:58:31 DEBUG } 19:58:31 DEBUG ] 19:58:31 DEBUG } 19:58:31 DEBUG ] 19:58:31 DEBUG } 19:58:31 DEBUG } 19:58:31 DEBUG } 19:58:31 DEBUG } 19:58:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553111" 19:58:31 INFO [http_cmd]: http status code OK 19:58:31 DEBUG --- status code --- 19:58:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:31 DEBUG --- http response --- 19:58:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:58:31 DEBUG ----- output ----- 19:58:31 DEBUG { 19:58:31 DEBUG "status": "success", 19:58:31 DEBUG "data": { 19:58:31 DEBUG "resultType": "vector", 19:58:31 DEBUG "result": [ 19:58:31 DEBUG { 19:58:31 DEBUG "metric": { 19:58:31 DEBUG "pod": "ds-cts-0" 19:58:31 DEBUG }, 19:58:31 DEBUG "value": [ 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG [...] 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG "pod": "ds-cts-2" 19:58:31 DEBUG }, 19:58:31 DEBUG "value": [ 19:58:31 DEBUG 1713553111, 19:58:31 DEBUG "8.733333333333333" 19:58:31 DEBUG ] 19:58:31 DEBUG } 19:58:31 DEBUG ] 19:58:31 DEBUG } 19:58:31 DEBUG } 19:58:31 INFO [http_cmd]: http status code OK 19:58:31 DEBUG --- status code --- 19:58:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:31 DEBUG --- http response --- 19:58:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:58:31 DEBUG ----- output ----- 19:58:31 DEBUG { 19:58:31 DEBUG "status": "success", 19:58:31 DEBUG "data": { 19:58:31 DEBUG "resultType": "vector", 19:58:31 DEBUG "result": [ 19:58:31 DEBUG { 19:58:31 DEBUG "metric": { 19:58:31 DEBUG "pod": "ds-idrepo-0" 19:58:31 DEBUG }, 19:58:31 DEBUG "value": [ 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG [...] 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG "pod": "ds-idrepo-2" 19:58:31 DEBUG }, 19:58:31 DEBUG "value": [ 19:58:31 DEBUG 1713553111, 19:58:31 DEBUG "0" 19:58:31 DEBUG ] 19:58:31 DEBUG } 19:58:31 DEBUG ] 19:58:31 DEBUG } 19:58:31 DEBUG } 19:58:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553111" 19:58:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553111" 19:58:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553111" 19:58:31 INFO [http_cmd]: http status code OK 19:58:31 INFO [http_cmd]: http status code OK 19:58:31 DEBUG --- status code --- 19:58:31 DEBUG --- status code --- 19:58:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:31 DEBUG --- http response --- 19:58:31 DEBUG --- http response --- 19:58:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:58:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:58:31 DEBUG ----- output ----- 19:58:31 DEBUG ----- output ----- 19:58:31 DEBUG { 19:58:31 DEBUG { 19:58:31 DEBUG "status": "success", 19:58:31 DEBUG "status": "success", 19:58:31 DEBUG "data": { 19:58:31 DEBUG "data": { 19:58:31 DEBUG "resultType": "vector", 19:58:31 DEBUG "resultType": "vector", 19:58:31 DEBUG "result": [ 19:58:31 DEBUG "result": [ 19:58:31 DEBUG { 19:58:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553111" 19:58:31 DEBUG { 19:58:31 DEBUG "metric": { 19:58:31 DEBUG "metric": { 19:58:31 DEBUG "pod": "am-767f6b67b5-9zw22" 19:58:31 DEBUG "instance": "10.142.0.107:9100" 19:58:31 DEBUG }, 19:58:31 DEBUG }, 19:58:31 DEBUG "value": [ 19:58:31 DEBUG "value": [ 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG [...] 19:58:31 DEBUG [...] 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG "pod": "am-767f6b67b5-vbltz" 19:58:31 DEBUG "instance": "10.142.0.43:9100" 19:58:31 DEBUG }, 19:58:31 DEBUG }, 19:58:31 DEBUG "value": [ 19:58:31 DEBUG "value": [ 19:58:31 DEBUG 1713553111, 19:58:31 DEBUG 1713553111, 19:58:31 DEBUG "2499.366666666667" 19:58:31 DEBUG "1990738394112" 19:58:31 DEBUG ] 19:58:31 DEBUG ] 19:58:31 DEBUG } 19:58:31 DEBUG } 19:58:31 DEBUG ] 19:58:31 DEBUG ] 19:58:31 DEBUG } 19:58:31 DEBUG } 19:58:31 DEBUG } 19:58:31 DEBUG } 19:58:31 INFO [http_cmd]: http status code OK 19:58:31 DEBUG --- status code --- 19:58:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:31 DEBUG --- http response --- 19:58:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:58:31 DEBUG ----- output ----- 19:58:31 DEBUG { 19:58:31 DEBUG "status": "success", 19:58:31 DEBUG "data": { 19:58:31 DEBUG "resultType": "vector", 19:58:31 DEBUG "result": [ 19:58:31 DEBUG { 19:58:31 DEBUG "metric": { 19:58:31 DEBUG "instance": "10.142.0.107:9100" 19:58:31 DEBUG }, 19:58:31 DEBUG "value": [ 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG [...] 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG "instance": "10.142.0.43:9100" 19:58:31 DEBUG }, 19:58:31 DEBUG "value": [ 19:58:31 DEBUG 1713553111, 19:58:31 DEBUG "0.07433333333332826" 19:58:31 DEBUG ] 19:58:31 DEBUG } 19:58:31 DEBUG ] 19:58:31 DEBUG } 19:58:31 DEBUG } 19:58:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553111" 19:58:31 INFO [http_cmd]: http status code OK 19:58:31 DEBUG --- status code --- 19:58:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:31 DEBUG --- http response --- 19:58:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:58:31 DEBUG ----- output ----- 19:58:31 DEBUG { 19:58:31 DEBUG "status": "success", 19:58:31 DEBUG "data": { 19:58:31 DEBUG "resultType": "vector", 19:58:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553111" 19:58:31 DEBUG "result": [ 19:58:31 DEBUG { 19:58:31 DEBUG "metric": { 19:58:31 DEBUG "pod": "am-767f6b67b5-9zw22" 19:58:31 DEBUG }, 19:58:31 DEBUG "value": [ 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG [...] 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG "pod": "am-767f6b67b5-vbltz" 19:58:31 DEBUG }, 19:58:31 DEBUG "value": [ 19:58:31 DEBUG 1713553111, 19:58:31 DEBUG "2496.9333333333334" 19:58:31 DEBUG ] 19:58:31 DEBUG } 19:58:31 DEBUG ] 19:58:31 DEBUG } 19:58:31 DEBUG } 19:58:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553111" 19:58:31 INFO [http_cmd]: http status code OK 19:58:31 DEBUG --- status code --- 19:58:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:31 DEBUG --- http response --- 19:58:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:58:31 DEBUG ----- output ----- 19:58:31 DEBUG { 19:58:31 DEBUG "status": "success", 19:58:31 DEBUG "data": { 19:58:31 DEBUG "resultType": "vector", 19:58:31 DEBUG "result": [ 19:58:31 DEBUG { 19:58:31 DEBUG "metric": { 19:58:31 DEBUG "pod": "ds-idrepo-1" 19:58:31 DEBUG }, 19:58:31 DEBUG "value": [ 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG [...] 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:58:31 DEBUG }, 19:58:31 DEBUG "value": [ 19:58:31 DEBUG 1713553111, 19:58:31 DEBUG "0" 19:58:31 DEBUG ] 19:58:31 DEBUG } 19:58:31 DEBUG ] 19:58:31 DEBUG } 19:58:31 DEBUG } 19:58:31 INFO [http_cmd]: http status code OK 19:58:31 DEBUG --- status code --- 19:58:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:31 DEBUG --- http response --- 19:58:31 INFO [http_cmd]: http status code OK 19:58:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:58:31 DEBUG --- status code --- 19:58:31 DEBUG ----- output ----- 19:58:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:31 DEBUG { 19:58:31 DEBUG --- http response --- 19:58:31 DEBUG "status": "success", 19:58:31 DEBUG "data": { 19:58:31 DEBUG "resultType": "vector", 19:58:31 DEBUG "result": [ 19:58:31 DEBUG { 19:58:31 DEBUG "metric": { 19:58:31 DEBUG "pod": "am-767f6b67b5-9zw22" 19:58:31 DEBUG }, 19:58:31 DEBUG "value": [ 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG [...] 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG "pod": "am-767f6b67b5-vbltz" 19:58:31 DEBUG }, 19:58:31 DEBUG "value": [ 19:58:31 DEBUG 1713553111, 19:58:31 DEBUG "2497" 19:58:31 DEBUG ] 19:58:31 DEBUG } 19:58:31 DEBUG ] 19:58:31 DEBUG } 19:58:31 DEBUG } 19:58:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:58:31 DEBUG ----- output ----- 19:58:31 DEBUG { 19:58:31 DEBUG "status": "success", 19:58:31 DEBUG "data": { 19:58:31 DEBUG "resultType": "vector", 19:58:31 DEBUG "result": [ 19:58:31 DEBUG { 19:58:31 DEBUG "metric": { 19:58:31 DEBUG "instance": "10.142.0.107:9100" 19:58:31 DEBUG }, 19:58:31 DEBUG "value": [ 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG [...] 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG "instance": "10.142.0.43:9100" 19:58:31 DEBUG }, 19:58:31 DEBUG "value": [ 19:58:31 DEBUG 1713553111, 19:58:31 DEBUG "75344967.8" 19:58:31 DEBUG ] 19:58:31 DEBUG } 19:58:31 DEBUG ] 19:58:31 DEBUG } 19:58:31 DEBUG } 19:58:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713553111" 19:58:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553111" 19:58:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553111" 19:58:31 INFO [http_cmd]: http status code OK 19:58:31 DEBUG --- status code --- 19:58:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:31 DEBUG --- http response --- 19:58:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:58:31 DEBUG ----- output ----- 19:58:31 DEBUG { 19:58:31 DEBUG "status": "success", 19:58:31 DEBUG "data": { 19:58:31 DEBUG "resultType": "vector", 19:58:31 DEBUG "result": [ 19:58:31 DEBUG { 19:58:31 DEBUG "metric": { 19:58:31 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:58:31 DEBUG }, 19:58:31 DEBUG "value": [ 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG [...] 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:58:31 DEBUG }, 19:58:31 DEBUG "value": [ 19:58:31 DEBUG 1713553111, 19:58:31 DEBUG "3589054464" 19:58:31 DEBUG ] 19:58:31 DEBUG } 19:58:31 DEBUG ] 19:58:31 DEBUG } 19:58:31 DEBUG } 19:58:31 INFO [http_cmd]: http status code OK 19:58:31 INFO [http_cmd]: http status code OK 19:58:31 DEBUG --- status code --- 19:58:31 DEBUG --- status code --- 19:58:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:31 DEBUG --- http response --- 19:58:31 DEBUG --- http response --- 19:58:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:58:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:58:31 DEBUG ----- output ----- 19:58:31 DEBUG ----- output ----- 19:58:31 DEBUG { 19:58:31 DEBUG { 19:58:31 DEBUG "status": "success", 19:58:31 DEBUG "status": "success", 19:58:31 DEBUG "data": { 19:58:31 DEBUG "data": { 19:58:31 DEBUG "resultType": "vector", 19:58:31 DEBUG "resultType": "vector", 19:58:31 DEBUG "result": [ 19:58:31 DEBUG "result": [ 19:58:31 DEBUG { 19:58:31 DEBUG { 19:58:31 DEBUG "metric": { 19:58:31 DEBUG "metric": { 19:58:31 DEBUG "pod": "ds-idrepo-1" 19:58:31 DEBUG "instance": "10.142.0.107:9100" 19:58:31 DEBUG }, 19:58:31 DEBUG }, 19:58:31 DEBUG "value": [ 19:58:31 DEBUG "value": [ 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG [...] 19:58:31 DEBUG [...] 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:58:31 DEBUG "instance": "10.142.0.43:9100" 19:58:31 DEBUG }, 19:58:31 DEBUG }, 19:58:31 DEBUG "value": [ 19:58:31 DEBUG "value": [ 19:58:31 DEBUG 1713553111, 19:58:31 DEBUG 1713553111, 19:58:31 DEBUG "0" 19:58:31 DEBUG "75341102.36666667" 19:58:31 DEBUG ] 19:58:31 DEBUG ] 19:58:31 DEBUG } 19:58:31 DEBUG } 19:58:31 DEBUG ] 19:58:31 DEBUG ] 19:58:31 DEBUG } 19:58:31 DEBUG } 19:58:31 DEBUG } 19:58:31 DEBUG } 19:58:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553111" 19:58:31 INFO [http_cmd]: http status code OK 19:58:31 DEBUG --- status code --- 19:58:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:31 DEBUG --- http response --- 19:58:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:58:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553111" 19:58:31 DEBUG ----- output ----- 19:58:31 DEBUG { 19:58:31 DEBUG "status": "success", 19:58:31 DEBUG "data": { 19:58:31 DEBUG "resultType": "vector", 19:58:31 DEBUG "result": [ 19:58:31 DEBUG { 19:58:31 DEBUG "metric": { 19:58:31 DEBUG "pod": "ds-idrepo-1" 19:58:31 DEBUG }, 19:58:31 DEBUG "value": [ 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG [...] 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG "pod": "ds-cts-1" 19:58:31 DEBUG }, 19:58:31 DEBUG "value": [ 19:58:31 DEBUG 1713553111, 19:58:31 DEBUG "32335804.02675888" 19:58:31 DEBUG ] 19:58:31 DEBUG } 19:58:31 DEBUG ] 19:58:31 DEBUG } 19:58:31 DEBUG } 19:58:31 INFO [http_cmd]: http status code OK 19:58:31 DEBUG --- status code --- 19:58:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:31 DEBUG --- http response --- 19:58:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:58:31 DEBUG ----- output ----- 19:58:31 DEBUG { 19:58:31 DEBUG "status": "success", 19:58:31 DEBUG "data": { 19:58:31 DEBUG "resultType": "vector", 19:58:31 DEBUG "result": [ 19:58:31 DEBUG { 19:58:31 DEBUG "metric": { 19:58:31 DEBUG "pod": "ds-cts-0" 19:58:31 DEBUG }, 19:58:31 DEBUG "value": [ 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG [...] 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG "pod": "ds-cts-2" 19:58:31 DEBUG }, 19:58:31 DEBUG "value": [ 19:58:31 DEBUG 1713553111, 19:58:31 DEBUG "0" 19:58:31 DEBUG ] 19:58:31 DEBUG } 19:58:31 DEBUG ] 19:58:31 DEBUG } 19:58:31 DEBUG } 19:58:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553111" 19:58:31 INFO [http_cmd]: http status code OK 19:58:31 DEBUG --- status code --- 19:58:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:31 DEBUG --- http response --- 19:58:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:58:31 DEBUG ----- output ----- 19:58:31 DEBUG { 19:58:31 DEBUG "status": "success", 19:58:31 DEBUG "data": { 19:58:31 DEBUG "resultType": "vector", 19:58:31 DEBUG "result": [ 19:58:31 DEBUG { 19:58:31 DEBUG "metric": { 19:58:31 DEBUG "pod": "am-767f6b67b5-9zw22" 19:58:31 DEBUG }, 19:58:31 DEBUG "value": [ 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG [...] 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG 19:58:31 DEBUG "pod": "am-767f6b67b5-vbltz" 19:58:31 DEBUG }, 19:58:31 DEBUG "value": [ 19:58:31 DEBUG 1713553111, 19:58:31 DEBUG "2497.633333333333" 19:58:31 DEBUG ] 19:58:31 DEBUG } 19:58:31 DEBUG ] 19:58:31 DEBUG } 19:58:31 DEBUG } 19:58:32 INFO 19:58:32 INFO [loop_until]: kubectl --namespace=xlou top pods 19:58:32 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:58:32 INFO [loop_until]: OK (rc = 0) 19:58:32 DEBUG --- stdout --- 19:58:32 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 15813m 9009Mi am-767f6b67b5-qcqz2 16570m 8811Mi am-767f6b67b5-vbltz 17614m 8996Mi ds-cts-0 17464m 10971Mi ds-cts-1 15536m 11217Mi ds-cts-2 18183m 10964Mi ds-idrepo-0 7135m 3695Mi ds-idrepo-1 6966m 3536Mi ds-idrepo-2 7389m 3493Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7850m 3391Mi 19:58:32 DEBUG --- stderr --- 19:58:32 DEBUG 19:58:33 INFO 19:58:33 INFO [loop_until]: kubectl --namespace=xlou top node 19:58:33 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:58:33 INFO [loop_until]: OK (rc = 0) 19:58:33 DEBUG --- stdout --- 19:58:33 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 69m 0% 6257Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7365m 24% 7292Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16148m 54% 11718Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 81m 0% 3348Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 17345m 58% 13940Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 156m 0% 5225Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 16533m 55% 11690Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15677m 52% 2993Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 18178m 60% 11205Mi 10% gke-xlou-cdm-ds-0f601751-j488 7393m 24% 6734Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 15936m 53% 15961Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 7426m 24% 7427Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 17720m 59% 13409Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 7924m 49% 7136Mi 12% 19:58:33 DEBUG --- stderr --- 19:58:33 DEBUG 19:59:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553171" 19:59:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553171" 19:59:31 INFO [http_cmd]: http status code OK 19:59:31 DEBUG --- status code --- 19:59:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:31 DEBUG --- http response --- 19:59:31 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:59:31 INFO [http_cmd]: http status code OK 19:59:31 DEBUG --- status code --- 19:59:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:31 DEBUG --- http response --- 19:59:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:59:31 DEBUG ----- output ----- 19:59:31 DEBUG { 19:59:31 DEBUG "status": "success", 19:59:31 DEBUG "data": { 19:59:31 DEBUG "resultType": "vector", 19:59:31 DEBUG "result": [ 19:59:31 DEBUG { 19:59:31 DEBUG "metric": { 19:59:31 DEBUG "pod": "ds-idrepo-0" 19:59:31 DEBUG }, 19:59:31 DEBUG "value": [ 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG [...] 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG "pod": "ds-idrepo-2" 19:59:31 DEBUG }, 19:59:31 DEBUG "value": [ 19:59:31 DEBUG 1713553171, 19:59:31 DEBUG "0" 19:59:31 DEBUG ] 19:59:31 DEBUG } 19:59:31 DEBUG ] 19:59:31 DEBUG } 19:59:31 DEBUG } 19:59:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553171" 19:59:31 INFO [http_cmd]: http status code OK 19:59:31 DEBUG --- status code --- 19:59:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:31 DEBUG --- http response --- 19:59:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:59:31 DEBUG ----- output ----- 19:59:31 DEBUG { 19:59:31 DEBUG "status": "success", 19:59:31 DEBUG "data": { 19:59:31 DEBUG "resultType": "vector", 19:59:31 DEBUG "result": [ 19:59:31 DEBUG { 19:59:31 DEBUG "metric": { 19:59:31 DEBUG "pod": "ds-cts-0" 19:59:31 DEBUG }, 19:59:31 DEBUG "value": [ 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG [...] 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG "pod": "ds-cts-2" 19:59:31 DEBUG }, 19:59:31 DEBUG "value": [ 19:59:31 DEBUG 1713553171, 19:59:31 DEBUG "0.00006666666666666667" 19:59:31 DEBUG ] 19:59:31 DEBUG } 19:59:31 DEBUG ] 19:59:31 DEBUG } 19:59:31 DEBUG } 19:59:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553171" 19:59:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553171" 19:59:31 INFO [http_cmd]: http status code OK 19:59:31 DEBUG --- status code --- 19:59:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:31 DEBUG --- http response --- 19:59:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:59:31 DEBUG ----- output ----- 19:59:31 DEBUG { 19:59:31 DEBUG "status": "success", 19:59:31 DEBUG "data": { 19:59:31 DEBUG "resultType": "vector", 19:59:31 DEBUG "result": [ 19:59:31 DEBUG { 19:59:31 DEBUG "metric": { 19:59:31 DEBUG "pod": "am-767f6b67b5-9zw22" 19:59:31 DEBUG }, 19:59:31 DEBUG "value": [ 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG [...] 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG "pod": "am-767f6b67b5-vbltz" 19:59:31 DEBUG }, 19:59:31 DEBUG "value": [ 19:59:31 DEBUG 1713553171, 19:59:31 DEBUG "450110" 19:59:31 DEBUG ] 19:59:31 DEBUG } 19:59:31 DEBUG ] 19:59:31 DEBUG } 19:59:31 DEBUG } 19:59:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713553171" 19:59:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553171" 19:59:31 INFO [http_cmd]: http status code OK 19:59:31 INFO [http_cmd]: http status code OK 19:59:31 DEBUG --- status code --- 19:59:31 DEBUG --- status code --- 19:59:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:31 DEBUG --- http response --- 19:59:31 DEBUG --- http response --- 19:59:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:59:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:59:31 DEBUG ----- output ----- 19:59:31 DEBUG ----- output ----- 19:59:31 DEBUG { 19:59:31 DEBUG { 19:59:31 DEBUG "status": "success", 19:59:31 DEBUG "status": "success", 19:59:31 DEBUG "data": { 19:59:31 DEBUG "data": { 19:59:31 DEBUG "resultType": "vector", 19:59:31 DEBUG "resultType": "vector", 19:59:31 DEBUG "result": [ 19:59:31 DEBUG "result": [ 19:59:31 DEBUG { 19:59:31 DEBUG { 19:59:31 DEBUG "metric": { 19:59:31 DEBUG "metric": { 19:59:31 DEBUG "pod": "am-767f6b67b5-9zw22" 19:59:31 DEBUG "pod": "ds-idrepo-0" 19:59:31 DEBUG }, 19:59:31 DEBUG }, 19:59:31 DEBUG "value": [ 19:59:31 DEBUG "value": [ 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG [...] 19:59:31 DEBUG [...] 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG "pod": "am-767f6b67b5-vbltz" 19:59:31 DEBUG "pod": "ds-cts-2" 19:59:31 DEBUG }, 19:59:31 DEBUG }, 19:59:31 DEBUG "value": [ 19:59:31 DEBUG "value": [ 19:59:31 DEBUG 1713553171, 19:59:31 DEBUG 1713553171, 19:59:31 DEBUG "0" 19:59:31 DEBUG "0" 19:59:31 DEBUG ] 19:59:31 DEBUG ] 19:59:31 DEBUG } 19:59:31 DEBUG } 19:59:31 DEBUG ] 19:59:31 DEBUG ] 19:59:31 DEBUG } 19:59:31 DEBUG } 19:59:31 DEBUG } 19:59:31 DEBUG } 19:59:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553171" 19:59:31 INFO [http_cmd]: http status code OK 19:59:31 DEBUG --- status code --- 19:59:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:31 DEBUG --- http response --- 19:59:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:59:31 DEBUG ----- output ----- 19:59:31 DEBUG { 19:59:31 DEBUG "status": "success", 19:59:31 DEBUG "data": { 19:59:31 DEBUG "resultType": "vector", 19:59:31 DEBUG "result": [ 19:59:31 DEBUG { 19:59:31 DEBUG "metric": { 19:59:31 DEBUG "pod": "ds-idrepo-0" 19:59:31 DEBUG }, 19:59:31 DEBUG "value": [ 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG [...] 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG "pod": "ds-idrepo-2" 19:59:31 DEBUG }, 19:59:31 DEBUG "value": [ 19:59:31 DEBUG 1713553171, 19:59:31 DEBUG "0" 19:59:31 DEBUG ] 19:59:31 DEBUG } 19:59:31 DEBUG ] 19:59:31 DEBUG } 19:59:31 DEBUG } 19:59:31 INFO [http_cmd]: http status code OK 19:59:31 DEBUG --- status code --- 19:59:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:31 DEBUG --- http response --- 19:59:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:59:31 DEBUG ----- output ----- 19:59:31 DEBUG { 19:59:31 DEBUG "status": "success", 19:59:31 DEBUG "data": { 19:59:31 DEBUG "resultType": "vector", 19:59:31 DEBUG "result": [ 19:59:31 DEBUG { 19:59:31 DEBUG "metric": { 19:59:31 DEBUG "instance": "10.142.0.107:9100" 19:59:31 DEBUG }, 19:59:31 DEBUG "value": [ 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG [...] 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG "instance": "10.142.0.43:9100" 19:59:31 DEBUG }, 19:59:31 DEBUG "value": [ 19:59:31 DEBUG 1713553171, 19:59:31 DEBUG "0.23576666666672375" 19:59:31 DEBUG ] 19:59:31 DEBUG } 19:59:31 DEBUG ] 19:59:31 DEBUG } 19:59:31 DEBUG } 19:59:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713553171" 19:59:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713553171" 19:59:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553171" 19:59:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553171" 19:59:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713553171" 19:59:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553171" 19:59:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553171" 19:59:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553171" 19:59:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553171" 19:59:31 ERROR [http_cmd]: ERROR 19:59:31 ERROR --- status code --- 19:59:31 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:59:31 ERROR --- http response --- 19:59:31 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:59:31 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:59:31 INFO [http_cmd]: http status code OK 19:59:31 DEBUG --- status code --- 19:59:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:31 DEBUG --- http response --- 19:59:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:59:31 INFO [http_cmd]: http status code OK 19:59:31 INFO [http_cmd]: http status code OK 19:59:31 DEBUG ----- output ----- 19:59:31 DEBUG --- status code --- 19:59:31 DEBUG --- status code --- 19:59:31 DEBUG { 19:59:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:31 DEBUG "status": "success", 19:59:31 DEBUG --- http response --- 19:59:31 DEBUG --- http response --- 19:59:31 DEBUG "data": { 19:59:31 INFO [http_cmd]: http status code OK 19:59:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:59:31 DEBUG "resultType": "vector", 19:59:31 DEBUG "result": [ 19:59:31 DEBUG --- status code --- 19:59:31 INFO [http_cmd]: http status code OK 19:59:31 INFO [http_cmd]: http status code OK 19:59:31 DEBUG ----- output ----- 19:59:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:59:31 DEBUG { 19:59:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:31 DEBUG --- status code --- 19:59:31 DEBUG --- status code --- 19:59:31 DEBUG { 19:59:31 DEBUG ----- output ----- 19:59:31 DEBUG "metric": { 19:59:31 DEBUG --- http response --- 19:59:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:31 DEBUG "status": "success", 19:59:31 DEBUG { 19:59:31 DEBUG "pod": "am-767f6b67b5-9zw22" 19:59:31 DEBUG --- http response --- 19:59:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:59:31 DEBUG --- http response --- 19:59:31 DEBUG "data": { 19:59:31 DEBUG "status": "success", 19:59:31 DEBUG }, 19:59:31 INFO [http_cmd]: http status code OK 19:59:31 DEBUG ----- output ----- 19:59:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:59:31 DEBUG "resultType": "vector", 19:59:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:59:31 DEBUG "data": { 19:59:31 DEBUG "value": [ 19:59:31 DEBUG --- status code --- 19:59:31 DEBUG { 19:59:31 DEBUG ----- output ----- 19:59:31 DEBUG "result": [ 19:59:31 DEBUG ----- output ----- 19:59:31 DEBUG "resultType": "vector", 19:59:31 DEBUG 19:59:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:31 DEBUG "status": "success", 19:59:31 DEBUG { 19:59:31 DEBUG { 19:59:31 DEBUG { 19:59:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553171" 19:59:31 INFO [http_cmd]: http status code OK 19:59:31 DEBUG "result": [ 19:59:31 DEBUG 19:59:31 DEBUG --- http response --- 19:59:31 DEBUG "data": { 19:59:31 DEBUG "resultType": "vector", 19:59:31 DEBUG "status": "success", 19:59:31 DEBUG "status": "success", 19:59:31 DEBUG --- status code --- 19:59:31 DEBUG { 19:59:31 DEBUG 19:59:31 DEBUG "metric": { 19:59:31 DEBUG "result": [ 19:59:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:59:31 DEBUG "data": { 19:59:31 DEBUG "data": { 19:59:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:31 DEBUG "metric": { 19:59:31 DEBUG [...] 19:59:31 DEBUG "pod": "ds-idrepo-0" 19:59:31 DEBUG { 19:59:31 DEBUG ----- output ----- 19:59:31 DEBUG "resultType": "vector", 19:59:31 DEBUG "resultType": "vector", 19:59:31 DEBUG --- http response --- 19:59:31 DEBUG "instance": "10.142.0.107:9100" 19:59:31 DEBUG 19:59:31 DEBUG }, 19:59:31 DEBUG "metric": { 19:59:31 DEBUG { 19:59:31 DEBUG "result": [ 19:59:31 DEBUG "result": [ 19:59:31 DEBUG }, 19:59:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:59:31 DEBUG 19:59:31 DEBUG "value": [ 19:59:31 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:59:31 DEBUG "status": "success", 19:59:31 DEBUG { 19:59:31 DEBUG { 19:59:31 DEBUG "value": [ 19:59:31 DEBUG ----- output ----- 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG }, 19:59:31 DEBUG "data": { 19:59:31 DEBUG "metric": { 19:59:31 DEBUG "metric": { 19:59:31 DEBUG 19:59:31 DEBUG { 19:59:31 DEBUG "pod": "am-767f6b67b5-vbltz" 19:59:31 DEBUG 19:59:31 DEBUG "value": [ 19:59:31 DEBUG "resultType": "vector", 19:59:31 DEBUG "pod": "ds-idrepo-1" 19:59:31 DEBUG "instance": "10.142.0.107:9100" 19:59:31 DEBUG 19:59:31 DEBUG "status": "success", 19:59:31 DEBUG }, 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG "result": [ 19:59:31 DEBUG }, 19:59:31 DEBUG }, 19:59:31 DEBUG 19:59:31 DEBUG "data": { 19:59:31 DEBUG "value": [ 19:59:31 DEBUG [...] 19:59:31 DEBUG 19:59:31 DEBUG { 19:59:31 DEBUG "value": [ 19:59:31 DEBUG "value": [ 19:59:31 DEBUG [...] 19:59:31 DEBUG "resultType": "vector", 19:59:31 DEBUG 1713553171, 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG "metric": { 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG "result": [ 19:59:31 DEBUG "0.014" 19:59:31 DEBUG 19:59:31 DEBUG [...] 19:59:31 DEBUG "pod": "ds-idrepo-1" 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG { 19:59:31 DEBUG ] 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG }, 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG "metric": { 19:59:31 DEBUG } 19:59:31 DEBUG "pod": "ds-idrepo-2" 19:59:31 DEBUG 19:59:31 DEBUG "value": [ 19:59:31 DEBUG [...] 19:59:31 DEBUG [...] 19:59:31 DEBUG "instance": "10.142.0.43:9100" 19:59:31 DEBUG "pod": "ds-cts-0" 19:59:31 DEBUG ] 19:59:31 DEBUG }, 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG }, 19:59:31 DEBUG }, 19:59:31 DEBUG } 19:59:31 DEBUG "value": [ 19:59:31 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG "value": [ 19:59:31 DEBUG "value": [ 19:59:31 DEBUG } 19:59:31 DEBUG 1713553171, 19:59:31 DEBUG }, 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG 1713553171, 19:59:31 DEBUG 19:59:31 DEBUG "0" 19:59:31 DEBUG "value": [ 19:59:31 DEBUG [...] 19:59:31 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:59:31 DEBUG "instance": "10.142.0.43:9100" 19:59:31 DEBUG "1994802650112" 19:59:31 DEBUG 19:59:31 DEBUG ] 19:59:31 DEBUG 1713553171, 19:59:31 DEBUG 19:59:31 DEBUG }, 19:59:31 DEBUG }, 19:59:31 DEBUG ] 19:59:31 DEBUG 19:59:31 DEBUG } 19:59:31 DEBUG "8.037429202747896" 19:59:31 DEBUG 19:59:31 DEBUG "value": [ 19:59:31 DEBUG "value": [ 19:59:31 DEBUG } 19:59:31 DEBUG [...] 19:59:31 DEBUG ] 19:59:31 DEBUG ] 19:59:31 DEBUG 19:59:31 DEBUG 1713553171, 19:59:31 DEBUG 1713553171, 19:59:31 DEBUG ] 19:59:31 DEBUG 19:59:31 DEBUG } 19:59:31 DEBUG } 19:59:31 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:59:31 DEBUG "0" 19:59:31 DEBUG "75589689.23333333" 19:59:31 DEBUG } 19:59:31 DEBUG 19:59:31 DEBUG } 19:59:31 DEBUG ] 19:59:31 DEBUG }, 19:59:31 DEBUG ] 19:59:31 DEBUG ] 19:59:31 DEBUG } 19:59:31 DEBUG 19:59:31 DEBUG } 19:59:31 DEBUG "value": [ 19:59:31 DEBUG 1713553171, 19:59:31 DEBUG "90635056.67940424" 19:59:31 DEBUG "pod": "ds-cts-2" 19:59:31 DEBUG } 19:59:31 DEBUG } 19:59:31 DEBUG ] 19:59:31 DEBUG } 19:59:31 DEBUG }, 19:59:31 DEBUG ] 19:59:31 DEBUG } 19:59:31 DEBUG ] 19:59:31 DEBUG "value": [ 19:59:31 DEBUG } 19:59:31 DEBUG ] 19:59:31 DEBUG } 19:59:31 DEBUG 1713553171, 19:59:31 DEBUG } 19:59:31 DEBUG } 19:59:31 DEBUG } 19:59:31 DEBUG "30" 19:59:31 DEBUG } 19:59:31 DEBUG ] 19:59:31 DEBUG } 19:59:31 DEBUG ] 19:59:31 DEBUG } 19:59:31 DEBUG } 19:59:31 INFO [http_cmd]: http status code OK 19:59:31 DEBUG --- status code --- 19:59:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:31 DEBUG --- http response --- 19:59:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553171" 19:59:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:59:31 DEBUG ----- output ----- 19:59:31 DEBUG { 19:59:31 DEBUG "status": "success", 19:59:31 DEBUG "data": { 19:59:31 DEBUG "resultType": "vector", 19:59:31 DEBUG "result": [ 19:59:31 DEBUG { 19:59:31 DEBUG "metric": { 19:59:31 DEBUG "instance": "10.142.0.107:9100" 19:59:31 DEBUG }, 19:59:31 DEBUG "value": [ 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG [...] 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG "instance": "10.142.0.43:9100" 19:59:31 DEBUG }, 19:59:31 DEBUG "value": [ 19:59:31 DEBUG 1713553171, 19:59:31 DEBUG "0.10199999999999629" 19:59:31 DEBUG ] 19:59:31 DEBUG } 19:59:31 DEBUG ] 19:59:31 DEBUG } 19:59:31 DEBUG } 19:59:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553171" 19:59:31 INFO [http_cmd]: http status code OK 19:59:31 DEBUG --- status code --- 19:59:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:31 DEBUG --- http response --- 19:59:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:59:31 DEBUG ----- output ----- 19:59:31 DEBUG { 19:59:31 DEBUG "status": "success", 19:59:31 DEBUG "data": { 19:59:31 DEBUG "resultType": "vector", 19:59:31 DEBUG "result": [ 19:59:31 DEBUG { 19:59:31 DEBUG "metric": { 19:59:31 DEBUG "pod": "am-767f6b67b5-9zw22" 19:59:31 DEBUG }, 19:59:31 DEBUG "value": [ 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG [...] 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG "pod": "am-767f6b67b5-vbltz" 19:59:31 DEBUG }, 19:59:31 DEBUG "value": [ 19:59:31 DEBUG 1713553171, 19:59:31 DEBUG "2505.6666666666665" 19:59:31 DEBUG ] 19:59:31 DEBUG } 19:59:31 DEBUG ] 19:59:31 DEBUG } 19:59:31 DEBUG } 19:59:31 INFO [http_cmd]: http status code OK 19:59:31 DEBUG --- status code --- 19:59:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:31 DEBUG --- http response --- 19:59:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:59:31 DEBUG ----- output ----- 19:59:31 DEBUG { 19:59:31 DEBUG "status": "success", 19:59:31 DEBUG "data": { 19:59:31 DEBUG "resultType": "vector", 19:59:31 DEBUG "result": [ 19:59:31 DEBUG { 19:59:31 DEBUG "metric": { 19:59:31 DEBUG "pod": "am-767f6b67b5-9zw22" 19:59:31 DEBUG }, 19:59:31 DEBUG "value": [ 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG [...] 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG "pod": "am-767f6b67b5-vbltz" 19:59:31 DEBUG }, 19:59:31 DEBUG "value": [ 19:59:31 DEBUG 1713553171, 19:59:31 DEBUG "2503.6" 19:59:31 DEBUG ] 19:59:31 DEBUG } 19:59:31 DEBUG ] 19:59:31 DEBUG } 19:59:31 DEBUG } 19:59:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553171" 19:59:31 INFO [http_cmd]: http status code OK 19:59:31 DEBUG --- status code --- 19:59:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:31 DEBUG --- http response --- 19:59:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:59:31 DEBUG ----- output ----- 19:59:31 DEBUG { 19:59:31 DEBUG "status": "success", 19:59:31 DEBUG "data": { 19:59:31 DEBUG "resultType": "vector", 19:59:31 DEBUG "result": [ 19:59:31 DEBUG { 19:59:31 DEBUG "metric": { 19:59:31 DEBUG "pod": "am-767f6b67b5-9zw22" 19:59:31 DEBUG }, 19:59:31 DEBUG "value": [ 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG [...] 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG "pod": "am-767f6b67b5-vbltz" 19:59:31 DEBUG }, 19:59:31 DEBUG "value": [ 19:59:31 DEBUG 1713553171, 19:59:31 DEBUG "2503.866666666667" 19:59:31 DEBUG ] 19:59:31 DEBUG } 19:59:31 DEBUG ] 19:59:31 DEBUG } 19:59:31 DEBUG } 19:59:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553171" 19:59:31 INFO [http_cmd]: http status code OK 19:59:31 DEBUG --- status code --- 19:59:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:31 DEBUG --- http response --- 19:59:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:59:31 DEBUG ----- output ----- 19:59:31 DEBUG { 19:59:31 DEBUG "status": "success", 19:59:31 DEBUG "data": { 19:59:31 DEBUG "resultType": "vector", 19:59:31 DEBUG "result": [ 19:59:31 DEBUG { 19:59:31 DEBUG "metric": { 19:59:31 DEBUG "pod": "ds-idrepo-1" 19:59:31 DEBUG }, 19:59:31 DEBUG "value": [ 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG [...] 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:59:31 DEBUG }, 19:59:31 DEBUG "value": [ 19:59:31 DEBUG 1713553171, 19:59:31 DEBUG "0" 19:59:31 DEBUG ] 19:59:31 DEBUG } 19:59:31 DEBUG ] 19:59:31 DEBUG } 19:59:31 DEBUG } 19:59:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553171" 19:59:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713553171" 19:59:31 INFO [http_cmd]: http status code OK 19:59:31 DEBUG --- status code --- 19:59:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:31 DEBUG --- http response --- 19:59:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:59:31 DEBUG ----- output ----- 19:59:31 DEBUG { 19:59:31 DEBUG "status": "success", 19:59:31 DEBUG "data": { 19:59:31 DEBUG "resultType": "vector", 19:59:31 DEBUG "result": [ 19:59:31 DEBUG { 19:59:31 DEBUG "metric": { 19:59:31 DEBUG "pod": "am-767f6b67b5-9zw22" 19:59:31 DEBUG }, 19:59:31 DEBUG "value": [ 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG [...] 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG "pod": "am-767f6b67b5-vbltz" 19:59:31 DEBUG }, 19:59:31 DEBUG "value": [ 19:59:31 DEBUG 1713553171, 19:59:31 DEBUG "2504.866666666667" 19:59:31 DEBUG ] 19:59:31 DEBUG } 19:59:31 DEBUG ] 19:59:31 DEBUG } 19:59:31 DEBUG } 19:59:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553171" 19:59:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553171" 19:59:31 INFO [http_cmd]: http status code OK 19:59:31 DEBUG --- status code --- 19:59:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:31 DEBUG --- http response --- 19:59:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:59:31 DEBUG ----- output ----- 19:59:31 DEBUG { 19:59:31 DEBUG "status": "success", 19:59:31 DEBUG "data": { 19:59:31 DEBUG "resultType": "vector", 19:59:31 DEBUG "result": [ 19:59:31 DEBUG { 19:59:31 DEBUG "metric": { 19:59:31 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:59:31 DEBUG }, 19:59:31 DEBUG "value": [ 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG [...] 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:59:31 DEBUG }, 19:59:31 DEBUG "value": [ 19:59:31 DEBUG 1713553171, 19:59:31 DEBUG "3655282688" 19:59:31 DEBUG ] 19:59:31 DEBUG } 19:59:31 DEBUG ] 19:59:31 DEBUG } 19:59:31 DEBUG } 19:59:31 INFO [http_cmd]: http status code OK 19:59:31 DEBUG --- status code --- 19:59:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:31 DEBUG --- http response --- 19:59:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:59:31 DEBUG ----- output ----- 19:59:31 DEBUG { 19:59:31 DEBUG "status": "success", 19:59:31 DEBUG "data": { 19:59:31 DEBUG "resultType": "vector", 19:59:31 DEBUG "result": [ 19:59:31 DEBUG { 19:59:31 DEBUG "metric": { 19:59:31 DEBUG "pod": "ds-idrepo-1" 19:59:31 DEBUG }, 19:59:31 DEBUG "value": [ 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG [...] 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 19:59:31 DEBUG }, 19:59:31 DEBUG "value": [ 19:59:31 DEBUG 1713553171, 19:59:31 DEBUG "3.270881972364265" 19:59:31 DEBUG ] 19:59:31 DEBUG } 19:59:31 DEBUG ] 19:59:31 DEBUG } 19:59:31 DEBUG } 19:59:31 INFO [http_cmd]: http status code OK 19:59:31 DEBUG --- status code --- 19:59:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:31 DEBUG --- http response --- 19:59:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:59:31 DEBUG ----- output ----- 19:59:31 DEBUG { 19:59:31 DEBUG "status": "success", 19:59:31 DEBUG "data": { 19:59:31 DEBUG "resultType": "vector", 19:59:31 DEBUG "result": [ 19:59:31 DEBUG { 19:59:31 DEBUG "metric": { 19:59:31 DEBUG "instance": "10.142.0.107:9100" 19:59:31 DEBUG }, 19:59:31 DEBUG "value": [ 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG [...] 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG "instance": "10.142.0.43:9100" 19:59:31 DEBUG }, 19:59:31 DEBUG "value": [ 19:59:31 DEBUG 1713553171, 19:59:31 DEBUG "75586792.19999999" 19:59:31 DEBUG ] 19:59:31 DEBUG } 19:59:31 DEBUG ] 19:59:31 DEBUG } 19:59:31 DEBUG } 19:59:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553171" 19:59:31 INFO [http_cmd]: http status code OK 19:59:31 DEBUG --- status code --- 19:59:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:31 DEBUG --- http response --- 19:59:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:59:31 DEBUG ----- output ----- 19:59:31 DEBUG { 19:59:31 DEBUG "status": "success", 19:59:31 DEBUG "data": { 19:59:31 DEBUG "resultType": "vector", 19:59:31 DEBUG "result": [ 19:59:31 DEBUG { 19:59:31 DEBUG "metric": { 19:59:31 DEBUG "pod": "ds-cts-0" 19:59:31 DEBUG }, 19:59:31 DEBUG "value": [ 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG [...] 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG 19:59:31 DEBUG "pod": "ds-cts-2" 19:59:31 DEBUG }, 19:59:31 DEBUG "value": [ 19:59:31 DEBUG 1713553171, 19:59:31 DEBUG "0" 19:59:31 DEBUG ] 19:59:31 DEBUG } 19:59:31 DEBUG ] 19:59:31 DEBUG } 19:59:31 DEBUG } 19:59:32 INFO 19:59:32 INFO [loop_until]: kubectl --namespace=xlou top pods 19:59:32 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:59:32 INFO [loop_until]: OK (rc = 0) 19:59:32 DEBUG --- stdout --- 19:59:32 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 16105m 9006Mi am-767f6b67b5-qcqz2 16457m 8809Mi am-767f6b67b5-vbltz 17724m 8997Mi ds-cts-0 17200m 10966Mi ds-cts-1 15929m 11218Mi ds-cts-2 17562m 10972Mi ds-idrepo-0 7221m 3695Mi ds-idrepo-1 7110m 3537Mi ds-idrepo-2 7092m 3493Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8054m 3392Mi 19:59:32 DEBUG --- stderr --- 19:59:32 DEBUG 19:59:33 INFO 19:59:33 INFO [loop_until]: kubectl --namespace=xlou top node 19:59:33 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:59:33 INFO [loop_until]: OK (rc = 0) 19:59:33 DEBUG --- stdout --- 19:59:33 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 68m 0% 6251Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7435m 24% 7290Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16159m 54% 11715Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 82m 0% 3356Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 17469m 58% 13941Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 158m 0% 5209Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 16460m 55% 11690Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15583m 52% 2989Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 18207m 60% 11216Mi 10% gke-xlou-cdm-ds-0f601751-j488 7712m 25% 6751Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 16050m 53% 15968Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 7446m 24% 7419Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 16830m 56% 13418Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 8197m 51% 7136Mi 12% 19:59:33 DEBUG --- stderr --- 19:59:33 DEBUG 20:00:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553231" 20:00:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553231" 20:00:31 INFO [http_cmd]: http status code OK 20:00:31 DEBUG --- status code --- 20:00:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:31 DEBUG --- http response --- 20:00:31 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:00:31 INFO [http_cmd]: http status code OK 20:00:31 DEBUG --- status code --- 20:00:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:31 DEBUG --- http response --- 20:00:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:00:31 DEBUG ----- output ----- 20:00:31 DEBUG { 20:00:31 DEBUG "status": "success", 20:00:31 DEBUG "data": { 20:00:31 DEBUG "resultType": "vector", 20:00:31 DEBUG "result": [ 20:00:31 DEBUG { 20:00:31 DEBUG "metric": { 20:00:31 DEBUG "pod": "ds-idrepo-0" 20:00:31 DEBUG }, 20:00:31 DEBUG "value": [ 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG [...] 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG "pod": "ds-idrepo-2" 20:00:31 DEBUG }, 20:00:31 DEBUG "value": [ 20:00:31 DEBUG 1713553231, 20:00:31 DEBUG "0" 20:00:31 DEBUG ] 20:00:31 DEBUG } 20:00:31 DEBUG ] 20:00:31 DEBUG } 20:00:31 DEBUG } 20:00:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553231" 20:00:31 INFO [http_cmd]: http status code OK 20:00:31 DEBUG --- status code --- 20:00:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:31 DEBUG --- http response --- 20:00:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:00:31 DEBUG ----- output ----- 20:00:31 DEBUG { 20:00:31 DEBUG "status": "success", 20:00:31 DEBUG "data": { 20:00:31 DEBUG "resultType": "vector", 20:00:31 DEBUG "result": [ 20:00:31 DEBUG { 20:00:31 DEBUG "metric": { 20:00:31 DEBUG "pod": "ds-cts-0" 20:00:31 DEBUG }, 20:00:31 DEBUG "value": [ 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG [...] 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG "pod": "ds-cts-2" 20:00:31 DEBUG }, 20:00:31 DEBUG "value": [ 20:00:31 DEBUG 1713553231, 20:00:31 DEBUG "0.0003802866666666666" 20:00:31 DEBUG ] 20:00:31 DEBUG } 20:00:31 DEBUG ] 20:00:31 DEBUG } 20:00:31 DEBUG } 20:00:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553231" 20:00:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553231" 20:00:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713553231" 20:00:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713553231" 20:00:31 INFO [http_cmd]: http status code OK 20:00:31 DEBUG --- status code --- 20:00:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:31 DEBUG --- http response --- 20:00:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553231" 20:00:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:00:31 DEBUG ----- output ----- 20:00:31 DEBUG { 20:00:31 DEBUG "status": "success", 20:00:31 DEBUG "data": { 20:00:31 DEBUG "resultType": "vector", 20:00:31 DEBUG "result": [ 20:00:31 DEBUG { 20:00:31 DEBUG "metric": { 20:00:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:00:31 DEBUG }, 20:00:31 DEBUG "value": [ 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG [...] 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:00:31 DEBUG }, 20:00:31 DEBUG "value": [ 20:00:31 DEBUG 1713553231, 20:00:31 DEBUG "450402.5" 20:00:31 DEBUG ] 20:00:31 DEBUG } 20:00:31 DEBUG ] 20:00:31 DEBUG } 20:00:31 DEBUG } 20:00:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553231" 20:00:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553231" 20:00:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553231" 20:00:31 INFO [http_cmd]: http status code OK 20:00:31 DEBUG --- status code --- 20:00:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:31 DEBUG --- http response --- 20:00:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:00:31 DEBUG ----- output ----- 20:00:31 DEBUG { 20:00:31 DEBUG "status": "success", 20:00:31 DEBUG "data": { 20:00:31 DEBUG "resultType": "vector", 20:00:31 DEBUG "result": [ 20:00:31 DEBUG { 20:00:31 DEBUG "metric": { 20:00:31 DEBUG "pod": "ds-idrepo-0" 20:00:31 DEBUG }, 20:00:31 DEBUG "value": [ 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG [...] 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG "pod": "ds-idrepo-2" 20:00:31 DEBUG }, 20:00:31 DEBUG "value": [ 20:00:31 DEBUG 1713553231, 20:00:31 DEBUG "0" 20:00:31 DEBUG ] 20:00:31 DEBUG } 20:00:31 DEBUG ] 20:00:31 DEBUG } 20:00:31 DEBUG } 20:00:31 ERROR [http_cmd]: ERROR 20:00:31 ERROR --- status code --- 20:00:31 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:00:31 ERROR --- http response --- 20:00:31 INFO [http_cmd]: http status code OK 20:00:31 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:00:31 DEBUG --- status code --- 20:00:31 INFO [http_cmd]: http status code OK 20:00:31 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:00:31 INFO [http_cmd]: http status code OK 20:00:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:31 DEBUG --- status code --- 20:00:31 DEBUG --- status code --- 20:00:31 DEBUG --- http response --- 20:00:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:00:31 DEBUG --- http response --- 20:00:31 DEBUG --- http response --- 20:00:31 DEBUG ----- output ----- 20:00:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:00:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 20:00:31 DEBUG { 20:00:31 DEBUG ----- output ----- 20:00:31 DEBUG ----- output ----- 20:00:31 DEBUG "status": "success", 20:00:31 INFO [http_cmd]: http status code OK 20:00:31 DEBUG { 20:00:31 DEBUG { 20:00:31 DEBUG "data": { 20:00:31 DEBUG --- status code --- 20:00:31 DEBUG "status": "success", 20:00:31 DEBUG "status": "success", 20:00:31 DEBUG "resultType": "vector", 20:00:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:31 DEBUG "data": { 20:00:31 DEBUG "data": { 20:00:31 DEBUG "result": [ 20:00:31 DEBUG --- http response --- 20:00:31 DEBUG "resultType": "vector", 20:00:31 DEBUG "resultType": "vector", 20:00:31 DEBUG { 20:00:31 INFO [http_cmd]: http status code OK 20:00:31 DEBUG "result": [ 20:00:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:00:31 DEBUG "result": [ 20:00:31 DEBUG "metric": { 20:00:31 DEBUG --- status code --- 20:00:31 DEBUG { 20:00:31 DEBUG ----- output ----- 20:00:31 DEBUG { 20:00:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:00:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:31 DEBUG "metric": { 20:00:31 DEBUG { 20:00:31 DEBUG "metric": { 20:00:31 DEBUG }, 20:00:31 DEBUG --- http response --- 20:00:31 DEBUG "pod": "ds-idrepo-0" 20:00:31 DEBUG "status": "success", 20:00:31 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:00:31 DEBUG "value": [ 20:00:31 DEBUG }, 20:00:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:00:31 DEBUG "data": { 20:00:31 DEBUG }, 20:00:31 DEBUG 20:00:31 DEBUG "value": [ 20:00:31 DEBUG ----- output ----- 20:00:31 DEBUG "resultType": "vector", 20:00:31 DEBUG "value": [ 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG { 20:00:31 DEBUG "result": [ 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG "status": "success", 20:00:31 DEBUG { 20:00:31 DEBUG 20:00:31 DEBUG [...] 20:00:31 DEBUG 20:00:31 DEBUG "data": { 20:00:31 DEBUG "metric": { 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG [...] 20:00:31 DEBUG "resultType": "vector", 20:00:31 DEBUG "instance": "10.142.0.107:9100" 20:00:31 DEBUG [...] 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG "result": [ 20:00:31 DEBUG }, 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG { 20:00:31 DEBUG "value": [ 20:00:31 DEBUG 20:00:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:00:31 DEBUG 20:00:31 DEBUG "metric": { 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG }, 20:00:31 DEBUG "pod": "ds-cts-2" 20:00:31 DEBUG "instance": "10.142.0.107:9100" 20:00:31 DEBUG 20:00:31 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 20:00:31 DEBUG "value": [ 20:00:31 DEBUG }, 20:00:31 DEBUG }, 20:00:31 DEBUG 20:00:31 DEBUG }, 20:00:31 DEBUG 1713553231, 20:00:31 DEBUG "value": [ 20:00:31 DEBUG "value": [ 20:00:31 DEBUG [...] 20:00:31 DEBUG "value": [ 20:00:31 DEBUG "0" 20:00:31 DEBUG 1713553231, 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG 1713553231, 20:00:31 DEBUG ] 20:00:31 DEBUG "0" 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG "13616.969340644844" 20:00:31 DEBUG } 20:00:31 DEBUG ] 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG ] 20:00:31 DEBUG ] 20:00:31 DEBUG } 20:00:31 DEBUG [...] 20:00:31 DEBUG "instance": "10.142.0.43:9100" 20:00:31 DEBUG } 20:00:31 DEBUG } 20:00:31 DEBUG ] 20:00:31 DEBUG 20:00:31 DEBUG }, 20:00:31 DEBUG ] 20:00:31 DEBUG } 20:00:31 DEBUG } 20:00:31 DEBUG 20:00:31 DEBUG "value": [ 20:00:31 DEBUG } 20:00:31 DEBUG } 20:00:31 DEBUG 20:00:31 DEBUG 1713553231, 20:00:31 DEBUG } 20:00:31 DEBUG "instance": "10.142.0.43:9100" 20:00:31 DEBUG "0.2053333333333209" 20:00:31 DEBUG }, 20:00:31 DEBUG ] 20:00:31 DEBUG "value": [ 20:00:31 DEBUG } 20:00:31 DEBUG 1713553231, 20:00:31 DEBUG ] 20:00:31 DEBUG "0.09033333333333454" 20:00:31 DEBUG } 20:00:31 DEBUG ] 20:00:31 DEBUG } 20:00:31 DEBUG } 20:00:31 DEBUG ] 20:00:31 DEBUG } 20:00:31 DEBUG } 20:00:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553231" 20:00:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553231" 20:00:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713553231" 20:00:31 INFO [http_cmd]: http status code OK 20:00:31 DEBUG --- status code --- 20:00:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:31 DEBUG --- http response --- 20:00:31 INFO [http_cmd]: http status code OK 20:00:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:00:31 DEBUG --- status code --- 20:00:31 DEBUG ----- output ----- 20:00:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:31 DEBUG { 20:00:31 DEBUG --- http response --- 20:00:31 DEBUG "status": "success", 20:00:31 DEBUG "data": { 20:00:31 DEBUG "resultType": "vector", 20:00:31 DEBUG "result": [ 20:00:31 DEBUG { 20:00:31 DEBUG "metric": { 20:00:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:00:31 DEBUG }, 20:00:31 DEBUG "value": [ 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG [...] 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:00:31 DEBUG }, 20:00:31 DEBUG "value": [ 20:00:31 DEBUG 1713553231, 20:00:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:00:31 INFO [http_cmd]: http status code OK 20:00:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553231" 20:00:31 DEBUG "2490.4" 20:00:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553231" 20:00:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713553231" 20:00:31 DEBUG ----- output ----- 20:00:31 DEBUG --- status code --- 20:00:31 DEBUG ] 20:00:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553231" 20:00:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553231" 20:00:31 DEBUG { 20:00:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553231" 20:00:31 DEBUG } 20:00:31 DEBUG "status": "success", 20:00:31 DEBUG --- http response --- 20:00:31 DEBUG ] 20:00:31 DEBUG "data": { 20:00:31 DEBUG } 20:00:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:00:31 DEBUG ----- output ----- 20:00:31 DEBUG } 20:00:31 DEBUG "resultType": "vector", 20:00:31 DEBUG { 20:00:31 DEBUG "result": [ 20:00:31 DEBUG "status": "success", 20:00:31 DEBUG { 20:00:31 DEBUG "data": { 20:00:31 DEBUG "metric": { 20:00:31 DEBUG "resultType": "vector", 20:00:31 DEBUG "pod": "ds-cts-0" 20:00:31 DEBUG "result": [ 20:00:31 DEBUG }, 20:00:31 INFO [http_cmd]: http status code OK 20:00:31 DEBUG { 20:00:31 DEBUG "value": [ 20:00:31 DEBUG --- status code --- 20:00:31 DEBUG "metric": { 20:00:31 DEBUG 20:00:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:00:31 DEBUG 20:00:31 DEBUG --- http response --- 20:00:31 DEBUG }, 20:00:31 DEBUG 20:00:31 DEBUG "value": [ 20:00:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:00:31 DEBUG [...] 20:00:31 INFO [http_cmd]: http status code OK 20:00:31 DEBUG 20:00:31 DEBUG ----- output ----- 20:00:31 DEBUG 20:00:31 DEBUG --- status code --- 20:00:31 DEBUG 20:00:31 DEBUG { 20:00:31 DEBUG 20:00:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:31 DEBUG 20:00:31 DEBUG "status": "success", 20:00:31 DEBUG 20:00:31 INFO [http_cmd]: http status code OK 20:00:31 DEBUG --- http response --- 20:00:31 DEBUG [...] 20:00:31 DEBUG "data": { 20:00:31 DEBUG "pod": "ds-cts-2" 20:00:31 DEBUG --- status code --- 20:00:31 DEBUG 20:00:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:00:31 DEBUG "resultType": "vector", 20:00:31 DEBUG }, 20:00:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:31 DEBUG 20:00:31 DEBUG ----- output ----- 20:00:31 DEBUG "result": [ 20:00:31 DEBUG "value": [ 20:00:31 DEBUG --- http response --- 20:00:31 DEBUG 20:00:31 DEBUG { 20:00:31 DEBUG { 20:00:31 DEBUG 1713553231, 20:00:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:00:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:00:31 DEBUG "status": "success", 20:00:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553231" 20:00:31 DEBUG "metric": { 20:00:31 DEBUG "29.966666666666665" 20:00:31 DEBUG }, 20:00:31 DEBUG ----- output ----- 20:00:31 DEBUG "data": { 20:00:31 DEBUG "pod": "ds-idrepo-0" 20:00:31 DEBUG ] 20:00:31 DEBUG "value": [ 20:00:31 DEBUG { 20:00:31 DEBUG "resultType": "vector", 20:00:31 DEBUG }, 20:00:31 DEBUG } 20:00:31 DEBUG 1713553231, 20:00:31 DEBUG "status": "success", 20:00:31 DEBUG "result": [ 20:00:31 DEBUG "value": [ 20:00:31 DEBUG ] 20:00:31 DEBUG "0.014" 20:00:31 DEBUG "data": { 20:00:31 DEBUG { 20:00:31 DEBUG 20:00:31 DEBUG } 20:00:31 DEBUG ] 20:00:31 DEBUG "resultType": "vector", 20:00:31 DEBUG "metric": { 20:00:31 DEBUG 20:00:31 DEBUG } 20:00:31 DEBUG } 20:00:31 DEBUG "result": [ 20:00:31 DEBUG "instance": "10.142.0.107:9100" 20:00:31 DEBUG 20:00:31 DEBUG ] 20:00:31 DEBUG { 20:00:31 DEBUG }, 20:00:31 DEBUG [...] 20:00:31 DEBUG } 20:00:31 DEBUG "metric": { 20:00:31 DEBUG "value": [ 20:00:31 DEBUG 20:00:31 DEBUG } 20:00:31 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG }, 20:00:31 DEBUG 20:00:31 INFO [http_cmd]: http status code OK 20:00:31 DEBUG 20:00:31 DEBUG "value": [ 20:00:31 INFO [http_cmd]: http status code OK 20:00:31 DEBUG 20:00:31 INFO [http_cmd]: http status code OK 20:00:31 DEBUG --- status code --- 20:00:31 DEBUG "pod": "ds-idrepo-2" 20:00:31 DEBUG 20:00:31 DEBUG --- status code --- 20:00:31 DEBUG [...] 20:00:31 DEBUG --- status code --- 20:00:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:31 DEBUG }, 20:00:31 DEBUG 20:00:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:31 DEBUG 20:00:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:31 DEBUG --- http response --- 20:00:31 DEBUG "value": [ 20:00:31 DEBUG 20:00:31 DEBUG --- http response --- 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG 1713553231, 20:00:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:00:31 DEBUG [...] 20:00:31 DEBUG --- http response --- 20:00:31 DEBUG "instance": "10.142.0.43:9100" 20:00:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:00:31 DEBUG "0" 20:00:31 DEBUG ----- output ----- 20:00:31 DEBUG 20:00:31 DEBUG }, 20:00:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:00:31 DEBUG ----- output ----- 20:00:31 DEBUG ] 20:00:31 DEBUG { 20:00:31 DEBUG 20:00:31 DEBUG "value": [ 20:00:31 DEBUG ----- output ----- 20:00:31 DEBUG { 20:00:31 DEBUG } 20:00:31 DEBUG "status": "success", 20:00:31 DEBUG 20:00:31 DEBUG 1713553231, 20:00:31 DEBUG { 20:00:31 DEBUG "status": "success", 20:00:31 DEBUG ] 20:00:31 DEBUG "data": { 20:00:31 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:00:31 DEBUG "1999131945984" 20:00:31 DEBUG "status": "success", 20:00:31 DEBUG "data": { 20:00:31 DEBUG } 20:00:31 DEBUG "resultType": "vector", 20:00:31 DEBUG }, 20:00:31 DEBUG ] 20:00:31 DEBUG "data": { 20:00:31 DEBUG "resultType": "vector", 20:00:31 DEBUG } 20:00:31 DEBUG "result": [ 20:00:31 DEBUG "value": [ 20:00:31 DEBUG } 20:00:31 DEBUG "resultType": "vector", 20:00:31 DEBUG "result": [ 20:00:31 DEBUG { 20:00:31 DEBUG 1713553231, 20:00:31 DEBUG ] 20:00:31 DEBUG "result": [ 20:00:31 DEBUG { 20:00:31 DEBUG "metric": { 20:00:31 DEBUG "8.04482962410805" 20:00:31 DEBUG } 20:00:31 DEBUG { 20:00:31 DEBUG "metric": { 20:00:31 DEBUG "instance": "10.142.0.107:9100" 20:00:31 DEBUG ] 20:00:31 DEBUG } 20:00:31 DEBUG "metric": { 20:00:31 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:00:31 DEBUG }, 20:00:31 DEBUG } 20:00:31 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:00:31 DEBUG }, 20:00:31 DEBUG "value": [ 20:00:31 INFO [http_cmd]: http status code OK 20:00:31 DEBUG ] 20:00:31 DEBUG }, 20:00:31 DEBUG "value": [ 20:00:31 DEBUG 20:00:31 DEBUG --- status code --- 20:00:31 DEBUG } 20:00:31 DEBUG "value": [ 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:31 DEBUG } 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG --- http response --- 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG [...] 20:00:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:00:31 DEBUG 20:00:31 DEBUG [...] 20:00:31 DEBUG 20:00:31 DEBUG ----- output ----- 20:00:31 DEBUG [...] 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG { 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG "status": "success", 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG "instance": "10.142.0.43:9100" 20:00:31 DEBUG "data": { 20:00:31 DEBUG 20:00:31 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 20:00:31 DEBUG }, 20:00:31 DEBUG "resultType": "vector", 20:00:31 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 20:00:31 DEBUG }, 20:00:31 DEBUG "value": [ 20:00:31 DEBUG "result": [ 20:00:31 DEBUG }, 20:00:31 DEBUG "value": [ 20:00:31 DEBUG 1713553231, 20:00:31 DEBUG { 20:00:31 DEBUG "value": [ 20:00:31 DEBUG 1713553231, 20:00:31 DEBUG "75430566.13333333" 20:00:31 DEBUG "metric": { 20:00:31 DEBUG 1713553231, 20:00:31 DEBUG "0" 20:00:31 DEBUG ] 20:00:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:00:31 DEBUG "0" 20:00:31 DEBUG ] 20:00:31 DEBUG } 20:00:31 DEBUG }, 20:00:31 DEBUG ] 20:00:31 DEBUG } 20:00:31 DEBUG ] 20:00:31 DEBUG "value": [ 20:00:31 DEBUG } 20:00:31 DEBUG ] 20:00:31 DEBUG } 20:00:31 DEBUG 20:00:31 DEBUG ] 20:00:31 DEBUG } 20:00:31 DEBUG } 20:00:31 DEBUG 20:00:31 DEBUG } 20:00:31 DEBUG } 20:00:31 DEBUG 20:00:31 DEBUG } 20:00:31 DEBUG [...] 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:00:31 DEBUG }, 20:00:31 DEBUG "value": [ 20:00:31 DEBUG 1713553231, 20:00:31 DEBUG "2501.1" 20:00:31 DEBUG ] 20:00:31 DEBUG } 20:00:31 DEBUG ] 20:00:31 DEBUG } 20:00:31 DEBUG } 20:00:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553231" 20:00:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553231" 20:00:31 INFO [http_cmd]: http status code OK 20:00:31 DEBUG --- status code --- 20:00:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:31 DEBUG --- http response --- 20:00:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:00:31 DEBUG ----- output ----- 20:00:31 DEBUG { 20:00:31 DEBUG "status": "success", 20:00:31 DEBUG "data": { 20:00:31 DEBUG "resultType": "vector", 20:00:31 DEBUG "result": [ 20:00:31 DEBUG { 20:00:31 DEBUG "metric": { 20:00:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:00:31 DEBUG }, 20:00:31 DEBUG "value": [ 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG [...] 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:00:31 DEBUG }, 20:00:31 DEBUG "value": [ 20:00:31 DEBUG 1713553231, 20:00:31 DEBUG "2490.5333333333333" 20:00:31 DEBUG ] 20:00:31 DEBUG } 20:00:31 DEBUG ] 20:00:31 DEBUG } 20:00:31 DEBUG } 20:00:31 INFO [http_cmd]: http status code OK 20:00:31 DEBUG --- status code --- 20:00:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:31 DEBUG --- http response --- 20:00:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:00:31 DEBUG ----- output ----- 20:00:31 DEBUG { 20:00:31 DEBUG "status": "success", 20:00:31 DEBUG "data": { 20:00:31 DEBUG "resultType": "vector", 20:00:31 DEBUG "result": [ 20:00:31 DEBUG { 20:00:31 DEBUG "metric": { 20:00:31 DEBUG "instance": "10.142.0.107:9100" 20:00:31 DEBUG }, 20:00:31 DEBUG "value": [ 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG [...] 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG "instance": "10.142.0.43:9100" 20:00:31 DEBUG }, 20:00:31 DEBUG "value": [ 20:00:31 DEBUG 1713553231, 20:00:31 DEBUG "75427789.36666667" 20:00:31 DEBUG ] 20:00:31 DEBUG } 20:00:31 DEBUG ] 20:00:31 DEBUG } 20:00:31 DEBUG } 20:00:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553231" 20:00:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713553231" 20:00:31 INFO [http_cmd]: http status code OK 20:00:31 DEBUG --- status code --- 20:00:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:31 DEBUG --- http response --- 20:00:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:00:31 DEBUG ----- output ----- 20:00:31 DEBUG { 20:00:31 DEBUG "status": "success", 20:00:31 DEBUG "data": { 20:00:31 DEBUG "resultType": "vector", 20:00:31 DEBUG "result": [ 20:00:31 DEBUG { 20:00:31 DEBUG "metric": { 20:00:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:00:31 DEBUG }, 20:00:31 DEBUG "value": [ 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG [...] 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:00:31 DEBUG }, 20:00:31 DEBUG "value": [ 20:00:31 DEBUG 1713553231, 20:00:31 DEBUG "2492.1666666666665" 20:00:31 DEBUG ] 20:00:31 DEBUG } 20:00:31 DEBUG ] 20:00:31 DEBUG } 20:00:31 DEBUG } 20:00:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553231" 20:00:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553231" 20:00:31 INFO [http_cmd]: http status code OK 20:00:31 DEBUG --- status code --- 20:00:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:31 DEBUG --- http response --- 20:00:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:00:31 DEBUG ----- output ----- 20:00:31 DEBUG { 20:00:31 DEBUG "status": "success", 20:00:31 DEBUG "data": { 20:00:31 DEBUG "resultType": "vector", 20:00:31 DEBUG "result": [ 20:00:31 DEBUG { 20:00:31 DEBUG "metric": { 20:00:31 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:00:31 DEBUG }, 20:00:31 DEBUG "value": [ 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG [...] 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:00:31 DEBUG }, 20:00:31 DEBUG "value": [ 20:00:31 DEBUG 1713553231, 20:00:31 DEBUG "3657531392" 20:00:31 DEBUG ] 20:00:31 DEBUG } 20:00:31 DEBUG ] 20:00:31 DEBUG } 20:00:31 DEBUG } 20:00:31 INFO [http_cmd]: http status code OK 20:00:31 DEBUG --- status code --- 20:00:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:31 DEBUG --- http response --- 20:00:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:00:31 DEBUG ----- output ----- 20:00:31 DEBUG { 20:00:31 DEBUG "status": "success", 20:00:31 DEBUG "data": { 20:00:31 DEBUG "resultType": "vector", 20:00:31 DEBUG "result": [ 20:00:31 DEBUG { 20:00:31 DEBUG "metric": { 20:00:31 DEBUG "pod": "ds-cts-0" 20:00:31 DEBUG }, 20:00:31 DEBUG "value": [ 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG [...] 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG "pod": "ds-cts-2" 20:00:31 DEBUG }, 20:00:31 DEBUG "value": [ 20:00:31 DEBUG 1713553231, 20:00:31 DEBUG "0" 20:00:31 DEBUG ] 20:00:31 DEBUG } 20:00:31 DEBUG ] 20:00:31 DEBUG } 20:00:31 DEBUG } 20:00:31 INFO [http_cmd]: http status code OK 20:00:31 DEBUG --- status code --- 20:00:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:31 DEBUG --- http response --- 20:00:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 20:00:31 DEBUG ----- output ----- 20:00:31 DEBUG { 20:00:31 DEBUG "status": "success", 20:00:31 DEBUG "data": { 20:00:31 DEBUG "resultType": "vector", 20:00:31 DEBUG "result": [ 20:00:31 DEBUG { 20:00:31 DEBUG "metric": { 20:00:31 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:00:31 DEBUG }, 20:00:31 DEBUG "value": [ 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG [...] 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG 20:00:31 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 20:00:31 DEBUG }, 20:00:31 DEBUG "value": [ 20:00:31 DEBUG 1713553231, 20:00:31 DEBUG "3.702600551420792" 20:00:31 DEBUG ] 20:00:31 DEBUG } 20:00:31 DEBUG ] 20:00:31 DEBUG } 20:00:31 DEBUG } 20:00:32 INFO 20:00:32 INFO [loop_until]: kubectl --namespace=xlou top pods 20:00:32 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:00:32 INFO [loop_until]: OK (rc = 0) 20:00:32 DEBUG --- stdout --- 20:00:32 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 15861m 9008Mi am-767f6b67b5-qcqz2 16120m 8810Mi am-767f6b67b5-vbltz 18030m 8998Mi ds-cts-0 17113m 10969Mi ds-cts-1 16419m 11212Mi ds-cts-2 17553m 10973Mi ds-idrepo-0 6893m 3696Mi ds-idrepo-1 6998m 3536Mi ds-idrepo-2 7176m 3493Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 1m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8042m 3395Mi 20:00:32 DEBUG --- stderr --- 20:00:32 DEBUG 20:00:33 INFO 20:00:33 INFO [loop_until]: kubectl --namespace=xlou top node 20:00:33 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:00:33 INFO [loop_until]: OK (rc = 0) 20:00:33 DEBUG --- stdout --- 20:00:33 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 71m 0% 6255Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7364m 24% 7291Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16046m 53% 11716Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 86m 0% 3354Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 17049m 57% 13952Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 277m 0% 5155Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 15796m 52% 11700Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15229m 51% 2986Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 18239m 61% 11217Mi 10% gke-xlou-cdm-ds-0f601751-j488 7567m 25% 6746Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 15986m 53% 15968Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 7470m 25% 7434Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 17366m 58% 13408Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 8205m 51% 7138Mi 12% 20:00:33 DEBUG --- stderr --- 20:00:33 DEBUG 20:01:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553291" 20:01:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553291" 20:01:31 INFO [http_cmd]: http status code OK 20:01:31 DEBUG --- status code --- 20:01:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:31 DEBUG --- http response --- 20:01:31 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:01:31 INFO [http_cmd]: http status code OK 20:01:31 DEBUG --- status code --- 20:01:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:31 DEBUG --- http response --- 20:01:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:01:31 DEBUG ----- output ----- 20:01:31 DEBUG { 20:01:31 DEBUG "status": "success", 20:01:31 DEBUG "data": { 20:01:31 DEBUG "resultType": "vector", 20:01:31 DEBUG "result": [ 20:01:31 DEBUG { 20:01:31 DEBUG "metric": { 20:01:31 DEBUG "pod": "ds-idrepo-0" 20:01:31 DEBUG }, 20:01:31 DEBUG "value": [ 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG [...] 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG "pod": "ds-idrepo-2" 20:01:31 DEBUG }, 20:01:31 DEBUG "value": [ 20:01:31 DEBUG 1713553291, 20:01:31 DEBUG "0" 20:01:31 DEBUG ] 20:01:31 DEBUG } 20:01:31 DEBUG ] 20:01:31 DEBUG } 20:01:31 DEBUG } 20:01:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713553291" 20:01:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553291" 20:01:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553291" 20:01:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553291" 20:01:31 ERROR [http_cmd]: ERROR 20:01:31 INFO [http_cmd]: http status code OK 20:01:31 ERROR --- status code --- 20:01:31 DEBUG --- status code --- 20:01:31 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:01:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:31 ERROR --- http response --- 20:01:31 DEBUG --- http response --- 20:01:31 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:01:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:01:31 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:01:31 DEBUG ----- output ----- 20:01:31 DEBUG { 20:01:31 DEBUG "status": "success", 20:01:31 DEBUG "data": { 20:01:31 DEBUG "resultType": "vector", 20:01:31 DEBUG "result": [ 20:01:31 DEBUG { 20:01:31 DEBUG "metric": { 20:01:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:01:31 DEBUG }, 20:01:31 DEBUG "value": [ 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG [...] 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:01:31 DEBUG }, 20:01:31 DEBUG "value": [ 20:01:31 DEBUG 1713553291, 20:01:31 DEBUG "0" 20:01:31 DEBUG ] 20:01:31 DEBUG } 20:01:31 DEBUG ] 20:01:31 DEBUG } 20:01:31 DEBUG } 20:01:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553291" 20:01:31 INFO [http_cmd]: http status code OK 20:01:31 DEBUG --- status code --- 20:01:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:31 DEBUG --- http response --- 20:01:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:01:31 DEBUG ----- output ----- 20:01:31 DEBUG { 20:01:31 DEBUG "status": "success", 20:01:31 DEBUG "data": { 20:01:31 DEBUG "resultType": "vector", 20:01:31 DEBUG "result": [ 20:01:31 DEBUG { 20:01:31 DEBUG "metric": { 20:01:31 DEBUG "pod": "ds-cts-0" 20:01:31 DEBUG }, 20:01:31 DEBUG "value": [ 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG [...] 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG "pod": "ds-cts-2" 20:01:31 DEBUG }, 20:01:31 DEBUG "value": [ 20:01:31 DEBUG 1713553291, 20:01:31 DEBUG "0.0004068866666666666" 20:01:31 DEBUG ] 20:01:31 DEBUG } 20:01:31 DEBUG ] 20:01:31 DEBUG } 20:01:31 DEBUG } 20:01:31 INFO [http_cmd]: http status code OK 20:01:31 INFO [http_cmd]: http status code OK 20:01:31 DEBUG --- status code --- 20:01:31 DEBUG --- status code --- 20:01:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:31 DEBUG --- http response --- 20:01:31 DEBUG --- http response --- 20:01:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:01:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:01:31 DEBUG ----- output ----- 20:01:31 DEBUG ----- output ----- 20:01:31 DEBUG { 20:01:31 DEBUG { 20:01:31 DEBUG "status": "success", 20:01:31 DEBUG "status": "success", 20:01:31 DEBUG "data": { 20:01:31 DEBUG "data": { 20:01:31 DEBUG "resultType": "vector", 20:01:31 DEBUG "resultType": "vector", 20:01:31 DEBUG "result": [ 20:01:31 DEBUG "result": [ 20:01:31 DEBUG { 20:01:31 DEBUG { 20:01:31 DEBUG "metric": { 20:01:31 DEBUG "metric": { 20:01:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:01:31 DEBUG "instance": "10.142.0.107:9100" 20:01:31 DEBUG }, 20:01:31 DEBUG }, 20:01:31 DEBUG "value": [ 20:01:31 DEBUG "value": [ 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG [...] 20:01:31 DEBUG [...] 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:01:31 DEBUG "instance": "10.142.0.43:9100" 20:01:31 DEBUG }, 20:01:31 DEBUG }, 20:01:31 DEBUG "value": [ 20:01:31 DEBUG "value": [ 20:01:31 DEBUG 1713553291, 20:01:31 DEBUG 1713553291, 20:01:31 DEBUG "450126" 20:01:31 DEBUG "0.2188000000000026" 20:01:31 DEBUG ] 20:01:31 DEBUG ] 20:01:31 DEBUG } 20:01:31 DEBUG } 20:01:31 DEBUG ] 20:01:31 DEBUG ] 20:01:31 DEBUG } 20:01:31 DEBUG } 20:01:31 DEBUG } 20:01:31 DEBUG } 20:01:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553291" 20:01:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713553291" 20:01:31 INFO [http_cmd]: http status code OK 20:01:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553291" 20:01:31 DEBUG --- status code --- 20:01:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553291" 20:01:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:31 DEBUG --- http response --- 20:01:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:01:31 DEBUG ----- output ----- 20:01:31 DEBUG { 20:01:31 DEBUG "status": "success", 20:01:31 DEBUG "data": { 20:01:31 DEBUG "resultType": "vector", 20:01:31 DEBUG "result": [ 20:01:31 DEBUG { 20:01:31 DEBUG "metric": { 20:01:31 DEBUG "pod": "ds-idrepo-0" 20:01:31 DEBUG }, 20:01:31 DEBUG "value": [ 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG [...] 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG "pod": "ds-idrepo-2" 20:01:31 DEBUG }, 20:01:31 DEBUG "value": [ 20:01:31 DEBUG 1713553291, 20:01:31 DEBUG "0" 20:01:31 DEBUG ] 20:01:31 DEBUG } 20:01:31 DEBUG ] 20:01:31 DEBUG } 20:01:31 DEBUG } 20:01:31 INFO [http_cmd]: http status code OK 20:01:31 DEBUG --- status code --- 20:01:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:31 DEBUG --- http response --- 20:01:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:01:31 DEBUG ----- output ----- 20:01:31 DEBUG { 20:01:31 DEBUG "status": "success", 20:01:31 DEBUG "data": { 20:01:31 DEBUG "resultType": "vector", 20:01:31 DEBUG "result": [ 20:01:31 DEBUG { 20:01:31 DEBUG "metric": { 20:01:31 DEBUG "pod": "ds-idrepo-0" 20:01:31 DEBUG }, 20:01:31 DEBUG "value": [ 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG [...] 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG "pod": "ds-cts-2" 20:01:31 DEBUG }, 20:01:31 DEBUG "value": [ 20:01:31 DEBUG 1713553291, 20:01:31 DEBUG "0" 20:01:31 INFO [http_cmd]: http status code OK 20:01:31 DEBUG ] 20:01:31 DEBUG --- status code --- 20:01:31 DEBUG } 20:01:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:31 DEBUG ] 20:01:31 DEBUG --- http response --- 20:01:31 DEBUG } 20:01:31 DEBUG } 20:01:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:01:31 DEBUG ----- output ----- 20:01:31 DEBUG { 20:01:31 DEBUG "status": "success", 20:01:31 DEBUG "data": { 20:01:31 DEBUG "resultType": "vector", 20:01:31 DEBUG "result": [ 20:01:31 DEBUG { 20:01:31 DEBUG "metric": { 20:01:31 DEBUG "pod": "ds-idrepo-1" 20:01:31 DEBUG }, 20:01:31 DEBUG "value": [ 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG [...] 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG "pod": "idm-b85f9d978-txpwv" 20:01:31 DEBUG }, 20:01:31 DEBUG "value": [ 20:01:31 DEBUG 1713553291, 20:01:31 DEBUG "7644.960597262548" 20:01:31 DEBUG ] 20:01:31 DEBUG } 20:01:31 DEBUG ] 20:01:31 DEBUG } 20:01:31 DEBUG } 20:01:31 INFO [http_cmd]: http status code OK 20:01:31 DEBUG --- status code --- 20:01:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:31 DEBUG --- http response --- 20:01:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553291" 20:01:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:01:31 DEBUG ----- output ----- 20:01:31 DEBUG { 20:01:31 DEBUG "status": "success", 20:01:31 DEBUG "data": { 20:01:31 DEBUG "resultType": "vector", 20:01:31 DEBUG "result": [ 20:01:31 DEBUG { 20:01:31 DEBUG "metric": { 20:01:31 DEBUG "instance": "10.142.0.107:9100" 20:01:31 DEBUG }, 20:01:31 DEBUG "value": [ 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG [...] 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG "instance": "10.142.0.43:9100" 20:01:31 DEBUG }, 20:01:31 DEBUG "value": [ 20:01:31 DEBUG 1713553291, 20:01:31 DEBUG "0.09566666666666303" 20:01:31 DEBUG ] 20:01:31 DEBUG } 20:01:31 DEBUG ] 20:01:31 DEBUG } 20:01:31 DEBUG } 20:01:31 INFO [http_cmd]: http status code OK 20:01:31 DEBUG --- status code --- 20:01:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:31 DEBUG --- http response --- 20:01:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:01:31 DEBUG ----- output ----- 20:01:31 DEBUG { 20:01:31 DEBUG "status": "success", 20:01:31 DEBUG "data": { 20:01:31 DEBUG "resultType": "vector", 20:01:31 DEBUG "result": [ 20:01:31 DEBUG { 20:01:31 DEBUG "metric": { 20:01:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:01:31 DEBUG }, 20:01:31 DEBUG "value": [ 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG [...] 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:01:31 DEBUG }, 20:01:31 DEBUG "value": [ 20:01:31 DEBUG 1713553291, 20:01:31 DEBUG "2498.1666666666665" 20:01:31 DEBUG ] 20:01:31 DEBUG } 20:01:31 DEBUG ] 20:01:31 DEBUG } 20:01:31 DEBUG } 20:01:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553291" 20:01:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553291" 20:01:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553291" 20:01:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713553291" 20:01:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553291" 20:01:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553291" 20:01:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553291" 20:01:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713553291" 20:01:31 INFO [http_cmd]: http status code OK 20:01:31 INFO [http_cmd]: http status code OK 20:01:31 DEBUG --- status code --- 20:01:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553291" 20:01:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553291" 20:01:31 DEBUG --- status code --- 20:01:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:31 DEBUG --- http response --- 20:01:31 INFO [http_cmd]: http status code OK 20:01:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:31 INFO [http_cmd]: http status code OK 20:01:31 INFO [http_cmd]: http status code OK 20:01:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:01:31 DEBUG --- status code --- 20:01:31 DEBUG --- http response --- 20:01:31 DEBUG --- status code --- 20:01:31 DEBUG --- status code --- 20:01:31 DEBUG ----- output ----- 20:01:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:01:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:31 DEBUG { 20:01:31 DEBUG --- http response --- 20:01:31 DEBUG ----- output ----- 20:01:31 DEBUG { 20:01:31 DEBUG --- http response --- 20:01:31 DEBUG "status": "success", 20:01:31 INFO [http_cmd]: http status code OK 20:01:31 DEBUG "status": "success", 20:01:31 DEBUG --- http response --- 20:01:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:01:31 DEBUG "data": { 20:01:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:01:31 INFO [http_cmd]: http status code OK 20:01:31 INFO [http_cmd]: http status code OK 20:01:31 DEBUG --- status code --- 20:01:31 DEBUG "data": { 20:01:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:01:31 DEBUG ----- output ----- 20:01:31 DEBUG "resultType": "vector", 20:01:31 DEBUG ----- output ----- 20:01:31 DEBUG --- status code --- 20:01:31 DEBUG --- status code --- 20:01:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:31 DEBUG "resultType": "vector", 20:01:31 DEBUG ----- output ----- 20:01:31 DEBUG { 20:01:31 INFO [http_cmd]: http status code OK 20:01:31 DEBUG "result": [ 20:01:31 DEBUG { 20:01:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:31 DEBUG --- http response --- 20:01:31 DEBUG "result": [ 20:01:31 DEBUG { 20:01:31 DEBUG "status": "success", 20:01:31 DEBUG --- status code --- 20:01:31 DEBUG { 20:01:31 DEBUG "status": "success", 20:01:31 DEBUG --- http response --- 20:01:31 DEBUG --- http response --- 20:01:31 DEBUG { 20:01:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:01:31 DEBUG "status": "success", 20:01:31 DEBUG "data": { 20:01:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:31 DEBUG "metric": { 20:01:31 DEBUG "data": { 20:01:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:01:31 DEBUG "metric": { 20:01:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:01:31 DEBUG ----- output ----- 20:01:31 DEBUG "data": { 20:01:31 DEBUG "resultType": "vector", 20:01:31 DEBUG --- http response --- 20:01:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:01:31 DEBUG "resultType": "vector", 20:01:31 DEBUG ----- output ----- 20:01:31 DEBUG "instance": "10.142.0.107:9100" 20:01:31 DEBUG ----- output ----- 20:01:31 DEBUG { 20:01:31 DEBUG "resultType": "vector", 20:01:31 DEBUG "result": [ 20:01:31 DEBUG }, 20:01:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:01:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553291" 20:01:31 DEBUG "result": [ 20:01:31 DEBUG { 20:01:31 DEBUG }, 20:01:31 DEBUG { 20:01:31 DEBUG "status": "success", 20:01:31 DEBUG "result": [ 20:01:31 DEBUG { 20:01:31 DEBUG "value": [ 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG "status": "success", 20:01:31 DEBUG "value": [ 20:01:31 DEBUG "status": "success", 20:01:31 DEBUG "data": { 20:01:31 DEBUG { 20:01:31 DEBUG "metric": { 20:01:31 DEBUG ----- output ----- 20:01:31 DEBUG 20:01:31 DEBUG { 20:01:31 DEBUG "data": { 20:01:31 DEBUG 20:01:31 DEBUG "data": { 20:01:31 DEBUG "resultType": "vector", 20:01:31 DEBUG "metric": { 20:01:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:01:31 DEBUG { 20:01:31 DEBUG [...] 20:01:31 DEBUG "metric": { 20:01:31 DEBUG "resultType": "vector", 20:01:31 DEBUG 20:01:31 DEBUG "resultType": "vector", 20:01:31 DEBUG "result": [ 20:01:31 INFO [http_cmd]: http status code OK 20:01:31 DEBUG "pod": "ds-cts-0" 20:01:31 DEBUG }, 20:01:31 DEBUG "status": "success", 20:01:31 DEBUG 20:01:31 DEBUG "pod": "ds-idrepo-0" 20:01:31 DEBUG "result": [ 20:01:31 DEBUG 20:01:31 DEBUG "result": [ 20:01:31 DEBUG { 20:01:31 DEBUG --- status code --- 20:01:31 DEBUG }, 20:01:31 DEBUG "value": [ 20:01:31 DEBUG "data": { 20:01:31 DEBUG 20:01:31 DEBUG }, 20:01:31 DEBUG { 20:01:31 DEBUG [...] 20:01:31 DEBUG { 20:01:31 DEBUG "metric": { 20:01:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:31 DEBUG "value": [ 20:01:31 DEBUG 20:01:31 DEBUG "resultType": "vector", 20:01:31 DEBUG 20:01:31 DEBUG "value": [ 20:01:31 DEBUG "metric": { 20:01:31 DEBUG 20:01:31 DEBUG "metric": { 20:01:31 DEBUG "instance": "10.142.0.107:9100" 20:01:31 DEBUG --- http response --- 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG "result": [ 20:01:31 DEBUG { 20:01:31 DEBUG 20:01:31 DEBUG "instance": "10.142.0.107:9100" 20:01:31 DEBUG 20:01:31 DEBUG "pod": "ds-idrepo-1" 20:01:31 DEBUG }, 20:01:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:01:31 DEBUG "metric": { 20:01:31 DEBUG 20:01:31 DEBUG }, 20:01:31 DEBUG 20:01:31 DEBUG }, 20:01:31 DEBUG "value": [ 20:01:31 DEBUG ----- output ----- 20:01:31 DEBUG 20:01:31 DEBUG [...] 20:01:31 DEBUG }, 20:01:31 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:01:31 DEBUG 20:01:31 DEBUG "value": [ 20:01:31 DEBUG "instance": "10.142.0.43:9100" 20:01:31 DEBUG "value": [ 20:01:31 DEBUG 20:01:31 DEBUG { 20:01:31 DEBUG [...] 20:01:31 DEBUG 20:01:31 DEBUG "value": [ 20:01:31 DEBUG }, 20:01:31 DEBUG [...] 20:01:31 DEBUG 20:01:31 DEBUG }, 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG "status": "success", 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG 1713553291, 20:01:31 DEBUG "value": [ 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG "value": [ 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG "data": { 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG "2498.1666666666665" 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG 1713553291, 20:01:31 DEBUG 20:01:31 DEBUG [...] 20:01:31 DEBUG "resultType": "vector", 20:01:31 DEBUG 20:01:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:01:31 DEBUG ] 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG [...] 20:01:31 DEBUG "75264067.86666667" 20:01:31 DEBUG [...] 20:01:31 DEBUG 20:01:31 DEBUG "result": [ 20:01:31 DEBUG "pod": "ds-cts-2" 20:01:31 DEBUG }, 20:01:31 DEBUG } 20:01:31 DEBUG 20:01:31 DEBUG "pod": "ds-idrepo-2" 20:01:31 DEBUG 20:01:31 DEBUG ] 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG { 20:01:31 DEBUG }, 20:01:31 DEBUG "value": [ 20:01:31 DEBUG ] 20:01:31 DEBUG [...] 20:01:31 DEBUG }, 20:01:31 DEBUG 20:01:31 DEBUG } 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG "metric": { 20:01:31 DEBUG "value": [ 20:01:31 DEBUG 1713553291, 20:01:31 DEBUG } 20:01:31 DEBUG 20:01:31 DEBUG "value": [ 20:01:31 DEBUG 20:01:31 DEBUG ] 20:01:31 DEBUG 20:01:31 DEBUG "pod": "idm-b85f9d978-txpwv" 20:01:31 DEBUG "instance": "10.142.0.43:9100" 20:01:31 DEBUG 1713553291, 20:01:31 DEBUG "0.014" 20:01:31 DEBUG } 20:01:31 DEBUG 20:01:31 DEBUG 1713553291, 20:01:31 DEBUG "instance": "10.142.0.43:9100" 20:01:31 INFO [http_cmd]: http status code OK 20:01:31 DEBUG } 20:01:31 DEBUG "pod": "ds-idrepo-1" 20:01:31 DEBUG }, 20:01:31 DEBUG }, 20:01:31 DEBUG "30" 20:01:31 DEBUG ] 20:01:31 DEBUG 20:01:31 DEBUG "0" 20:01:31 DEBUG }, 20:01:31 DEBUG --- status code --- 20:01:31 DEBUG } 20:01:31 DEBUG }, 20:01:31 DEBUG "value": [ 20:01:31 DEBUG "value": [ 20:01:31 DEBUG ] 20:01:31 DEBUG } 20:01:31 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:01:31 DEBUG ] 20:01:31 DEBUG "value": [ 20:01:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:31 DEBUG "value": [ 20:01:31 DEBUG 1713553291, 20:01:31 DEBUG 1713553291, 20:01:31 DEBUG } 20:01:31 DEBUG ] 20:01:31 DEBUG }, 20:01:31 DEBUG } 20:01:31 DEBUG 1713553291, 20:01:31 DEBUG --- http response --- 20:01:31 DEBUG 20:01:31 DEBUG "566.2933775750035" 20:01:31 DEBUG "75265840.96666667" 20:01:31 DEBUG ] 20:01:31 DEBUG } 20:01:31 DEBUG "value": [ 20:01:31 DEBUG ] 20:01:31 DEBUG "2003352386560" 20:01:31 DEBUG 20:01:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:01:31 DEBUG ] 20:01:31 DEBUG ] 20:01:31 DEBUG } 20:01:31 DEBUG } 20:01:31 DEBUG 1713553291, 20:01:31 DEBUG } 20:01:31 DEBUG ] 20:01:31 DEBUG 20:01:31 DEBUG ----- output ----- 20:01:31 DEBUG } 20:01:31 DEBUG } 20:01:31 DEBUG } 20:01:31 DEBUG "8.042264169107114" 20:01:31 DEBUG } 20:01:31 DEBUG } 20:01:31 DEBUG [...] 20:01:31 DEBUG { 20:01:31 DEBUG ] 20:01:31 DEBUG ] 20:01:31 DEBUG } 20:01:31 DEBUG ] 20:01:31 DEBUG 20:01:31 DEBUG "status": "success", 20:01:31 DEBUG } 20:01:31 DEBUG ] 20:01:31 DEBUG } 20:01:31 DEBUG } 20:01:31 DEBUG 20:01:31 DEBUG "data": { 20:01:31 DEBUG } 20:01:31 DEBUG } 20:01:31 DEBUG } 20:01:31 DEBUG 20:01:31 DEBUG "resultType": "vector", 20:01:31 DEBUG ] 20:01:31 DEBUG "pod": "idm-b85f9d978-txpwv" 20:01:31 DEBUG "result": [ 20:01:31 DEBUG } 20:01:31 DEBUG }, 20:01:31 DEBUG { 20:01:31 DEBUG } 20:01:31 DEBUG "value": [ 20:01:31 DEBUG "metric": { 20:01:31 DEBUG 1713553291, 20:01:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:01:31 DEBUG "0" 20:01:31 DEBUG }, 20:01:31 DEBUG ] 20:01:31 DEBUG "value": [ 20:01:31 DEBUG } 20:01:31 DEBUG 20:01:31 DEBUG ] 20:01:31 DEBUG 20:01:31 DEBUG } 20:01:31 DEBUG 20:01:31 DEBUG } 20:01:31 DEBUG [...] 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:01:31 DEBUG }, 20:01:31 DEBUG "value": [ 20:01:31 DEBUG 1713553291, 20:01:31 DEBUG "2499.5666666666666" 20:01:31 DEBUG ] 20:01:31 DEBUG } 20:01:31 DEBUG ] 20:01:31 DEBUG } 20:01:31 DEBUG } 20:01:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713553291" 20:01:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553291" 20:01:31 INFO [http_cmd]: http status code OK 20:01:31 DEBUG --- status code --- 20:01:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:31 DEBUG --- http response --- 20:01:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:01:31 DEBUG ----- output ----- 20:01:31 DEBUG { 20:01:31 DEBUG "status": "success", 20:01:31 DEBUG "data": { 20:01:31 DEBUG "resultType": "vector", 20:01:31 DEBUG "result": [ 20:01:31 DEBUG { 20:01:31 DEBUG "metric": { 20:01:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:01:31 DEBUG }, 20:01:31 DEBUG "value": [ 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG [...] 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:01:31 DEBUG }, 20:01:31 DEBUG "value": [ 20:01:31 DEBUG 1713553291, 20:01:31 DEBUG "2498.133333333333" 20:01:31 DEBUG ] 20:01:31 DEBUG } 20:01:31 DEBUG ] 20:01:31 DEBUG } 20:01:31 DEBUG } 20:01:31 INFO [http_cmd]: http status code OK 20:01:31 DEBUG --- status code --- 20:01:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:31 DEBUG --- http response --- 20:01:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:01:31 DEBUG ----- output ----- 20:01:31 DEBUG { 20:01:31 DEBUG "status": "success", 20:01:31 DEBUG "data": { 20:01:31 DEBUG "resultType": "vector", 20:01:31 DEBUG "result": [ 20:01:31 DEBUG { 20:01:31 DEBUG "metric": { 20:01:31 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:01:31 DEBUG }, 20:01:31 DEBUG "value": [ 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG [...] 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:01:31 DEBUG }, 20:01:31 DEBUG "value": [ 20:01:31 DEBUG 1713553291, 20:01:31 DEBUG "3659931648" 20:01:31 DEBUG ] 20:01:31 DEBUG } 20:01:31 DEBUG ] 20:01:31 DEBUG } 20:01:31 DEBUG } 20:01:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553291" 20:01:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553291" 20:01:31 INFO [http_cmd]: http status code OK 20:01:31 DEBUG --- status code --- 20:01:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:31 DEBUG --- http response --- 20:01:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:01:31 DEBUG ----- output ----- 20:01:31 DEBUG { 20:01:31 DEBUG "status": "success", 20:01:31 DEBUG "data": { 20:01:31 DEBUG "resultType": "vector", 20:01:31 DEBUG "result": [ 20:01:31 DEBUG { 20:01:31 DEBUG "metric": { 20:01:31 DEBUG "pod": "ds-cts-0" 20:01:31 DEBUG }, 20:01:31 DEBUG "value": [ 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG [...] 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG "pod": "ds-cts-2" 20:01:31 DEBUG }, 20:01:31 DEBUG "value": [ 20:01:31 DEBUG 1713553291, 20:01:31 DEBUG "0" 20:01:31 DEBUG ] 20:01:31 DEBUG } 20:01:31 DEBUG ] 20:01:31 DEBUG } 20:01:31 DEBUG } 20:01:31 INFO [http_cmd]: http status code OK 20:01:31 DEBUG --- status code --- 20:01:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:31 DEBUG --- http response --- 20:01:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:01:31 DEBUG ----- output ----- 20:01:31 DEBUG { 20:01:31 DEBUG "status": "success", 20:01:31 DEBUG "data": { 20:01:31 DEBUG "resultType": "vector", 20:01:31 DEBUG "result": [ 20:01:31 DEBUG { 20:01:31 DEBUG "metric": { 20:01:31 DEBUG "pod": "ds-idrepo-1" 20:01:31 DEBUG }, 20:01:31 DEBUG "value": [ 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG [...] 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG 20:01:31 DEBUG "pod": "idm-b85f9d978-txpwv" 20:01:31 DEBUG }, 20:01:31 DEBUG "value": [ 20:01:31 DEBUG 1713553291, 20:01:31 DEBUG "0.003458350615235467" 20:01:31 DEBUG ] 20:01:31 DEBUG } 20:01:31 DEBUG ] 20:01:31 DEBUG } 20:01:31 DEBUG } 20:01:32 INFO 20:01:32 INFO [loop_until]: kubectl --namespace=xlou top pods 20:01:32 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:01:33 INFO [loop_until]: OK (rc = 0) 20:01:33 DEBUG --- stdout --- 20:01:33 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 15978m 9006Mi am-767f6b67b5-qcqz2 16347m 8808Mi am-767f6b67b5-vbltz 17226m 8996Mi ds-cts-0 17106m 10968Mi ds-cts-1 16280m 11211Mi ds-cts-2 17628m 10973Mi ds-idrepo-0 7194m 3687Mi ds-idrepo-1 7250m 3536Mi ds-idrepo-2 7263m 3493Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8319m 3397Mi 20:01:33 DEBUG --- stderr --- 20:01:33 DEBUG 20:01:33 INFO 20:01:33 INFO [loop_until]: kubectl --namespace=xlou top node 20:01:33 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:01:34 INFO [loop_until]: OK (rc = 0) 20:01:34 DEBUG --- stdout --- 20:01:34 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 69m 0% 6249Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7411m 24% 7289Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16101m 53% 11708Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 80m 0% 3351Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 17217m 57% 13953Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 159m 0% 5175Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 16442m 55% 11693Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15245m 51% 2981Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 18120m 60% 11212Mi 10% gke-xlou-cdm-ds-0f601751-j488 7620m 25% 6730Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 16050m 53% 15965Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 7488m 25% 7428Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 17615m 59% 13414Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 8192m 51% 7137Mi 12% 20:01:34 DEBUG --- stderr --- 20:01:34 DEBUG 20:02:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553351" 20:02:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713553351" 20:02:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553351" 20:02:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553351" 20:02:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553351" 20:02:31 INFO [http_cmd]: http status code OK 20:02:31 ERROR [http_cmd]: ERROR 20:02:31 INFO [http_cmd]: http status code OK 20:02:31 DEBUG --- status code --- 20:02:31 ERROR --- status code --- 20:02:31 DEBUG --- status code --- 20:02:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:31 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:02:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:31 DEBUG --- http response --- 20:02:31 ERROR --- http response --- 20:02:31 DEBUG --- http response --- 20:02:31 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:02:31 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:02:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:02:31 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:02:31 DEBUG ----- output ----- 20:02:31 DEBUG { 20:02:31 DEBUG "status": "success", 20:02:31 DEBUG "data": { 20:02:31 DEBUG "resultType": "vector", 20:02:31 DEBUG "result": [ 20:02:31 DEBUG { 20:02:31 DEBUG "metric": { 20:02:31 DEBUG "pod": "ds-idrepo-0" 20:02:31 DEBUG }, 20:02:31 DEBUG "value": [ 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG [...] 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG "pod": "ds-idrepo-2" 20:02:31 DEBUG }, 20:02:31 DEBUG "value": [ 20:02:31 DEBUG 1713553351, 20:02:31 DEBUG "0" 20:02:31 DEBUG ] 20:02:31 DEBUG } 20:02:31 DEBUG ] 20:02:31 DEBUG } 20:02:31 DEBUG } 20:02:31 INFO [http_cmd]: http status code OK 20:02:31 DEBUG --- status code --- 20:02:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:31 DEBUG --- http response --- 20:02:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:02:31 DEBUG ----- output ----- 20:02:31 DEBUG { 20:02:31 DEBUG "status": "success", 20:02:31 DEBUG "data": { 20:02:31 DEBUG "resultType": "vector", 20:02:31 DEBUG "result": [ 20:02:31 DEBUG { 20:02:31 DEBUG "metric": { 20:02:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:02:31 DEBUG }, 20:02:31 DEBUG "value": [ 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG [...] 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:02:31 DEBUG }, 20:02:31 DEBUG "value": [ 20:02:31 DEBUG 1713553351, 20:02:31 DEBUG "449969.5" 20:02:31 DEBUG ] 20:02:31 DEBUG } 20:02:31 DEBUG ] 20:02:31 DEBUG } 20:02:31 DEBUG } 20:02:31 INFO [http_cmd]: http status code OK 20:02:31 DEBUG --- status code --- 20:02:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:31 DEBUG --- http response --- 20:02:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553351" 20:02:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:02:31 DEBUG ----- output ----- 20:02:31 DEBUG { 20:02:31 DEBUG "status": "success", 20:02:31 DEBUG "data": { 20:02:31 DEBUG "resultType": "vector", 20:02:31 DEBUG "result": [ 20:02:31 DEBUG { 20:02:31 DEBUG "metric": { 20:02:31 DEBUG "instance": "10.142.0.107:9100" 20:02:31 DEBUG }, 20:02:31 DEBUG "value": [ 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG [...] 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG "instance": "10.142.0.43:9100" 20:02:31 DEBUG }, 20:02:31 DEBUG "value": [ 20:02:31 DEBUG 1713553351, 20:02:31 DEBUG "0.15216666666668366" 20:02:31 DEBUG ] 20:02:31 DEBUG } 20:02:31 DEBUG ] 20:02:31 DEBUG } 20:02:31 DEBUG } 20:02:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553351" 20:02:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553351" 20:02:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553351" 20:02:31 INFO [http_cmd]: http status code OK 20:02:31 DEBUG --- status code --- 20:02:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:31 DEBUG --- http response --- 20:02:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:02:31 DEBUG ----- output ----- 20:02:31 DEBUG { 20:02:31 DEBUG "status": "success", 20:02:31 DEBUG "data": { 20:02:31 DEBUG "resultType": "vector", 20:02:31 DEBUG "result": [ 20:02:31 DEBUG { 20:02:31 DEBUG "metric": { 20:02:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:02:31 DEBUG }, 20:02:31 DEBUG "value": [ 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG [...] 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:02:31 DEBUG }, 20:02:31 DEBUG "value": [ 20:02:31 DEBUG 1713553351, 20:02:31 DEBUG "0" 20:02:31 DEBUG ] 20:02:31 DEBUG } 20:02:31 DEBUG ] 20:02:31 DEBUG } 20:02:31 DEBUG } 20:02:31 INFO [http_cmd]: http status code OK 20:02:31 DEBUG --- status code --- 20:02:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:31 DEBUG --- http response --- 20:02:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:02:31 DEBUG ----- output ----- 20:02:31 DEBUG { 20:02:31 DEBUG "status": "success", 20:02:31 DEBUG "data": { 20:02:31 DEBUG "resultType": "vector", 20:02:31 DEBUG "result": [ 20:02:31 DEBUG { 20:02:31 DEBUG "metric": { 20:02:31 DEBUG "instance": "10.142.0.107:9100" 20:02:31 DEBUG }, 20:02:31 DEBUG "value": [ 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG [...] 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG "instance": "10.142.0.43:9100" 20:02:31 DEBUG }, 20:02:31 DEBUG "value": [ 20:02:31 DEBUG 1713553351, 20:02:31 DEBUG "0.0646666666666647" 20:02:31 DEBUG ] 20:02:31 DEBUG } 20:02:31 DEBUG ] 20:02:31 DEBUG } 20:02:31 DEBUG } 20:02:31 INFO [http_cmd]: http status code OK 20:02:31 DEBUG --- status code --- 20:02:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:31 DEBUG --- http response --- 20:02:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:02:31 DEBUG ----- output ----- 20:02:31 DEBUG { 20:02:31 DEBUG "status": "success", 20:02:31 DEBUG "data": { 20:02:31 DEBUG "resultType": "vector", 20:02:31 DEBUG "result": [ 20:02:31 DEBUG { 20:02:31 DEBUG "metric": { 20:02:31 DEBUG "pod": "ds-cts-0" 20:02:31 DEBUG }, 20:02:31 DEBUG "value": [ 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG [...] 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG "pod": "ds-cts-2" 20:02:31 DEBUG }, 20:02:31 DEBUG "value": [ 20:02:31 DEBUG 1713553351, 20:02:31 DEBUG "0.0005365333333333333" 20:02:31 DEBUG ] 20:02:31 DEBUG } 20:02:31 DEBUG ] 20:02:31 DEBUG } 20:02:31 DEBUG } 20:02:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553351" 20:02:31 INFO [http_cmd]: http status code OK 20:02:31 DEBUG --- status code --- 20:02:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:31 DEBUG --- http response --- 20:02:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:02:31 DEBUG ----- output ----- 20:02:31 DEBUG { 20:02:31 DEBUG "status": "success", 20:02:31 DEBUG "data": { 20:02:31 DEBUG "resultType": "vector", 20:02:31 DEBUG "result": [ 20:02:31 DEBUG { 20:02:31 DEBUG "metric": { 20:02:31 DEBUG "pod": "ds-idrepo-0" 20:02:31 DEBUG }, 20:02:31 DEBUG "value": [ 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG [...] 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG "pod": "ds-idrepo-2" 20:02:31 DEBUG }, 20:02:31 DEBUG "value": [ 20:02:31 DEBUG 1713553351, 20:02:31 DEBUG "0" 20:02:31 DEBUG ] 20:02:31 DEBUG } 20:02:31 DEBUG ] 20:02:31 DEBUG } 20:02:31 DEBUG } 20:02:31 INFO [http_cmd]: http status code OK 20:02:31 DEBUG --- status code --- 20:02:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:31 DEBUG --- http response --- 20:02:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:02:31 DEBUG ----- output ----- 20:02:31 DEBUG { 20:02:31 DEBUG "status": "success", 20:02:31 DEBUG "data": { 20:02:31 DEBUG "resultType": "vector", 20:02:31 DEBUG "result": [ 20:02:31 DEBUG { 20:02:31 DEBUG "metric": { 20:02:31 DEBUG "pod": "ds-idrepo-1" 20:02:31 DEBUG }, 20:02:31 DEBUG "value": [ 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG [...] 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 20:02:31 DEBUG }, 20:02:31 DEBUG "value": [ 20:02:31 DEBUG 1713553351, 20:02:31 DEBUG "181505240.92851025" 20:02:31 DEBUG ] 20:02:31 DEBUG } 20:02:31 DEBUG ] 20:02:31 DEBUG } 20:02:31 DEBUG } 20:02:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713553351" 20:02:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553351" 20:02:31 INFO [http_cmd]: http status code OK 20:02:31 DEBUG --- status code --- 20:02:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:31 DEBUG --- http response --- 20:02:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:02:31 DEBUG ----- output ----- 20:02:31 DEBUG { 20:02:31 DEBUG "status": "success", 20:02:31 DEBUG "data": { 20:02:31 DEBUG "resultType": "vector", 20:02:31 DEBUG "result": [ 20:02:31 DEBUG { 20:02:31 DEBUG "metric": { 20:02:31 DEBUG "pod": "ds-idrepo-0" 20:02:31 DEBUG }, 20:02:31 DEBUG "value": [ 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG [...] 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG "pod": "ds-cts-2" 20:02:31 DEBUG }, 20:02:31 DEBUG "value": [ 20:02:31 DEBUG 1713553351, 20:02:31 DEBUG "0" 20:02:31 DEBUG ] 20:02:31 DEBUG } 20:02:31 DEBUG ] 20:02:31 DEBUG } 20:02:31 DEBUG } 20:02:31 INFO [http_cmd]: http status code OK 20:02:31 DEBUG --- status code --- 20:02:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:31 DEBUG --- http response --- 20:02:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:02:31 DEBUG ----- output ----- 20:02:31 DEBUG { 20:02:31 DEBUG "status": "success", 20:02:31 DEBUG "data": { 20:02:31 DEBUG "resultType": "vector", 20:02:31 DEBUG "result": [ 20:02:31 DEBUG { 20:02:31 DEBUG "metric": { 20:02:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:02:31 DEBUG }, 20:02:31 DEBUG "value": [ 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG [...] 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:02:31 DEBUG }, 20:02:31 DEBUG "value": [ 20:02:31 DEBUG 1713553351, 20:02:31 DEBUG "2505.1666666666665" 20:02:31 DEBUG ] 20:02:31 DEBUG } 20:02:31 DEBUG ] 20:02:31 DEBUG } 20:02:31 DEBUG } 20:02:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553351" 20:02:31 INFO [http_cmd]: http status code OK 20:02:31 DEBUG --- status code --- 20:02:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:31 DEBUG --- http response --- 20:02:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:02:31 DEBUG ----- output ----- 20:02:31 DEBUG { 20:02:31 DEBUG "status": "success", 20:02:31 DEBUG "data": { 20:02:31 DEBUG "resultType": "vector", 20:02:31 DEBUG "result": [ 20:02:31 DEBUG { 20:02:31 DEBUG "metric": { 20:02:31 DEBUG "instance": "10.142.0.107:9100" 20:02:31 DEBUG }, 20:02:31 DEBUG "value": [ 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG [...] 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG "instance": "10.142.0.43:9100" 20:02:31 DEBUG }, 20:02:31 DEBUG "value": [ 20:02:31 DEBUG 1713553351, 20:02:31 DEBUG "2006729151488" 20:02:31 DEBUG ] 20:02:31 DEBUG } 20:02:31 DEBUG ] 20:02:31 DEBUG } 20:02:31 DEBUG } 20:02:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553351" 20:02:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553351" 20:02:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553351" 20:02:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713553351" 20:02:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553351" 20:02:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553351" 20:02:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553351" 20:02:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553351" 20:02:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713553351" 20:02:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553351" 20:02:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553351" 20:02:31 INFO [http_cmd]: http status code OK 20:02:31 DEBUG --- status code --- 20:02:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:31 DEBUG --- http response --- 20:02:31 INFO [http_cmd]: http status code OK 20:02:31 DEBUG --- status code --- 20:02:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:31 DEBUG --- http response --- 20:02:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:02:31 DEBUG ----- output ----- 20:02:31 DEBUG { 20:02:31 DEBUG "status": "success", 20:02:31 DEBUG "data": { 20:02:31 DEBUG "resultType": "vector", 20:02:31 DEBUG "result": [ 20:02:31 DEBUG { 20:02:31 DEBUG "metric": { 20:02:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:02:31 DEBUG }, 20:02:31 DEBUG "value": [ 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG [...] 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:02:31 DEBUG }, 20:02:31 DEBUG "value": [ 20:02:31 DEBUG 1713553351, 20:02:31 DEBUG "2501.5666666666666" 20:02:31 DEBUG ] 20:02:31 DEBUG } 20:02:31 DEBUG ] 20:02:31 DEBUG } 20:02:31 DEBUG } 20:02:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:02:31 DEBUG ----- output ----- 20:02:31 DEBUG { 20:02:31 DEBUG "status": "success", 20:02:31 DEBUG "data": { 20:02:31 DEBUG "resultType": "vector", 20:02:31 DEBUG "result": [ 20:02:31 DEBUG { 20:02:31 DEBUG "metric": { 20:02:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:02:31 DEBUG }, 20:02:31 DEBUG "value": [ 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG [...] 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:02:31 DEBUG }, 20:02:31 DEBUG "value": [ 20:02:31 DEBUG 1713553351, 20:02:31 DEBUG "0.014" 20:02:31 DEBUG ] 20:02:31 DEBUG } 20:02:31 DEBUG ] 20:02:31 DEBUG } 20:02:31 DEBUG } 20:02:31 INFO [http_cmd]: http status code OK 20:02:31 INFO [http_cmd]: http status code OK 20:02:31 INFO [http_cmd]: http status code OK 20:02:31 DEBUG --- status code --- 20:02:31 INFO [http_cmd]: http status code OK 20:02:31 DEBUG --- status code --- 20:02:31 DEBUG --- status code --- 20:02:31 INFO [http_cmd]: http status code OK 20:02:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:31 INFO [http_cmd]: http status code OK 20:02:31 DEBUG --- status code --- 20:02:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:31 DEBUG --- status code --- 20:02:31 DEBUG --- http response --- 20:02:31 DEBUG --- status code --- 20:02:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:31 DEBUG --- http response --- 20:02:31 DEBUG --- http response --- 20:02:31 INFO [http_cmd]: http status code OK 20:02:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:02:31 DEBUG --- http response --- 20:02:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:02:31 INFO [http_cmd]: http status code OK 20:02:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:02:31 DEBUG --- status code --- 20:02:31 DEBUG --- http response --- 20:02:31 DEBUG --- http response --- 20:02:31 DEBUG ----- output ----- 20:02:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:02:31 DEBUG ----- output ----- 20:02:31 DEBUG --- status code --- 20:02:31 DEBUG ----- output ----- 20:02:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:02:31 DEBUG { 20:02:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:02:31 DEBUG ----- output ----- 20:02:31 DEBUG { 20:02:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:31 DEBUG { 20:02:31 DEBUG "status": "success", 20:02:31 INFO [http_cmd]: http status code OK 20:02:31 DEBUG ----- output ----- 20:02:31 DEBUG "status": "success", 20:02:31 DEBUG ----- output ----- 20:02:31 DEBUG { 20:02:31 DEBUG "status": "success", 20:02:31 DEBUG --- http response --- 20:02:31 DEBUG --- http response --- 20:02:31 DEBUG "data": { 20:02:31 DEBUG --- status code --- 20:02:31 DEBUG { 20:02:31 DEBUG "data": { 20:02:31 DEBUG { 20:02:31 DEBUG "status": "success", 20:02:31 DEBUG "data": { 20:02:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:02:31 DEBUG "resultType": "vector", 20:02:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:02:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:31 DEBUG "status": "success", 20:02:31 DEBUG "resultType": "vector", 20:02:31 DEBUG "status": "success", 20:02:31 DEBUG "data": { 20:02:31 DEBUG "resultType": "vector", 20:02:31 DEBUG ----- output ----- 20:02:31 DEBUG "result": [ 20:02:31 DEBUG ----- output ----- 20:02:31 DEBUG --- http response --- 20:02:31 DEBUG "data": { 20:02:31 DEBUG "result": [ 20:02:31 DEBUG "data": { 20:02:31 DEBUG "resultType": "vector", 20:02:31 DEBUG "result": [ 20:02:31 DEBUG { 20:02:31 DEBUG { 20:02:31 DEBUG { 20:02:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:02:31 DEBUG "resultType": "vector", 20:02:31 DEBUG { 20:02:31 DEBUG "resultType": "vector", 20:02:31 DEBUG "result": [ 20:02:31 DEBUG { 20:02:31 DEBUG "status": "success", 20:02:31 DEBUG "metric": { 20:02:31 DEBUG "status": "success", 20:02:31 DEBUG ----- output ----- 20:02:31 DEBUG "result": [ 20:02:31 DEBUG "metric": { 20:02:31 DEBUG "result": [ 20:02:31 DEBUG { 20:02:31 DEBUG "metric": { 20:02:31 DEBUG "data": { 20:02:31 DEBUG "pod": "ds-cts-0" 20:02:31 DEBUG "data": { 20:02:31 DEBUG { 20:02:31 DEBUG { 20:02:31 DEBUG "instance": "10.142.0.107:9100" 20:02:31 DEBUG { 20:02:31 DEBUG "metric": { 20:02:31 DEBUG "pod": "ds-idrepo-1" 20:02:31 DEBUG "resultType": "vector", 20:02:31 DEBUG }, 20:02:31 DEBUG "resultType": "vector", 20:02:31 DEBUG "status": "success", 20:02:31 DEBUG "metric": { 20:02:31 DEBUG }, 20:02:31 DEBUG "metric": { 20:02:31 DEBUG "pod": "ds-cts-0" 20:02:31 DEBUG }, 20:02:31 DEBUG "result": [ 20:02:31 DEBUG "value": [ 20:02:31 DEBUG "result": [ 20:02:31 DEBUG "data": { 20:02:31 DEBUG "pod": "ds-idrepo-0" 20:02:31 DEBUG "value": [ 20:02:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:02:31 DEBUG }, 20:02:31 DEBUG "value": [ 20:02:31 DEBUG { 20:02:31 DEBUG 20:02:31 DEBUG { 20:02:31 DEBUG "resultType": "vector", 20:02:31 DEBUG }, 20:02:31 DEBUG 20:02:31 DEBUG }, 20:02:31 DEBUG "value": [ 20:02:31 DEBUG 20:02:31 DEBUG "metric": { 20:02:31 DEBUG 20:02:31 DEBUG "metric": { 20:02:31 DEBUG "result": [ 20:02:31 DEBUG "value": [ 20:02:31 DEBUG 20:02:31 DEBUG "value": [ 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:02:31 DEBUG 20:02:31 DEBUG "instance": "10.142.0.107:9100" 20:02:31 DEBUG { 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG }, 20:02:31 DEBUG [...] 20:02:31 DEBUG }, 20:02:31 DEBUG "metric": { 20:02:31 DEBUG 20:02:31 DEBUG [...] 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG [...] 20:02:31 DEBUG "value": [ 20:02:31 DEBUG 20:02:31 DEBUG "value": [ 20:02:31 DEBUG "pod": "ds-idrepo-1" 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG [...] 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG }, 20:02:31 DEBUG [...] 20:02:31 DEBUG 20:02:31 DEBUG [...] 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG "value": [ 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG "pod": "ds-cts-2" 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG "instance": "10.142.0.43:9100" 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553351" 20:02:31 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 20:02:31 DEBUG [...] 20:02:31 DEBUG }, 20:02:31 DEBUG [...] 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG }, 20:02:31 DEBUG 20:02:31 DEBUG "pod": "ds-cts-2" 20:02:31 DEBUG }, 20:02:31 DEBUG 20:02:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713553351" 20:02:31 DEBUG "value": [ 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG "pod": "ds-idrepo-2" 20:02:31 DEBUG "value": [ 20:02:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:02:31 DEBUG }, 20:02:31 DEBUG "value": [ 20:02:31 DEBUG 20:02:31 DEBUG 1713553351, 20:02:31 DEBUG 20:02:31 DEBUG [...] 20:02:31 DEBUG }, 20:02:31 DEBUG 1713553351, 20:02:31 DEBUG }, 20:02:31 DEBUG "value": [ 20:02:31 DEBUG 1713553351, 20:02:31 DEBUG 20:02:31 DEBUG "0" 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG "value": [ 20:02:31 DEBUG "75319860.73333333" 20:02:31 DEBUG "value": [ 20:02:31 DEBUG 1713553351, 20:02:31 DEBUG "0" 20:02:31 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:02:31 DEBUG ] 20:02:31 DEBUG "instance": "10.142.0.43:9100" 20:02:31 DEBUG 20:02:31 DEBUG 1713553351, 20:02:31 DEBUG ] 20:02:31 DEBUG 1713553351, 20:02:31 DEBUG "30" 20:02:31 DEBUG ] 20:02:31 DEBUG }, 20:02:31 DEBUG } 20:02:31 DEBUG }, 20:02:31 DEBUG 20:02:31 DEBUG "0" 20:02:31 DEBUG } 20:02:31 DEBUG "2500.1666666666665" 20:02:31 DEBUG ] 20:02:31 DEBUG } 20:02:31 DEBUG "value": [ 20:02:31 DEBUG ] 20:02:31 DEBUG "value": [ 20:02:31 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 20:02:31 DEBUG ] 20:02:31 DEBUG ] 20:02:31 DEBUG ] 20:02:31 DEBUG } 20:02:31 DEBUG ] 20:02:31 DEBUG 1713553351, 20:02:31 DEBUG } 20:02:31 DEBUG 1713553351, 20:02:31 DEBUG }, 20:02:31 DEBUG } 20:02:31 DEBUG } 20:02:31 DEBUG } 20:02:31 DEBUG ] 20:02:31 DEBUG } 20:02:31 DEBUG "8.032592778162389" 20:02:31 DEBUG } 20:02:31 DEBUG "75322798.46666667" 20:02:31 DEBUG "value": [ 20:02:31 DEBUG ] 20:02:31 DEBUG } 20:02:31 DEBUG ] 20:02:31 DEBUG } 20:02:31 DEBUG } 20:02:31 DEBUG ] 20:02:31 DEBUG ] 20:02:31 DEBUG 1713553351, 20:02:31 DEBUG } 20:02:31 DEBUG } 20:02:31 DEBUG } 20:02:31 DEBUG } 20:02:31 DEBUG } 20:02:31 DEBUG "0" 20:02:31 DEBUG } 20:02:31 DEBUG } 20:02:31 DEBUG ] 20:02:31 DEBUG ] 20:02:31 DEBUG ] 20:02:31 DEBUG } 20:02:31 DEBUG } 20:02:31 DEBUG } 20:02:31 DEBUG } 20:02:31 DEBUG } 20:02:31 DEBUG ] 20:02:31 DEBUG } 20:02:31 DEBUG } 20:02:31 INFO [http_cmd]: http status code OK 20:02:31 DEBUG --- status code --- 20:02:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:31 DEBUG --- http response --- 20:02:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:02:31 DEBUG ----- output ----- 20:02:31 DEBUG { 20:02:31 DEBUG "status": "success", 20:02:31 DEBUG "data": { 20:02:31 DEBUG "resultType": "vector", 20:02:31 DEBUG "result": [ 20:02:31 DEBUG { 20:02:31 DEBUG "metric": { 20:02:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:02:31 DEBUG }, 20:02:31 DEBUG "value": [ 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG [...] 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:02:31 DEBUG }, 20:02:31 DEBUG "value": [ 20:02:31 DEBUG 1713553351, 20:02:31 DEBUG "2501.233333333333" 20:02:31 DEBUG ] 20:02:31 DEBUG } 20:02:31 DEBUG ] 20:02:31 DEBUG } 20:02:31 DEBUG } 20:02:31 INFO [http_cmd]: http status code OK 20:02:31 DEBUG --- status code --- 20:02:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:31 DEBUG --- http response --- 20:02:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:02:31 DEBUG ----- output ----- 20:02:31 DEBUG { 20:02:31 DEBUG "status": "success", 20:02:31 DEBUG "data": { 20:02:31 DEBUG "resultType": "vector", 20:02:31 DEBUG "result": [ 20:02:31 DEBUG { 20:02:31 DEBUG "metric": { 20:02:31 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:02:31 DEBUG }, 20:02:31 DEBUG "value": [ 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG [...] 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:02:31 DEBUG }, 20:02:31 DEBUG "value": [ 20:02:31 DEBUG 1713553351, 20:02:31 DEBUG "3662077952" 20:02:31 DEBUG ] 20:02:31 DEBUG } 20:02:31 DEBUG ] 20:02:31 DEBUG } 20:02:31 DEBUG } 20:02:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553351" 20:02:31 INFO [http_cmd]: http status code OK 20:02:31 DEBUG --- status code --- 20:02:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:31 DEBUG --- http response --- 20:02:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:02:31 DEBUG ----- output ----- 20:02:31 DEBUG { 20:02:31 DEBUG "status": "success", 20:02:31 DEBUG "data": { 20:02:31 DEBUG "resultType": "vector", 20:02:31 DEBUG "result": [ 20:02:31 DEBUG { 20:02:31 DEBUG "metric": { 20:02:31 DEBUG "pod": "ds-idrepo-1" 20:02:31 DEBUG }, 20:02:31 DEBUG "value": [ 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG [...] 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG 20:02:31 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 20:02:31 DEBUG }, 20:02:31 DEBUG "value": [ 20:02:31 DEBUG 1713553351, 20:02:31 DEBUG "8.544264333157736" 20:02:31 DEBUG ] 20:02:31 DEBUG } 20:02:31 DEBUG ] 20:02:31 DEBUG } 20:02:31 DEBUG } 20:02:33 INFO 20:02:33 INFO [loop_until]: kubectl --namespace=xlou top pods 20:02:33 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:02:33 INFO [loop_until]: OK (rc = 0) 20:02:33 DEBUG --- stdout --- 20:02:33 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 16275m 9009Mi am-767f6b67b5-qcqz2 16226m 8812Mi am-767f6b67b5-vbltz 17410m 8996Mi ds-cts-0 16887m 10969Mi ds-cts-1 15216m 11238Mi ds-cts-2 17606m 10972Mi ds-idrepo-0 7126m 3687Mi ds-idrepo-1 7129m 3537Mi ds-idrepo-2 7346m 3493Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 1m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7701m 3399Mi 20:02:33 DEBUG --- stderr --- 20:02:33 DEBUG 20:02:34 INFO 20:02:34 INFO [loop_until]: kubectl --namespace=xlou top node 20:02:34 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:02:34 INFO [loop_until]: OK (rc = 0) 20:02:34 DEBUG --- stdout --- 20:02:34 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 69m 0% 6240Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7399m 24% 7277Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16212m 54% 11710Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 3348Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 17305m 57% 13946Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 145m 0% 5176Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 16514m 55% 11697Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15661m 52% 2980Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 18197m 60% 11209Mi 10% gke-xlou-cdm-ds-0f601751-j488 7665m 25% 6743Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 15756m 52% 15968Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 7439m 24% 7409Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 17660m 59% 13417Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 7928m 49% 7133Mi 12% 20:02:34 DEBUG --- stderr --- 20:02:34 DEBUG 20:03:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553411" 20:03:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553411" 20:03:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553411" 20:03:31 INFO [http_cmd]: http status code OK 20:03:31 DEBUG --- status code --- 20:03:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:31 DEBUG --- http response --- 20:03:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553411" 20:03:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:03:31 DEBUG ----- output ----- 20:03:31 DEBUG { 20:03:31 DEBUG "status": "success", 20:03:31 DEBUG "data": { 20:03:31 DEBUG "resultType": "vector", 20:03:31 DEBUG "result": [ 20:03:31 DEBUG { 20:03:31 DEBUG "metric": { 20:03:31 DEBUG "pod": "ds-idrepo-0" 20:03:31 DEBUG }, 20:03:31 DEBUG "value": [ 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG [...] 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG "pod": "ds-idrepo-2" 20:03:31 DEBUG }, 20:03:31 DEBUG "value": [ 20:03:31 DEBUG 1713553411, 20:03:31 DEBUG "0" 20:03:31 DEBUG ] 20:03:31 DEBUG } 20:03:31 DEBUG ] 20:03:31 DEBUG } 20:03:31 DEBUG } 20:03:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553411" 20:03:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553411" 20:03:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713553411" 20:03:31 INFO [http_cmd]: http status code OK 20:03:31 DEBUG --- status code --- 20:03:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:31 DEBUG --- http response --- 20:03:31 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:03:31 INFO [http_cmd]: http status code OK 20:03:31 INFO [http_cmd]: http status code OK 20:03:31 INFO [http_cmd]: http status code OK 20:03:31 DEBUG --- status code --- 20:03:31 DEBUG --- status code --- 20:03:31 DEBUG --- status code --- 20:03:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:31 DEBUG --- http response --- 20:03:31 DEBUG --- http response --- 20:03:31 DEBUG --- http response --- 20:03:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:03:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:03:31 DEBUG ----- output ----- 20:03:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:03:31 DEBUG ----- output ----- 20:03:31 DEBUG { 20:03:31 DEBUG ----- output ----- 20:03:31 DEBUG { 20:03:31 DEBUG "status": "success", 20:03:31 DEBUG { 20:03:31 DEBUG "status": "success", 20:03:31 DEBUG "data": { 20:03:31 DEBUG "status": "success", 20:03:31 DEBUG "data": { 20:03:31 DEBUG "resultType": "vector", 20:03:31 DEBUG "data": { 20:03:31 DEBUG "resultType": "vector", 20:03:31 DEBUG "result": [ 20:03:31 DEBUG "resultType": "vector", 20:03:31 DEBUG "result": [ 20:03:31 DEBUG { 20:03:31 DEBUG "result": [ 20:03:31 DEBUG { 20:03:31 DEBUG "metric": { 20:03:31 DEBUG { 20:03:31 DEBUG "metric": { 20:03:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:03:31 DEBUG "metric": { 20:03:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:03:31 DEBUG }, 20:03:31 DEBUG "instance": "10.142.0.107:9100" 20:03:31 DEBUG }, 20:03:31 DEBUG "value": [ 20:03:31 DEBUG }, 20:03:31 DEBUG "value": [ 20:03:31 DEBUG 20:03:31 DEBUG "value": [ 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG [...] 20:03:31 DEBUG 20:03:31 DEBUG [...] 20:03:31 DEBUG 20:03:31 DEBUG [...] 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:03:31 DEBUG 20:03:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:03:31 DEBUG }, 20:03:31 DEBUG "instance": "10.142.0.43:9100" 20:03:31 DEBUG }, 20:03:31 DEBUG "value": [ 20:03:31 DEBUG }, 20:03:31 DEBUG "value": [ 20:03:31 DEBUG 1713553411, 20:03:31 DEBUG "value": [ 20:03:31 DEBUG 1713553411, 20:03:31 DEBUG "0" 20:03:31 DEBUG 1713553411, 20:03:31 DEBUG "449965" 20:03:31 DEBUG ] 20:03:31 DEBUG "0.264933333333397" 20:03:31 DEBUG ] 20:03:31 DEBUG } 20:03:31 DEBUG ] 20:03:31 DEBUG } 20:03:31 DEBUG ] 20:03:31 DEBUG } 20:03:31 DEBUG ] 20:03:31 DEBUG } 20:03:31 DEBUG ] 20:03:31 DEBUG } 20:03:31 DEBUG } 20:03:31 DEBUG } 20:03:31 DEBUG } 20:03:31 DEBUG } 20:03:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553411" 20:03:31 ERROR [http_cmd]: ERROR 20:03:31 INFO [http_cmd]: http status code OK 20:03:31 ERROR --- status code --- 20:03:31 DEBUG --- status code --- 20:03:31 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:03:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:31 ERROR --- http response --- 20:03:31 DEBUG --- http response --- 20:03:31 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:03:31 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:03:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:03:31 DEBUG ----- output ----- 20:03:31 DEBUG { 20:03:31 DEBUG "status": "success", 20:03:31 DEBUG "data": { 20:03:31 DEBUG "resultType": "vector", 20:03:31 DEBUG "result": [ 20:03:31 DEBUG { 20:03:31 DEBUG "metric": { 20:03:31 DEBUG "pod": "ds-idrepo-1" 20:03:31 DEBUG }, 20:03:31 DEBUG "value": [ 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG [...] 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:03:31 DEBUG }, 20:03:31 DEBUG "value": [ 20:03:31 DEBUG 1713553411, 20:03:31 DEBUG "0" 20:03:31 DEBUG ] 20:03:31 DEBUG } 20:03:31 DEBUG ] 20:03:31 DEBUG } 20:03:31 DEBUG } 20:03:31 INFO [http_cmd]: http status code OK 20:03:31 DEBUG --- status code --- 20:03:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:31 DEBUG --- http response --- 20:03:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:03:31 DEBUG ----- output ----- 20:03:31 DEBUG { 20:03:31 DEBUG "status": "success", 20:03:31 DEBUG "data": { 20:03:31 DEBUG "resultType": "vector", 20:03:31 DEBUG "result": [ 20:03:31 DEBUG { 20:03:31 DEBUG "metric": { 20:03:31 DEBUG "pod": "ds-idrepo-0" 20:03:31 DEBUG }, 20:03:31 DEBUG "value": [ 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG [...] 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG "pod": "ds-idrepo-2" 20:03:31 DEBUG }, 20:03:31 DEBUG "value": [ 20:03:31 DEBUG 1713553411, 20:03:31 DEBUG "0" 20:03:31 DEBUG ] 20:03:31 DEBUG } 20:03:31 DEBUG ] 20:03:31 DEBUG } 20:03:31 DEBUG } 20:03:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553411" 20:03:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553411" 20:03:31 INFO [http_cmd]: http status code OK 20:03:31 DEBUG --- status code --- 20:03:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:31 DEBUG --- http response --- 20:03:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:03:31 DEBUG ----- output ----- 20:03:31 DEBUG { 20:03:31 DEBUG "status": "success", 20:03:31 DEBUG "data": { 20:03:31 DEBUG "resultType": "vector", 20:03:31 DEBUG "result": [ 20:03:31 DEBUG { 20:03:31 DEBUG "metric": { 20:03:31 DEBUG "instance": "10.142.0.107:9100" 20:03:31 DEBUG }, 20:03:31 DEBUG "value": [ 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG [...] 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG "instance": "10.142.0.43:9100" 20:03:31 DEBUG }, 20:03:31 DEBUG "value": [ 20:03:31 DEBUG 1713553411, 20:03:31 DEBUG "0.10633333333333893" 20:03:31 DEBUG ] 20:03:31 DEBUG } 20:03:31 DEBUG ] 20:03:31 DEBUG } 20:03:31 INFO [http_cmd]: http status code OK 20:03:31 DEBUG } 20:03:31 DEBUG --- status code --- 20:03:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:31 DEBUG --- http response --- 20:03:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:03:31 DEBUG ----- output ----- 20:03:31 DEBUG { 20:03:31 DEBUG "status": "success", 20:03:31 DEBUG "data": { 20:03:31 DEBUG "resultType": "vector", 20:03:31 DEBUG "result": [ 20:03:31 DEBUG { 20:03:31 DEBUG "metric": { 20:03:31 DEBUG "pod": "ds-cts-0" 20:03:31 DEBUG }, 20:03:31 DEBUG "value": [ 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG [...] 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG "pod": "ds-cts-2" 20:03:31 DEBUG }, 20:03:31 DEBUG "value": [ 20:03:31 DEBUG 1713553411, 20:03:31 DEBUG "0.00006666666666666667" 20:03:31 DEBUG ] 20:03:31 DEBUG } 20:03:31 DEBUG ] 20:03:31 DEBUG } 20:03:31 DEBUG } 20:03:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713553411" 20:03:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553411" 20:03:31 INFO [http_cmd]: http status code OK 20:03:31 DEBUG --- status code --- 20:03:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:31 DEBUG --- http response --- 20:03:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:03:31 DEBUG ----- output ----- 20:03:31 DEBUG { 20:03:31 DEBUG "status": "success", 20:03:31 DEBUG "data": { 20:03:31 DEBUG "resultType": "vector", 20:03:31 DEBUG "result": [ 20:03:31 DEBUG { 20:03:31 DEBUG "metric": { 20:03:31 DEBUG "pod": "ds-idrepo-0" 20:03:31 DEBUG }, 20:03:31 DEBUG "value": [ 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG [...] 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG "pod": "ds-cts-2" 20:03:31 DEBUG }, 20:03:31 DEBUG "value": [ 20:03:31 DEBUG 1713553411, 20:03:31 DEBUG "0" 20:03:31 DEBUG ] 20:03:31 DEBUG } 20:03:31 DEBUG ] 20:03:31 DEBUG } 20:03:31 DEBUG } 20:03:31 INFO [http_cmd]: http status code OK 20:03:31 DEBUG --- status code --- 20:03:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:31 DEBUG --- http response --- 20:03:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:03:31 DEBUG ----- output ----- 20:03:31 DEBUG { 20:03:31 DEBUG "status": "success", 20:03:31 DEBUG "data": { 20:03:31 DEBUG "resultType": "vector", 20:03:31 DEBUG "result": [ 20:03:31 DEBUG { 20:03:31 DEBUG "metric": { 20:03:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:03:31 DEBUG }, 20:03:31 DEBUG "value": [ 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG [...] 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:03:31 DEBUG }, 20:03:31 DEBUG "value": [ 20:03:31 DEBUG 1713553411, 20:03:31 DEBUG "2491.5333333333333" 20:03:31 DEBUG ] 20:03:31 DEBUG } 20:03:31 DEBUG ] 20:03:31 DEBUG } 20:03:31 DEBUG } 20:03:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553411" 20:03:31 INFO [http_cmd]: http status code OK 20:03:31 DEBUG --- status code --- 20:03:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:31 DEBUG --- http response --- 20:03:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:03:31 DEBUG ----- output ----- 20:03:31 DEBUG { 20:03:31 DEBUG "status": "success", 20:03:31 DEBUG "data": { 20:03:31 DEBUG "resultType": "vector", 20:03:31 DEBUG "result": [ 20:03:31 DEBUG { 20:03:31 DEBUG "metric": { 20:03:31 DEBUG "instance": "10.142.0.107:9100" 20:03:31 DEBUG }, 20:03:31 DEBUG "value": [ 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG [...] 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG "instance": "10.142.0.43:9100" 20:03:31 DEBUG }, 20:03:31 DEBUG "value": [ 20:03:31 DEBUG 1713553411, 20:03:31 DEBUG "2010421007360" 20:03:31 DEBUG ] 20:03:31 DEBUG } 20:03:31 DEBUG ] 20:03:31 DEBUG } 20:03:31 DEBUG } 20:03:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553411" 20:03:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713553411" 20:03:31 INFO [http_cmd]: http status code OK 20:03:31 DEBUG --- status code --- 20:03:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:31 DEBUG --- http response --- 20:03:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:03:31 DEBUG ----- output ----- 20:03:31 DEBUG { 20:03:31 DEBUG "status": "success", 20:03:31 DEBUG "data": { 20:03:31 DEBUG "resultType": "vector", 20:03:31 DEBUG "result": [ 20:03:31 DEBUG { 20:03:31 DEBUG "metric": { 20:03:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:03:31 DEBUG }, 20:03:31 DEBUG "value": [ 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG [...] 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:03:31 DEBUG }, 20:03:31 DEBUG "value": [ 20:03:31 DEBUG 1713553411, 20:03:31 DEBUG "2490.4666666666667" 20:03:31 DEBUG ] 20:03:31 DEBUG } 20:03:31 DEBUG ] 20:03:31 DEBUG } 20:03:31 DEBUG } 20:03:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553411" 20:03:31 INFO [http_cmd]: http status code OK 20:03:31 DEBUG --- status code --- 20:03:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:31 DEBUG --- http response --- 20:03:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:03:31 DEBUG ----- output ----- 20:03:31 DEBUG { 20:03:31 DEBUG "status": "success", 20:03:31 DEBUG "data": { 20:03:31 DEBUG "resultType": "vector", 20:03:31 DEBUG "result": [ 20:03:31 DEBUG { 20:03:31 DEBUG "metric": { 20:03:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:03:31 DEBUG }, 20:03:31 DEBUG "value": [ 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG [...] 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:03:31 DEBUG }, 20:03:31 DEBUG "value": [ 20:03:31 DEBUG 1713553411, 20:03:31 DEBUG "0.014499999999999999" 20:03:31 DEBUG ] 20:03:31 DEBUG } 20:03:31 DEBUG ] 20:03:31 DEBUG } 20:03:31 DEBUG } 20:03:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553411" 20:03:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553411" 20:03:31 INFO [http_cmd]: http status code OK 20:03:31 DEBUG --- status code --- 20:03:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:31 DEBUG --- http response --- 20:03:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:03:31 DEBUG ----- output ----- 20:03:31 DEBUG { 20:03:31 DEBUG "status": "success", 20:03:31 DEBUG "data": { 20:03:31 DEBUG "resultType": "vector", 20:03:31 DEBUG "result": [ 20:03:31 DEBUG { 20:03:31 DEBUG "metric": { 20:03:31 DEBUG "pod": "ds-idrepo-0" 20:03:31 DEBUG }, 20:03:31 DEBUG "value": [ 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG [...] 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG "pod": "ds-idrepo-2" 20:03:31 DEBUG }, 20:03:31 DEBUG "value": [ 20:03:31 DEBUG 1713553411, 20:03:31 DEBUG "0" 20:03:31 DEBUG ] 20:03:31 DEBUG } 20:03:31 DEBUG ] 20:03:31 DEBUG } 20:03:31 DEBUG } 20:03:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553411" 20:03:31 INFO [http_cmd]: http status code OK 20:03:31 DEBUG --- status code --- 20:03:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:31 DEBUG --- http response --- 20:03:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:03:31 DEBUG ----- output ----- 20:03:31 DEBUG { 20:03:31 DEBUG "status": "success", 20:03:31 DEBUG "data": { 20:03:31 DEBUG "resultType": "vector", 20:03:31 DEBUG "result": [ 20:03:31 DEBUG { 20:03:31 DEBUG "metric": { 20:03:31 DEBUG "pod": "ds-cts-0" 20:03:31 DEBUG }, 20:03:31 DEBUG "value": [ 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG [...] 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG "pod": "ds-cts-2" 20:03:31 DEBUG }, 20:03:31 DEBUG "value": [ 20:03:31 DEBUG 1713553411, 20:03:31 DEBUG "30" 20:03:31 DEBUG ] 20:03:31 DEBUG } 20:03:31 DEBUG ] 20:03:31 DEBUG } 20:03:31 DEBUG } 20:03:31 INFO [http_cmd]: http status code OK 20:03:31 DEBUG --- status code --- 20:03:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:31 DEBUG --- http response --- 20:03:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:03:31 DEBUG ----- output ----- 20:03:31 DEBUG { 20:03:31 DEBUG "status": "success", 20:03:31 DEBUG "data": { 20:03:31 DEBUG "resultType": "vector", 20:03:31 DEBUG "result": [ 20:03:31 DEBUG { 20:03:31 DEBUG "metric": { 20:03:31 DEBUG "pod": "ds-idrepo-1" 20:03:31 DEBUG }, 20:03:31 DEBUG "value": [ 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG [...] 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:03:31 DEBUG }, 20:03:31 DEBUG "value": [ 20:03:31 DEBUG 1713553411, 20:03:31 DEBUG "0" 20:03:31 DEBUG ] 20:03:31 DEBUG } 20:03:31 DEBUG ] 20:03:31 DEBUG } 20:03:31 DEBUG } 20:03:31 INFO [http_cmd]: http status code OK 20:03:31 DEBUG --- status code --- 20:03:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:31 DEBUG --- http response --- 20:03:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:03:31 DEBUG ----- output ----- 20:03:31 DEBUG { 20:03:31 DEBUG "status": "success", 20:03:31 DEBUG "data": { 20:03:31 DEBUG "resultType": "vector", 20:03:31 DEBUG "result": [ 20:03:31 DEBUG { 20:03:31 DEBUG "metric": { 20:03:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:03:31 DEBUG }, 20:03:31 DEBUG "value": [ 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG [...] 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:03:31 DEBUG }, 20:03:31 DEBUG "value": [ 20:03:31 DEBUG 1713553411, 20:03:31 DEBUG "2491.133333333333" 20:03:31 DEBUG ] 20:03:31 DEBUG } 20:03:31 DEBUG ] 20:03:31 DEBUG } 20:03:31 DEBUG } 20:03:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553411" 20:03:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553411" 20:03:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553411" 20:03:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713553411" 20:03:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553411" 20:03:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553411" 20:03:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713553411" 20:03:31 INFO [http_cmd]: http status code OK 20:03:31 DEBUG --- status code --- 20:03:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:31 DEBUG --- http response --- 20:03:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:03:31 DEBUG ----- output ----- 20:03:31 DEBUG { 20:03:31 DEBUG "status": "success", 20:03:31 DEBUG "data": { 20:03:31 DEBUG "resultType": "vector", 20:03:31 DEBUG "result": [ 20:03:31 DEBUG { 20:03:31 DEBUG "metric": { 20:03:31 DEBUG "pod": "ds-cts-0" 20:03:31 DEBUG }, 20:03:31 DEBUG "value": [ 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG [...] 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG "pod": "ds-cts-2" 20:03:31 DEBUG }, 20:03:31 DEBUG "value": [ 20:03:31 DEBUG 1713553411, 20:03:31 DEBUG "0" 20:03:31 DEBUG ] 20:03:31 DEBUG } 20:03:31 DEBUG ] 20:03:31 DEBUG } 20:03:31 DEBUG } 20:03:31 INFO [http_cmd]: http status code OK 20:03:31 DEBUG --- status code --- 20:03:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:31 DEBUG --- http response --- 20:03:31 INFO [http_cmd]: http status code OK 20:03:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:03:31 DEBUG --- status code --- 20:03:31 INFO [http_cmd]: http status code OK 20:03:31 DEBUG ----- output ----- 20:03:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:31 DEBUG --- status code --- 20:03:31 DEBUG { 20:03:31 DEBUG --- http response --- 20:03:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:31 DEBUG "status": "success", 20:03:31 DEBUG --- http response --- 20:03:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:03:31 DEBUG "data": { 20:03:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:03:31 DEBUG ----- output ----- 20:03:31 DEBUG "resultType": "vector", 20:03:31 INFO [http_cmd]: http status code OK 20:03:31 INFO [http_cmd]: http status code OK 20:03:31 DEBUG ----- output ----- 20:03:31 DEBUG { 20:03:31 DEBUG "result": [ 20:03:31 DEBUG --- status code --- 20:03:31 DEBUG --- status code --- 20:03:31 DEBUG { 20:03:31 DEBUG "status": "success", 20:03:31 DEBUG { 20:03:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:31 DEBUG "status": "success", 20:03:31 DEBUG "data": { 20:03:31 DEBUG "metric": { 20:03:31 DEBUG --- http response --- 20:03:31 DEBUG --- http response --- 20:03:31 DEBUG "data": { 20:03:31 DEBUG "resultType": "vector", 20:03:31 DEBUG "instance": "10.142.0.107:9100" 20:03:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:03:31 DEBUG "resultType": "vector", 20:03:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:03:31 DEBUG "result": [ 20:03:31 DEBUG }, 20:03:31 DEBUG ----- output ----- 20:03:31 DEBUG "result": [ 20:03:31 DEBUG ----- output ----- 20:03:31 DEBUG { 20:03:31 DEBUG "value": [ 20:03:31 DEBUG { 20:03:31 DEBUG { 20:03:31 DEBUG { 20:03:31 DEBUG "metric": { 20:03:31 DEBUG 20:03:31 DEBUG "status": "success", 20:03:31 DEBUG "metric": { 20:03:31 DEBUG "status": "success", 20:03:31 DEBUG "pod": "ds-idrepo-1" 20:03:31 DEBUG 20:03:31 DEBUG "data": { 20:03:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:03:31 DEBUG "data": { 20:03:31 DEBUG }, 20:03:31 DEBUG 20:03:31 DEBUG "resultType": "vector", 20:03:31 DEBUG }, 20:03:31 DEBUG "resultType": "vector", 20:03:31 DEBUG "value": [ 20:03:31 DEBUG [...] 20:03:31 DEBUG "result": [ 20:03:31 DEBUG "value": [ 20:03:31 DEBUG "result": [ 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG { 20:03:31 DEBUG 20:03:31 DEBUG { 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG "metric": { 20:03:31 DEBUG 20:03:31 DEBUG "metric": { 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:03:31 DEBUG 20:03:31 DEBUG "instance": "10.142.0.107:9100" 20:03:31 DEBUG [...] 20:03:31 DEBUG "instance": "10.142.0.43:9100" 20:03:31 DEBUG }, 20:03:31 DEBUG [...] 20:03:31 DEBUG }, 20:03:31 DEBUG 20:03:31 DEBUG }, 20:03:31 DEBUG "value": [ 20:03:31 DEBUG 20:03:31 DEBUG "value": [ 20:03:31 DEBUG 20:03:31 DEBUG "value": [ 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 1713553411, 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 INFO [http_cmd]: http status code OK 20:03:31 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:03:31 DEBUG "75413378.06666666" 20:03:31 DEBUG 20:03:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:03:31 DEBUG 20:03:31 DEBUG --- status code --- 20:03:31 DEBUG }, 20:03:31 DEBUG ] 20:03:31 DEBUG [...] 20:03:31 DEBUG }, 20:03:31 DEBUG [...] 20:03:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:31 DEBUG "value": [ 20:03:31 DEBUG } 20:03:31 DEBUG 20:03:31 DEBUG "value": [ 20:03:31 DEBUG 20:03:31 DEBUG --- http response --- 20:03:31 DEBUG 1713553411, 20:03:31 DEBUG ] 20:03:31 DEBUG 20:03:31 DEBUG 1713553411, 20:03:31 DEBUG 20:03:31 DEBUG "0" 20:03:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:03:31 DEBUG } 20:03:31 DEBUG 20:03:31 DEBUG "2489.733333333333" 20:03:31 DEBUG 20:03:31 DEBUG ] 20:03:31 DEBUG ----- output ----- 20:03:31 DEBUG } 20:03:31 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:03:31 DEBUG ] 20:03:31 DEBUG "instance": "10.142.0.43:9100" 20:03:31 DEBUG } 20:03:31 DEBUG { 20:03:31 DEBUG }, 20:03:31 DEBUG } 20:03:31 DEBUG }, 20:03:31 DEBUG ] 20:03:31 DEBUG "status": "success", 20:03:31 DEBUG "value": [ 20:03:31 DEBUG ] 20:03:31 DEBUG "value": [ 20:03:31 DEBUG } 20:03:31 DEBUG "data": { 20:03:31 DEBUG 1713553411, 20:03:31 DEBUG } 20:03:31 DEBUG 1713553411, 20:03:31 DEBUG } 20:03:31 DEBUG "resultType": "vector", 20:03:31 DEBUG "8.032133062889121" 20:03:31 DEBUG } 20:03:31 DEBUG "75415818.96666667" 20:03:31 DEBUG "result": [ 20:03:31 DEBUG ] 20:03:31 DEBUG ] 20:03:31 DEBUG { 20:03:31 DEBUG } 20:03:31 DEBUG } 20:03:31 DEBUG "metric": { 20:03:31 DEBUG ] 20:03:31 DEBUG ] 20:03:31 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:03:31 DEBUG } 20:03:31 DEBUG } 20:03:31 DEBUG }, 20:03:31 DEBUG } 20:03:31 DEBUG } 20:03:31 DEBUG "value": [ 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG [...] 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:03:31 DEBUG }, 20:03:31 DEBUG "value": [ 20:03:31 DEBUG 1713553411, 20:03:31 DEBUG "3661131776" 20:03:31 DEBUG ] 20:03:31 DEBUG } 20:03:31 DEBUG ] 20:03:31 DEBUG } 20:03:31 DEBUG } 20:03:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553411" 20:03:31 INFO [http_cmd]: http status code OK 20:03:31 DEBUG --- status code --- 20:03:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:31 DEBUG --- http response --- 20:03:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:03:31 DEBUG ----- output ----- 20:03:31 DEBUG { 20:03:31 DEBUG "status": "success", 20:03:31 DEBUG "data": { 20:03:31 DEBUG "resultType": "vector", 20:03:31 DEBUG "result": [ 20:03:31 DEBUG { 20:03:31 DEBUG "metric": { 20:03:31 DEBUG "pod": "ds-idrepo-1" 20:03:31 DEBUG }, 20:03:31 DEBUG "value": [ 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG [...] 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG 20:03:31 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:03:31 DEBUG }, 20:03:31 DEBUG "value": [ 20:03:31 DEBUG 1713553411, 20:03:31 DEBUG "0.000027895612040598717" 20:03:31 DEBUG ] 20:03:31 DEBUG } 20:03:31 DEBUG ] 20:03:31 DEBUG } 20:03:31 DEBUG } 20:03:33 INFO 20:03:33 INFO [loop_until]: kubectl --namespace=xlou top pods 20:03:33 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:03:33 INFO [loop_until]: OK (rc = 0) 20:03:33 DEBUG --- stdout --- 20:03:33 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 16265m 9005Mi am-767f6b67b5-qcqz2 16382m 8813Mi am-767f6b67b5-vbltz 17532m 8996Mi ds-cts-0 16956m 10984Mi ds-cts-1 15287m 11239Mi ds-cts-2 17968m 10967Mi ds-idrepo-0 7115m 3686Mi ds-idrepo-1 6804m 3537Mi ds-idrepo-2 7238m 3493Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8063m 3399Mi 20:03:33 DEBUG --- stderr --- 20:03:33 DEBUG 20:03:34 INFO 20:03:34 INFO [loop_until]: kubectl --namespace=xlou top node 20:03:34 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:03:34 INFO [loop_until]: OK (rc = 0) 20:03:34 DEBUG --- stdout --- 20:03:34 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 70m 0% 6240Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7351m 24% 7280Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16020m 53% 11707Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 85m 0% 3344Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 17114m 57% 13945Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 148m 0% 5187Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 16437m 55% 11689Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15104m 50% 2979Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 18091m 60% 11216Mi 10% gke-xlou-cdm-ds-0f601751-j488 7593m 25% 6744Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 15380m 51% 15968Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 7377m 24% 7428Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 17620m 59% 13416Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 8204m 51% 7134Mi 12% 20:03:34 DEBUG --- stderr --- 20:03:34 DEBUG 20:04:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553471" 20:04:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553471" 20:04:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553471" 20:04:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553471" 20:04:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713553471" 20:04:31 INFO [http_cmd]: http status code OK 20:04:31 INFO [http_cmd]: http status code OK 20:04:31 INFO [http_cmd]: http status code OK 20:04:31 INFO [http_cmd]: http status code OK 20:04:31 DEBUG --- status code --- 20:04:31 DEBUG --- status code --- 20:04:31 DEBUG --- status code --- 20:04:31 DEBUG --- status code --- 20:04:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:31 DEBUG --- http response --- 20:04:31 DEBUG --- http response --- 20:04:31 DEBUG --- http response --- 20:04:31 DEBUG --- http response --- 20:04:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:04:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:04:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:04:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:04:31 DEBUG ----- output ----- 20:04:31 DEBUG ----- output ----- 20:04:31 DEBUG ----- output ----- 20:04:31 DEBUG ----- output ----- 20:04:31 DEBUG { 20:04:31 DEBUG { 20:04:31 DEBUG { 20:04:31 DEBUG { 20:04:31 DEBUG "status": "success", 20:04:31 DEBUG "status": "success", 20:04:31 DEBUG "status": "success", 20:04:31 DEBUG "status": "success", 20:04:31 DEBUG "data": { 20:04:31 DEBUG "data": { 20:04:31 DEBUG "data": { 20:04:31 DEBUG "data": { 20:04:31 DEBUG "resultType": "vector", 20:04:31 DEBUG "resultType": "vector", 20:04:31 DEBUG "resultType": "vector", 20:04:31 DEBUG "resultType": "vector", 20:04:31 DEBUG "result": [ 20:04:31 DEBUG "result": [ 20:04:31 DEBUG "result": [ 20:04:31 DEBUG "result": [ 20:04:31 DEBUG { 20:04:31 DEBUG { 20:04:31 DEBUG { 20:04:31 DEBUG { 20:04:31 DEBUG "metric": { 20:04:31 DEBUG "metric": { 20:04:31 DEBUG "metric": { 20:04:31 DEBUG "metric": { 20:04:31 DEBUG "pod": "ds-idrepo-0" 20:04:31 DEBUG "instance": "10.142.0.107:9100" 20:04:31 DEBUG "pod": "ds-idrepo-0" 20:04:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:04:31 DEBUG }, 20:04:31 DEBUG }, 20:04:31 DEBUG }, 20:04:31 DEBUG }, 20:04:31 DEBUG "value": [ 20:04:31 DEBUG "value": [ 20:04:31 DEBUG "value": [ 20:04:31 DEBUG "value": [ 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG [...] 20:04:31 DEBUG [...] 20:04:31 DEBUG [...] 20:04:31 DEBUG [...] 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553471" 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG "pod": "ds-idrepo-2" 20:04:31 DEBUG "instance": "10.142.0.43:9100" 20:04:31 DEBUG "pod": "ds-idrepo-2" 20:04:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:04:31 DEBUG }, 20:04:31 DEBUG }, 20:04:31 DEBUG }, 20:04:31 DEBUG }, 20:04:31 DEBUG "value": [ 20:04:31 DEBUG "value": [ 20:04:31 DEBUG "value": [ 20:04:31 DEBUG "value": [ 20:04:31 DEBUG 1713553471, 20:04:31 DEBUG 1713553471, 20:04:31 DEBUG 1713553471, 20:04:31 DEBUG 1713553471, 20:04:31 DEBUG "0" 20:04:31 DEBUG "0.18199999999993016" 20:04:31 DEBUG "0" 20:04:31 DEBUG "0" 20:04:31 DEBUG ] 20:04:31 DEBUG ] 20:04:31 DEBUG ] 20:04:31 DEBUG ] 20:04:31 DEBUG } 20:04:31 DEBUG } 20:04:31 DEBUG } 20:04:31 DEBUG } 20:04:31 DEBUG ] 20:04:31 DEBUG ] 20:04:31 DEBUG ] 20:04:31 DEBUG ] 20:04:31 DEBUG } 20:04:31 DEBUG } 20:04:31 DEBUG } 20:04:31 DEBUG } 20:04:31 DEBUG } 20:04:31 DEBUG } 20:04:31 DEBUG } 20:04:31 DEBUG } 20:04:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553471" 20:04:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553471" 20:04:31 INFO [http_cmd]: http status code OK 20:04:31 ERROR [http_cmd]: ERROR 20:04:31 DEBUG --- status code --- 20:04:31 INFO [http_cmd]: http status code OK 20:04:31 ERROR --- status code --- 20:04:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:31 DEBUG --- status code --- 20:04:31 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:04:31 DEBUG --- http response --- 20:04:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:31 ERROR --- http response --- 20:04:31 DEBUG --- http response --- 20:04:31 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:04:31 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:04:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:04:31 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:04:31 DEBUG ----- output ----- 20:04:31 DEBUG { 20:04:31 DEBUG "status": "success", 20:04:31 DEBUG "data": { 20:04:31 DEBUG "resultType": "vector", 20:04:31 DEBUG "result": [ 20:04:31 DEBUG { 20:04:31 DEBUG "metric": { 20:04:31 DEBUG "pod": "ds-idrepo-0" 20:04:31 DEBUG }, 20:04:31 DEBUG "value": [ 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG [...] 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:04:31 DEBUG }, 20:04:31 DEBUG "value": [ 20:04:31 DEBUG 1713553471, 20:04:31 DEBUG "0" 20:04:31 DEBUG ] 20:04:31 DEBUG } 20:04:31 DEBUG ] 20:04:31 DEBUG } 20:04:31 DEBUG } 20:04:31 INFO [http_cmd]: http status code OK 20:04:31 DEBUG --- status code --- 20:04:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:31 DEBUG --- http response --- 20:04:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:04:31 DEBUG ----- output ----- 20:04:31 DEBUG { 20:04:31 DEBUG "status": "success", 20:04:31 DEBUG "data": { 20:04:31 DEBUG "resultType": "vector", 20:04:31 DEBUG "result": [ 20:04:31 DEBUG { 20:04:31 DEBUG "metric": { 20:04:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:04:31 DEBUG }, 20:04:31 DEBUG "value": [ 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG [...] 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:04:31 DEBUG }, 20:04:31 DEBUG "value": [ 20:04:31 DEBUG 1713553471, 20:04:31 DEBUG "450129.5" 20:04:31 DEBUG ] 20:04:31 DEBUG } 20:04:31 DEBUG ] 20:04:31 DEBUG } 20:04:31 DEBUG } 20:04:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553471" 20:04:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553471" 20:04:31 INFO [http_cmd]: http status code OK 20:04:31 DEBUG --- status code --- 20:04:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:31 DEBUG --- http response --- 20:04:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:04:31 DEBUG ----- output ----- 20:04:31 DEBUG { 20:04:31 DEBUG "status": "success", 20:04:31 DEBUG "data": { 20:04:31 DEBUG "resultType": "vector", 20:04:31 DEBUG "result": [ 20:04:31 DEBUG { 20:04:31 DEBUG "metric": { 20:04:31 DEBUG "instance": "10.142.0.107:9100" 20:04:31 DEBUG }, 20:04:31 DEBUG "value": [ 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG [...] 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG "instance": "10.142.0.43:9100" 20:04:31 DEBUG }, 20:04:31 DEBUG "value": [ 20:04:31 DEBUG 1713553471, 20:04:31 DEBUG "0.08133333333333324" 20:04:31 DEBUG ] 20:04:31 DEBUG } 20:04:31 DEBUG ] 20:04:31 DEBUG } 20:04:31 DEBUG } 20:04:31 INFO [http_cmd]: http status code OK 20:04:31 DEBUG --- status code --- 20:04:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:31 DEBUG --- http response --- 20:04:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:04:31 DEBUG ----- output ----- 20:04:31 DEBUG { 20:04:31 DEBUG "status": "success", 20:04:31 DEBUG "data": { 20:04:31 DEBUG "resultType": "vector", 20:04:31 DEBUG "result": [ 20:04:31 DEBUG { 20:04:31 DEBUG "metric": { 20:04:31 DEBUG "pod": "ds-cts-0" 20:04:31 DEBUG }, 20:04:31 DEBUG "value": [ 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG [...] 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG "pod": "ds-cts-2" 20:04:31 DEBUG }, 20:04:31 DEBUG "value": [ 20:04:31 DEBUG 1713553471, 20:04:31 DEBUG "0.0012205066666666666" 20:04:31 DEBUG ] 20:04:31 DEBUG } 20:04:31 DEBUG ] 20:04:31 DEBUG } 20:04:31 DEBUG } 20:04:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713553471" 20:04:31 INFO [http_cmd]: http status code OK 20:04:31 DEBUG --- status code --- 20:04:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:31 DEBUG --- http response --- 20:04:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:04:31 DEBUG ----- output ----- 20:04:31 DEBUG { 20:04:31 DEBUG "status": "success", 20:04:31 DEBUG "data": { 20:04:31 DEBUG "resultType": "vector", 20:04:31 DEBUG "result": [ 20:04:31 DEBUG { 20:04:31 DEBUG "metric": { 20:04:31 DEBUG "pod": "ds-idrepo-0" 20:04:31 DEBUG }, 20:04:31 DEBUG "value": [ 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG [...] 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG "pod": "ds-cts-2" 20:04:31 DEBUG }, 20:04:31 DEBUG "value": [ 20:04:31 DEBUG 1713553471, 20:04:31 DEBUG "0" 20:04:31 DEBUG ] 20:04:31 DEBUG } 20:04:31 DEBUG ] 20:04:31 DEBUG } 20:04:31 DEBUG } 20:04:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553471" 20:04:31 INFO [http_cmd]: http status code OK 20:04:31 DEBUG --- status code --- 20:04:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:31 DEBUG --- http response --- 20:04:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:04:31 DEBUG ----- output ----- 20:04:31 DEBUG { 20:04:31 DEBUG "status": "success", 20:04:31 DEBUG "data": { 20:04:31 DEBUG "resultType": "vector", 20:04:31 DEBUG "result": [ 20:04:31 DEBUG { 20:04:31 DEBUG "metric": { 20:04:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:04:31 DEBUG }, 20:04:31 DEBUG "value": [ 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG [...] 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:04:31 DEBUG }, 20:04:31 DEBUG "value": [ 20:04:31 DEBUG 1713553471, 20:04:31 DEBUG "2499.9666666666667" 20:04:31 DEBUG ] 20:04:31 DEBUG } 20:04:31 DEBUG ] 20:04:31 DEBUG } 20:04:31 DEBUG } 20:04:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553471" 20:04:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553471" 20:04:31 INFO [http_cmd]: http status code OK 20:04:31 DEBUG --- status code --- 20:04:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:31 DEBUG --- http response --- 20:04:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:04:31 DEBUG ----- output ----- 20:04:31 DEBUG { 20:04:31 DEBUG "status": "success", 20:04:31 DEBUG "data": { 20:04:31 DEBUG "resultType": "vector", 20:04:31 DEBUG "result": [ 20:04:31 DEBUG { 20:04:31 DEBUG "metric": { 20:04:31 DEBUG "instance": "10.142.0.107:9100" 20:04:31 DEBUG }, 20:04:31 DEBUG "value": [ 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG [...] 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG "instance": "10.142.0.43:9100" 20:04:31 DEBUG }, 20:04:31 DEBUG "value": [ 20:04:31 DEBUG 1713553471, 20:04:31 DEBUG "2014088832000" 20:04:31 DEBUG ] 20:04:31 DEBUG } 20:04:31 DEBUG ] 20:04:31 DEBUG } 20:04:31 DEBUG } 20:04:31 INFO [http_cmd]: http status code OK 20:04:31 DEBUG --- status code --- 20:04:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:31 DEBUG --- http response --- 20:04:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:04:31 DEBUG ----- output ----- 20:04:31 DEBUG { 20:04:31 DEBUG "status": "success", 20:04:31 DEBUG "data": { 20:04:31 DEBUG "resultType": "vector", 20:04:31 DEBUG "result": [ 20:04:31 DEBUG { 20:04:31 DEBUG "metric": { 20:04:31 DEBUG "pod": "ds-idrepo-0" 20:04:31 DEBUG }, 20:04:31 DEBUG "value": [ 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG [...] 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:04:31 DEBUG }, 20:04:31 DEBUG "value": [ 20:04:31 DEBUG 1713553471, 20:04:31 DEBUG "0" 20:04:31 DEBUG ] 20:04:31 DEBUG } 20:04:31 DEBUG ] 20:04:31 DEBUG } 20:04:31 DEBUG } 20:04:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553471" 20:04:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713553471" 20:04:31 INFO [http_cmd]: http status code OK 20:04:31 DEBUG --- status code --- 20:04:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:31 DEBUG --- http response --- 20:04:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:04:31 DEBUG ----- output ----- 20:04:31 DEBUG { 20:04:31 DEBUG "status": "success", 20:04:31 DEBUG "data": { 20:04:31 DEBUG "resultType": "vector", 20:04:31 DEBUG "result": [ 20:04:31 DEBUG { 20:04:31 DEBUG "metric": { 20:04:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:04:31 DEBUG }, 20:04:31 DEBUG "value": [ 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG [...] 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:04:31 DEBUG }, 20:04:31 DEBUG "value": [ 20:04:31 DEBUG 1713553471, 20:04:31 DEBUG "2498.733333333333" 20:04:31 DEBUG ] 20:04:31 DEBUG } 20:04:31 DEBUG ] 20:04:31 DEBUG } 20:04:31 DEBUG } 20:04:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553471" 20:04:31 INFO [http_cmd]: http status code OK 20:04:31 DEBUG --- status code --- 20:04:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:31 DEBUG --- http response --- 20:04:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:04:31 DEBUG ----- output ----- 20:04:31 DEBUG { 20:04:31 DEBUG "status": "success", 20:04:31 DEBUG "data": { 20:04:31 DEBUG "resultType": "vector", 20:04:31 DEBUG "result": [ 20:04:31 DEBUG { 20:04:31 DEBUG "metric": { 20:04:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:04:31 DEBUG }, 20:04:31 DEBUG "value": [ 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG [...] 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:04:31 DEBUG }, 20:04:31 DEBUG "value": [ 20:04:31 DEBUG 1713553471, 20:04:31 DEBUG "0.015" 20:04:31 DEBUG ] 20:04:31 DEBUG } 20:04:31 DEBUG ] 20:04:31 DEBUG } 20:04:31 DEBUG } 20:04:31 INFO [http_cmd]: http status code OK 20:04:31 DEBUG --- status code --- 20:04:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:31 DEBUG --- http response --- 20:04:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:04:31 DEBUG ----- output ----- 20:04:31 DEBUG { 20:04:31 DEBUG "status": "success", 20:04:31 DEBUG "data": { 20:04:31 DEBUG "resultType": "vector", 20:04:31 DEBUG "result": [ 20:04:31 DEBUG { 20:04:31 DEBUG "metric": { 20:04:31 DEBUG "pod": "ds-idrepo-0" 20:04:31 DEBUG }, 20:04:31 DEBUG "value": [ 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG [...] 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG "pod": "ds-idrepo-2" 20:04:31 DEBUG }, 20:04:31 DEBUG "value": [ 20:04:31 DEBUG 1713553471, 20:04:31 DEBUG "0" 20:04:31 DEBUG ] 20:04:31 DEBUG } 20:04:31 DEBUG ] 20:04:31 DEBUG } 20:04:31 DEBUG } 20:04:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553471" 20:04:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553471" 20:04:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713553471" 20:04:31 INFO [http_cmd]: http status code OK 20:04:31 DEBUG --- status code --- 20:04:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:31 DEBUG --- http response --- 20:04:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:04:31 DEBUG ----- output ----- 20:04:31 DEBUG { 20:04:31 DEBUG "status": "success", 20:04:31 DEBUG "data": { 20:04:31 DEBUG "resultType": "vector", 20:04:31 DEBUG "result": [ 20:04:31 DEBUG { 20:04:31 DEBUG "metric": { 20:04:31 DEBUG "pod": "ds-cts-0" 20:04:31 DEBUG }, 20:04:31 DEBUG "value": [ 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG [...] 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG "pod": "ds-cts-2" 20:04:31 DEBUG }, 20:04:31 DEBUG "value": [ 20:04:31 DEBUG 1713553471, 20:04:31 DEBUG "29.933333333333334" 20:04:31 DEBUG ] 20:04:31 DEBUG } 20:04:31 DEBUG ] 20:04:31 DEBUG } 20:04:31 DEBUG } 20:04:31 INFO [http_cmd]: http status code OK 20:04:31 DEBUG --- status code --- 20:04:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:31 DEBUG --- http response --- 20:04:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:04:31 DEBUG ----- output ----- 20:04:31 DEBUG { 20:04:31 DEBUG "status": "success", 20:04:31 DEBUG "data": { 20:04:31 DEBUG "resultType": "vector", 20:04:31 DEBUG "result": [ 20:04:31 DEBUG { 20:04:31 DEBUG "metric": { 20:04:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:04:31 DEBUG }, 20:04:31 DEBUG "value": [ 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG [...] 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:04:31 DEBUG }, 20:04:31 DEBUG "value": [ 20:04:31 DEBUG 1713553471, 20:04:31 DEBUG "2498.8" 20:04:31 DEBUG ] 20:04:31 DEBUG } 20:04:31 DEBUG ] 20:04:31 DEBUG } 20:04:31 DEBUG } 20:04:31 INFO [http_cmd]: http status code OK 20:04:31 DEBUG --- status code --- 20:04:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:31 DEBUG --- http response --- 20:04:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:04:31 DEBUG ----- output ----- 20:04:31 DEBUG { 20:04:31 DEBUG "status": "success", 20:04:31 DEBUG "data": { 20:04:31 DEBUG "resultType": "vector", 20:04:31 DEBUG "result": [ 20:04:31 DEBUG { 20:04:31 DEBUG "metric": { 20:04:31 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:04:31 DEBUG }, 20:04:31 DEBUG "value": [ 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG [...] 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:04:31 DEBUG }, 20:04:31 DEBUG "value": [ 20:04:31 DEBUG 1713553471, 20:04:31 DEBUG "3663593472" 20:04:31 DEBUG ] 20:04:31 DEBUG } 20:04:31 DEBUG ] 20:04:31 DEBUG } 20:04:31 DEBUG } 20:04:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553471" 20:04:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553471" 20:04:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553471" 20:04:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553471" 20:04:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713553471" 20:04:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553471" 20:04:31 INFO [http_cmd]: http status code OK 20:04:31 INFO [http_cmd]: http status code OK 20:04:31 DEBUG --- status code --- 20:04:31 INFO [http_cmd]: http status code OK 20:04:31 DEBUG --- status code --- 20:04:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:31 DEBUG --- status code --- 20:04:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:31 DEBUG --- http response --- 20:04:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:31 DEBUG --- http response --- 20:04:31 DEBUG --- http response --- 20:04:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:04:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:04:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:04:31 INFO [http_cmd]: http status code OK 20:04:31 DEBUG ----- output ----- 20:04:31 INFO [http_cmd]: http status code OK 20:04:31 DEBUG ----- output ----- 20:04:31 DEBUG ----- output ----- 20:04:31 DEBUG --- status code --- 20:04:31 DEBUG { 20:04:31 DEBUG --- status code --- 20:04:31 DEBUG { 20:04:31 DEBUG { 20:04:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:31 DEBUG "status": "success", 20:04:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:31 DEBUG "status": "success", 20:04:31 DEBUG "status": "success", 20:04:31 DEBUG --- http response --- 20:04:31 DEBUG "data": { 20:04:31 DEBUG --- http response --- 20:04:31 DEBUG "data": { 20:04:31 INFO [http_cmd]: http status code OK 20:04:31 DEBUG "data": { 20:04:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:04:31 DEBUG "resultType": "vector", 20:04:31 DEBUG "resultType": "vector", 20:04:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:04:31 DEBUG --- status code --- 20:04:31 DEBUG "resultType": "vector", 20:04:31 DEBUG ----- output ----- 20:04:31 DEBUG "result": [ 20:04:31 DEBUG "result": [ 20:04:31 DEBUG ----- output ----- 20:04:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:31 DEBUG "result": [ 20:04:31 DEBUG { 20:04:31 DEBUG { 20:04:31 DEBUG { 20:04:31 DEBUG { 20:04:31 DEBUG --- http response --- 20:04:31 DEBUG { 20:04:31 DEBUG "status": "success", 20:04:31 DEBUG "metric": { 20:04:31 DEBUG "metric": { 20:04:31 DEBUG "status": "success", 20:04:31 DEBUG "metric": { 20:04:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:04:31 DEBUG "data": { 20:04:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:04:31 DEBUG "instance": "10.142.0.107:9100" 20:04:31 DEBUG "data": { 20:04:31 DEBUG "pod": "ds-idrepo-0" 20:04:31 DEBUG ----- output ----- 20:04:31 DEBUG "resultType": "vector", 20:04:31 DEBUG }, 20:04:31 DEBUG }, 20:04:31 DEBUG "resultType": "vector", 20:04:31 DEBUG }, 20:04:31 DEBUG { 20:04:31 DEBUG "result": [ 20:04:31 DEBUG "value": [ 20:04:31 DEBUG "value": [ 20:04:31 DEBUG "result": [ 20:04:31 DEBUG "value": [ 20:04:31 DEBUG "status": "success", 20:04:31 DEBUG { 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG { 20:04:31 DEBUG 20:04:31 DEBUG "data": { 20:04:31 DEBUG "metric": { 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553471" 20:04:31 DEBUG "metric": { 20:04:31 DEBUG 20:04:31 DEBUG "resultType": "vector", 20:04:31 DEBUG "pod": "ds-cts-0" 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG "instance": "10.142.0.107:9100" 20:04:31 DEBUG 20:04:31 DEBUG "result": [ 20:04:31 DEBUG }, 20:04:31 DEBUG [...] 20:04:31 DEBUG [...] 20:04:31 DEBUG }, 20:04:31 DEBUG [...] 20:04:31 DEBUG { 20:04:31 DEBUG "value": [ 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG "value": [ 20:04:31 DEBUG 20:04:31 DEBUG "metric": { 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG }, 20:04:31 DEBUG 20:04:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:04:31 DEBUG "instance": "10.142.0.43:9100" 20:04:31 DEBUG 20:04:31 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:04:31 DEBUG "value": [ 20:04:31 DEBUG [...] 20:04:31 DEBUG }, 20:04:31 DEBUG }, 20:04:31 DEBUG [...] 20:04:31 DEBUG }, 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG "value": [ 20:04:31 DEBUG "value": [ 20:04:31 DEBUG 20:04:31 DEBUG "value": [ 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 1713553471, 20:04:31 DEBUG 1713553471, 20:04:31 DEBUG 20:04:31 DEBUG 1713553471, 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG "2498.866666666667" 20:04:31 DEBUG "75428941.83333333" 20:04:31 DEBUG 20:04:31 DEBUG "0" 20:04:31 DEBUG [...] 20:04:31 DEBUG "pod": "ds-cts-2" 20:04:31 DEBUG ] 20:04:31 DEBUG ] 20:04:31 DEBUG "instance": "10.142.0.43:9100" 20:04:31 DEBUG ] 20:04:31 DEBUG 20:04:31 DEBUG }, 20:04:31 DEBUG } 20:04:31 DEBUG } 20:04:31 DEBUG }, 20:04:31 DEBUG } 20:04:31 DEBUG 20:04:31 DEBUG "value": [ 20:04:31 DEBUG ] 20:04:31 DEBUG ] 20:04:31 DEBUG "value": [ 20:04:31 DEBUG ] 20:04:31 DEBUG 20:04:31 DEBUG 1713553471, 20:04:31 DEBUG } 20:04:31 DEBUG } 20:04:31 DEBUG 1713553471, 20:04:31 DEBUG } 20:04:31 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:04:31 DEBUG "0" 20:04:31 DEBUG } 20:04:31 DEBUG } 20:04:31 DEBUG "75426820.8" 20:04:31 DEBUG } 20:04:31 DEBUG }, 20:04:31 DEBUG ] 20:04:31 DEBUG ] 20:04:31 DEBUG "value": [ 20:04:31 DEBUG } 20:04:31 DEBUG } 20:04:31 DEBUG 1713553471, 20:04:31 DEBUG ] 20:04:31 DEBUG ] 20:04:31 DEBUG "8.040227422732896" 20:04:31 DEBUG } 20:04:31 DEBUG } 20:04:31 DEBUG ] 20:04:31 DEBUG } 20:04:31 DEBUG } 20:04:31 DEBUG } 20:04:31 DEBUG ] 20:04:31 DEBUG } 20:04:31 DEBUG } 20:04:31 INFO [http_cmd]: http status code OK 20:04:31 DEBUG --- status code --- 20:04:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:31 DEBUG --- http response --- 20:04:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:04:31 DEBUG ----- output ----- 20:04:31 DEBUG { 20:04:31 DEBUG "status": "success", 20:04:31 DEBUG "data": { 20:04:31 DEBUG "resultType": "vector", 20:04:31 DEBUG "result": [ 20:04:31 DEBUG { 20:04:31 DEBUG "metric": { 20:04:31 DEBUG "pod": "ds-idrepo-0" 20:04:31 DEBUG }, 20:04:31 DEBUG "value": [ 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG [...] 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG 20:04:31 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:04:31 DEBUG }, 20:04:31 DEBUG "value": [ 20:04:31 DEBUG 1713553471, 20:04:31 DEBUG "0.000017042869641283716" 20:04:31 DEBUG ] 20:04:31 DEBUG } 20:04:31 DEBUG ] 20:04:31 DEBUG } 20:04:31 DEBUG } 20:04:33 INFO 20:04:33 INFO [loop_until]: kubectl --namespace=xlou top pods 20:04:33 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:04:33 INFO [loop_until]: OK (rc = 0) 20:04:33 DEBUG --- stdout --- 20:04:33 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 16041m 9009Mi am-767f6b67b5-qcqz2 16672m 8812Mi am-767f6b67b5-vbltz 17551m 8999Mi ds-cts-0 17462m 10968Mi ds-cts-1 15754m 11202Mi ds-cts-2 17651m 10968Mi ds-idrepo-0 7115m 3686Mi ds-idrepo-1 7087m 3537Mi ds-idrepo-2 7283m 3493Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8294m 3402Mi 20:04:33 DEBUG --- stderr --- 20:04:33 DEBUG 20:04:34 INFO 20:04:34 INFO [loop_until]: kubectl --namespace=xlou top node 20:04:34 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:04:34 INFO [loop_until]: OK (rc = 0) 20:04:34 DEBUG --- stdout --- 20:04:34 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 72m 0% 6240Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7395m 24% 7289Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16109m 53% 11720Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 86m 0% 3336Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 17240m 57% 13944Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 159m 0% 5189Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 16414m 54% 11697Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15373m 51% 2990Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 18089m 60% 11209Mi 10% gke-xlou-cdm-ds-0f601751-j488 7663m 25% 6734Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 15880m 53% 15972Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 7459m 24% 7430Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 17851m 59% 13417Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 8194m 51% 7132Mi 12% 20:04:34 DEBUG --- stderr --- 20:04:34 DEBUG 20:05:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553531" 20:05:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553531" 20:05:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553531" 20:05:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553531" 20:05:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553531" 20:05:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713553531" 20:05:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553531" 20:05:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553531" 20:05:31 INFO [http_cmd]: http status code OK 20:05:31 DEBUG --- status code --- 20:05:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:31 DEBUG --- http response --- 20:05:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553531" 20:05:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:05:31 DEBUG ----- output ----- 20:05:31 DEBUG { 20:05:31 DEBUG "status": "success", 20:05:31 DEBUG "data": { 20:05:31 DEBUG "resultType": "vector", 20:05:31 DEBUG "result": [ 20:05:31 DEBUG { 20:05:31 DEBUG "metric": { 20:05:31 DEBUG "pod": "ds-idrepo-0" 20:05:31 DEBUG }, 20:05:31 DEBUG "value": [ 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG [...] 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG "pod": "ds-idrepo-2" 20:05:31 DEBUG }, 20:05:31 DEBUG "value": [ 20:05:31 DEBUG 1713553531, 20:05:31 DEBUG "0" 20:05:31 DEBUG ] 20:05:31 DEBUG } 20:05:31 DEBUG ] 20:05:31 DEBUG } 20:05:31 DEBUG } 20:05:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553531" 20:05:31 INFO [http_cmd]: http status code OK 20:05:31 DEBUG --- status code --- 20:05:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:31 DEBUG --- http response --- 20:05:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:05:31 DEBUG ----- output ----- 20:05:31 DEBUG { 20:05:31 DEBUG "status": "success", 20:05:31 INFO [http_cmd]: http status code OK 20:05:31 DEBUG "data": { 20:05:31 INFO [http_cmd]: http status code OK 20:05:31 DEBUG --- status code --- 20:05:31 DEBUG "resultType": "vector", 20:05:31 DEBUG --- status code --- 20:05:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:31 DEBUG "result": [ 20:05:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:31 DEBUG --- http response --- 20:05:31 DEBUG { 20:05:31 DEBUG --- http response --- 20:05:31 DEBUG "metric": { 20:05:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:05:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:05:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:05:31 DEBUG ----- output ----- 20:05:31 DEBUG ----- output ----- 20:05:31 DEBUG }, 20:05:31 DEBUG { 20:05:31 DEBUG { 20:05:31 DEBUG "value": [ 20:05:31 DEBUG "status": "success", 20:05:31 DEBUG "status": "success", 20:05:31 DEBUG 20:05:31 DEBUG "data": { 20:05:31 DEBUG "data": { 20:05:31 DEBUG 20:05:31 DEBUG "resultType": "vector", 20:05:31 DEBUG "resultType": "vector", 20:05:31 DEBUG 20:05:31 DEBUG "result": [ 20:05:31 DEBUG "result": [ 20:05:31 DEBUG [...] 20:05:31 DEBUG { 20:05:31 DEBUG { 20:05:31 DEBUG 20:05:31 DEBUG "metric": { 20:05:31 DEBUG "metric": { 20:05:31 DEBUG 20:05:31 DEBUG "pod": "ds-idrepo-0" 20:05:31 DEBUG "instance": "10.142.0.107:9100" 20:05:31 DEBUG 20:05:31 DEBUG }, 20:05:31 DEBUG }, 20:05:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:05:31 DEBUG "value": [ 20:05:31 DEBUG "value": [ 20:05:31 DEBUG }, 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG "value": [ 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 1713553531, 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG "0" 20:05:31 DEBUG [...] 20:05:31 DEBUG [...] 20:05:31 DEBUG ] 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG } 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG ] 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG } 20:05:31 DEBUG "pod": "ds-idrepo-2" 20:05:31 DEBUG "instance": "10.142.0.43:9100" 20:05:31 DEBUG } 20:05:31 DEBUG }, 20:05:31 DEBUG }, 20:05:31 DEBUG "value": [ 20:05:31 DEBUG "value": [ 20:05:31 DEBUG 1713553531, 20:05:31 DEBUG 1713553531, 20:05:31 DEBUG "0" 20:05:31 DEBUG "0.22626666666661396" 20:05:31 DEBUG ] 20:05:31 DEBUG ] 20:05:31 DEBUG } 20:05:31 DEBUG } 20:05:31 DEBUG ] 20:05:31 DEBUG ] 20:05:31 DEBUG } 20:05:31 DEBUG } 20:05:31 DEBUG } 20:05:31 DEBUG } 20:05:31 ERROR [http_cmd]: ERROR 20:05:31 ERROR --- status code --- 20:05:31 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:05:31 ERROR --- http response --- 20:05:31 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:05:31 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:05:31 INFO [http_cmd]: http status code OK 20:05:31 INFO [http_cmd]: http status code OK 20:05:31 DEBUG --- status code --- 20:05:31 DEBUG --- status code --- 20:05:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:31 DEBUG --- http response --- 20:05:31 DEBUG --- http response --- 20:05:31 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:05:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:05:31 DEBUG ----- output ----- 20:05:31 DEBUG { 20:05:31 DEBUG "status": "success", 20:05:31 DEBUG "data": { 20:05:31 DEBUG "resultType": "vector", 20:05:31 DEBUG "result": [ 20:05:31 DEBUG { 20:05:31 DEBUG "metric": { 20:05:31 DEBUG "pod": "ds-cts-0" 20:05:31 DEBUG }, 20:05:31 DEBUG "value": [ 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG [...] 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG "pod": "ds-cts-2" 20:05:31 DEBUG }, 20:05:31 DEBUG "value": [ 20:05:31 DEBUG 1713553531, 20:05:31 DEBUG "0.00059086" 20:05:31 DEBUG ] 20:05:31 DEBUG } 20:05:31 DEBUG ] 20:05:31 DEBUG } 20:05:31 DEBUG } 20:05:31 INFO [http_cmd]: http status code OK 20:05:31 INFO [http_cmd]: http status code OK 20:05:31 DEBUG --- status code --- 20:05:31 DEBUG --- status code --- 20:05:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:31 DEBUG --- http response --- 20:05:31 DEBUG --- http response --- 20:05:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:05:31 DEBUG ----- output ----- 20:05:31 DEBUG { 20:05:31 DEBUG "status": "success", 20:05:31 DEBUG "data": { 20:05:31 DEBUG "resultType": "vector", 20:05:31 DEBUG "result": [ 20:05:31 DEBUG { 20:05:31 DEBUG "metric": { 20:05:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:05:31 DEBUG }, 20:05:31 DEBUG "value": [ 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG [...] 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:05:31 DEBUG }, 20:05:31 DEBUG "value": [ 20:05:31 DEBUG 1713553531, 20:05:31 DEBUG "449955.5" 20:05:31 DEBUG ] 20:05:31 DEBUG } 20:05:31 DEBUG ] 20:05:31 DEBUG } 20:05:31 DEBUG } 20:05:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:05:31 DEBUG ----- output ----- 20:05:31 DEBUG { 20:05:31 DEBUG "status": "success", 20:05:31 DEBUG "data": { 20:05:31 DEBUG "resultType": "vector", 20:05:31 DEBUG "result": [ 20:05:31 DEBUG { 20:05:31 DEBUG "metric": { 20:05:31 DEBUG "pod": "ds-idrepo-0" 20:05:31 DEBUG }, 20:05:31 DEBUG "value": [ 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG [...] 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG "pod": "idm-b85f9d978-wnhxb" 20:05:31 DEBUG }, 20:05:31 DEBUG "value": [ 20:05:31 DEBUG 1713553531, 20:05:31 DEBUG "0" 20:05:31 DEBUG ] 20:05:31 DEBUG } 20:05:31 DEBUG ] 20:05:31 DEBUG } 20:05:31 DEBUG } 20:05:31 INFO [http_cmd]: http status code OK 20:05:31 DEBUG --- status code --- 20:05:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:31 DEBUG --- http response --- 20:05:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:05:31 DEBUG ----- output ----- 20:05:31 DEBUG { 20:05:31 DEBUG "status": "success", 20:05:31 DEBUG "data": { 20:05:31 DEBUG "resultType": "vector", 20:05:31 DEBUG "result": [ 20:05:31 DEBUG { 20:05:31 DEBUG "metric": { 20:05:31 DEBUG "instance": "10.142.0.107:9100" 20:05:31 DEBUG }, 20:05:31 DEBUG "value": [ 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG [...] 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG "instance": "10.142.0.43:9100" 20:05:31 DEBUG }, 20:05:31 DEBUG "value": [ 20:05:31 DEBUG 1713553531, 20:05:31 DEBUG "0.10066666666666227" 20:05:31 DEBUG ] 20:05:31 DEBUG } 20:05:31 DEBUG ] 20:05:31 DEBUG } 20:05:31 DEBUG } 20:05:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713553531" 20:05:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553531" 20:05:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713553531" 20:05:31 INFO [http_cmd]: http status code OK 20:05:31 DEBUG --- status code --- 20:05:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:31 DEBUG --- http response --- 20:05:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:05:31 DEBUG ----- output ----- 20:05:31 DEBUG { 20:05:31 DEBUG "status": "success", 20:05:31 DEBUG "data": { 20:05:31 DEBUG "resultType": "vector", 20:05:31 DEBUG "result": [ 20:05:31 DEBUG { 20:05:31 DEBUG "metric": { 20:05:31 DEBUG "pod": "ds-idrepo-0" 20:05:31 DEBUG }, 20:05:31 DEBUG "value": [ 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG [...] 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG "pod": "ds-cts-2" 20:05:31 DEBUG }, 20:05:31 DEBUG "value": [ 20:05:31 DEBUG 1713553531, 20:05:31 DEBUG "0" 20:05:31 DEBUG ] 20:05:31 DEBUG } 20:05:31 DEBUG ] 20:05:31 DEBUG } 20:05:31 DEBUG } 20:05:31 INFO [http_cmd]: http status code OK 20:05:31 DEBUG --- status code --- 20:05:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:31 DEBUG --- http response --- 20:05:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:05:31 DEBUG ----- output ----- 20:05:31 DEBUG { 20:05:31 DEBUG "status": "success", 20:05:31 DEBUG "data": { 20:05:31 DEBUG "resultType": "vector", 20:05:31 DEBUG "result": [ 20:05:31 DEBUG { 20:05:31 DEBUG "metric": { 20:05:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:05:31 DEBUG }, 20:05:31 DEBUG "value": [ 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG [...] 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:05:31 DEBUG }, 20:05:31 DEBUG "value": [ 20:05:31 DEBUG 1713553531, 20:05:31 DEBUG "2500.6" 20:05:31 DEBUG ] 20:05:31 DEBUG } 20:05:31 DEBUG ] 20:05:31 DEBUG } 20:05:31 DEBUG } 20:05:31 INFO [http_cmd]: http status code OK 20:05:31 DEBUG --- status code --- 20:05:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:31 DEBUG --- http response --- 20:05:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:05:31 DEBUG ----- output ----- 20:05:31 DEBUG { 20:05:31 DEBUG "status": "success", 20:05:31 DEBUG "data": { 20:05:31 DEBUG "resultType": "vector", 20:05:31 DEBUG "result": [ 20:05:31 DEBUG { 20:05:31 DEBUG "metric": { 20:05:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:05:31 DEBUG }, 20:05:31 DEBUG "value": [ 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG [...] 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:05:31 DEBUG }, 20:05:31 DEBUG "value": [ 20:05:31 DEBUG 1713553531, 20:05:31 DEBUG "0.015" 20:05:31 DEBUG ] 20:05:31 DEBUG } 20:05:31 DEBUG ] 20:05:31 DEBUG } 20:05:31 DEBUG } 20:05:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553531" 20:05:31 INFO [http_cmd]: http status code OK 20:05:31 DEBUG --- status code --- 20:05:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:31 DEBUG --- http response --- 20:05:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:05:31 DEBUG ----- output ----- 20:05:31 DEBUG { 20:05:31 DEBUG "status": "success", 20:05:31 DEBUG "data": { 20:05:31 DEBUG "resultType": "vector", 20:05:31 DEBUG "result": [ 20:05:31 DEBUG { 20:05:31 DEBUG "metric": { 20:05:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:05:31 DEBUG }, 20:05:31 DEBUG "value": [ 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG [...] 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:05:31 DEBUG }, 20:05:31 DEBUG "value": [ 20:05:31 DEBUG 1713553531, 20:05:31 DEBUG "2498.8" 20:05:31 DEBUG ] 20:05:31 DEBUG } 20:05:31 DEBUG ] 20:05:31 DEBUG } 20:05:31 DEBUG } 20:05:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553531" 20:05:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553531" 20:05:31 INFO [http_cmd]: http status code OK 20:05:31 DEBUG --- status code --- 20:05:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:31 DEBUG --- http response --- 20:05:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:05:31 DEBUG ----- output ----- 20:05:31 DEBUG { 20:05:31 DEBUG "status": "success", 20:05:31 DEBUG "data": { 20:05:31 DEBUG "resultType": "vector", 20:05:31 DEBUG "result": [ 20:05:31 DEBUG { 20:05:31 DEBUG "metric": { 20:05:31 DEBUG "instance": "10.142.0.107:9100" 20:05:31 DEBUG }, 20:05:31 DEBUG "value": [ 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG [...] 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG "instance": "10.142.0.43:9100" 20:05:31 DEBUG }, 20:05:31 DEBUG "value": [ 20:05:31 DEBUG 1713553531, 20:05:31 DEBUG "2017723405312" 20:05:31 DEBUG ] 20:05:31 DEBUG } 20:05:31 DEBUG ] 20:05:31 DEBUG } 20:05:31 DEBUG } 20:05:31 INFO [http_cmd]: http status code OK 20:05:31 DEBUG --- status code --- 20:05:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:31 DEBUG --- http response --- 20:05:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 20:05:31 DEBUG ----- output ----- 20:05:31 DEBUG { 20:05:31 DEBUG "status": "success", 20:05:31 DEBUG "data": { 20:05:31 DEBUG "resultType": "vector", 20:05:31 DEBUG "result": [ 20:05:31 DEBUG { 20:05:31 DEBUG "metric": { 20:05:31 DEBUG "pod": "ds-idrepo-0" 20:05:31 DEBUG }, 20:05:31 DEBUG "value": [ 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG [...] 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG "pod": "ds-idrepo-1" 20:05:31 DEBUG }, 20:05:31 DEBUG "value": [ 20:05:31 DEBUG 1713553531, 20:05:31 DEBUG "4101.848776065275" 20:05:31 DEBUG ] 20:05:31 DEBUG } 20:05:31 DEBUG ] 20:05:31 DEBUG } 20:05:31 DEBUG } 20:05:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553531" 20:05:31 INFO [http_cmd]: http status code OK 20:05:31 DEBUG --- status code --- 20:05:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:31 DEBUG --- http response --- 20:05:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:05:31 DEBUG ----- output ----- 20:05:31 DEBUG { 20:05:31 DEBUG "status": "success", 20:05:31 DEBUG "data": { 20:05:31 DEBUG "resultType": "vector", 20:05:31 DEBUG "result": [ 20:05:31 DEBUG { 20:05:31 DEBUG "metric": { 20:05:31 DEBUG "pod": "ds-idrepo-0" 20:05:31 DEBUG }, 20:05:31 DEBUG "value": [ 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG [...] 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG "pod": "ds-idrepo-2" 20:05:31 DEBUG }, 20:05:31 DEBUG "value": [ 20:05:31 DEBUG 1713553531, 20:05:31 DEBUG "0" 20:05:31 DEBUG ] 20:05:31 DEBUG } 20:05:31 DEBUG ] 20:05:31 DEBUG } 20:05:31 DEBUG } 20:05:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553531" 20:05:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553531" 20:05:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713553531" 20:05:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713553531" 20:05:31 INFO [http_cmd]: http status code OK 20:05:31 DEBUG --- status code --- 20:05:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:31 DEBUG --- http response --- 20:05:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:05:31 DEBUG ----- output ----- 20:05:31 DEBUG { 20:05:31 DEBUG "status": "success", 20:05:31 DEBUG "data": { 20:05:31 DEBUG "resultType": "vector", 20:05:31 DEBUG "result": [ 20:05:31 DEBUG { 20:05:31 DEBUG "metric": { 20:05:31 DEBUG "pod": "ds-cts-0" 20:05:31 DEBUG }, 20:05:31 DEBUG "value": [ 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG [...] 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG "pod": "ds-cts-2" 20:05:31 DEBUG }, 20:05:31 DEBUG "value": [ 20:05:31 DEBUG 1713553531, 20:05:31 DEBUG "30" 20:05:31 DEBUG ] 20:05:31 DEBUG } 20:05:31 DEBUG ] 20:05:31 DEBUG } 20:05:31 DEBUG } 20:05:31 INFO [http_cmd]: http status code OK 20:05:31 DEBUG --- status code --- 20:05:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:31 DEBUG --- http response --- 20:05:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:05:31 DEBUG ----- output ----- 20:05:31 DEBUG { 20:05:31 DEBUG "status": "success", 20:05:31 DEBUG "data": { 20:05:31 DEBUG "resultType": "vector", 20:05:31 DEBUG "result": [ 20:05:31 DEBUG { 20:05:31 DEBUG "metric": { 20:05:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:05:31 DEBUG }, 20:05:31 DEBUG "value": [ 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG [...] 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:05:31 DEBUG }, 20:05:31 DEBUG "value": [ 20:05:31 DEBUG 1713553531, 20:05:31 DEBUG "2499.6" 20:05:31 DEBUG ] 20:05:31 DEBUG } 20:05:31 DEBUG ] 20:05:31 DEBUG } 20:05:31 DEBUG } 20:05:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553531" 20:05:31 INFO [http_cmd]: http status code OK 20:05:31 INFO [http_cmd]: http status code OK 20:05:31 DEBUG --- status code --- 20:05:31 DEBUG --- status code --- 20:05:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:31 DEBUG --- http response --- 20:05:31 DEBUG --- http response --- 20:05:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:05:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:05:31 DEBUG ----- output ----- 20:05:31 DEBUG ----- output ----- 20:05:31 DEBUG { 20:05:31 DEBUG { 20:05:31 DEBUG "status": "success", 20:05:31 DEBUG "status": "success", 20:05:31 DEBUG "data": { 20:05:31 DEBUG "data": { 20:05:31 DEBUG "resultType": "vector", 20:05:31 DEBUG "resultType": "vector", 20:05:31 DEBUG "result": [ 20:05:31 DEBUG "result": [ 20:05:31 DEBUG { 20:05:31 DEBUG { 20:05:31 DEBUG "metric": { 20:05:31 DEBUG "metric": { 20:05:31 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:05:31 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:05:31 DEBUG }, 20:05:31 DEBUG }, 20:05:31 DEBUG "value": [ 20:05:31 DEBUG "value": [ 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG [...] 20:05:31 DEBUG [...] 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553531" 20:05:31 DEBUG 20:05:31 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:05:31 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:05:31 DEBUG }, 20:05:31 DEBUG }, 20:05:31 DEBUG "value": [ 20:05:31 DEBUG "value": [ 20:05:31 DEBUG 1713553531, 20:05:31 DEBUG 1713553531, 20:05:31 DEBUG "8.041144532903584" 20:05:31 DEBUG "3665592320" 20:05:31 DEBUG ] 20:05:31 DEBUG ] 20:05:31 DEBUG } 20:05:31 DEBUG } 20:05:31 DEBUG ] 20:05:31 DEBUG ] 20:05:31 DEBUG } 20:05:31 DEBUG } 20:05:31 DEBUG } 20:05:31 DEBUG } 20:05:31 INFO [http_cmd]: http status code OK 20:05:31 DEBUG --- status code --- 20:05:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:31 DEBUG --- http response --- 20:05:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:05:31 DEBUG ----- output ----- 20:05:31 DEBUG { 20:05:31 DEBUG "status": "success", 20:05:31 DEBUG "data": { 20:05:31 DEBUG "resultType": "vector", 20:05:31 DEBUG "result": [ 20:05:31 DEBUG { 20:05:31 DEBUG "metric": { 20:05:31 DEBUG "instance": "10.142.0.107:9100" 20:05:31 DEBUG }, 20:05:31 DEBUG "value": [ 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG [...] 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG "instance": "10.142.0.43:9100" 20:05:31 DEBUG }, 20:05:31 DEBUG "value": [ 20:05:31 DEBUG 1713553531, 20:05:31 DEBUG "75623020.56666666" 20:05:31 DEBUG ] 20:05:31 DEBUG } 20:05:31 DEBUG ] 20:05:31 DEBUG } 20:05:31 DEBUG } 20:05:31 INFO [http_cmd]: http status code OK 20:05:31 DEBUG --- status code --- 20:05:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:31 DEBUG --- http response --- 20:05:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:05:31 DEBUG ----- output ----- 20:05:31 DEBUG { 20:05:31 DEBUG "status": "success", 20:05:31 DEBUG "data": { 20:05:31 DEBUG "resultType": "vector", 20:05:31 DEBUG "result": [ 20:05:31 DEBUG { 20:05:31 DEBUG "metric": { 20:05:31 DEBUG "instance": "10.142.0.107:9100" 20:05:31 DEBUG }, 20:05:31 DEBUG "value": [ 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG [...] 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG "instance": "10.142.0.43:9100" 20:05:31 DEBUG }, 20:05:31 DEBUG "value": [ 20:05:31 DEBUG 1713553531, 20:05:31 DEBUG "75624253.96666667" 20:05:31 DEBUG ] 20:05:31 DEBUG } 20:05:31 DEBUG ] 20:05:31 DEBUG } 20:05:31 DEBUG } 20:05:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553531" 20:05:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553531" 20:05:31 INFO [http_cmd]: http status code OK 20:05:31 DEBUG --- status code --- 20:05:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:31 DEBUG --- http response --- 20:05:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:05:31 DEBUG ----- output ----- 20:05:31 DEBUG { 20:05:31 DEBUG "status": "success", 20:05:31 DEBUG "data": { 20:05:31 DEBUG "resultType": "vector", 20:05:31 DEBUG "result": [ 20:05:31 DEBUG { 20:05:31 DEBUG "metric": { 20:05:31 DEBUG "pod": "ds-idrepo-0" 20:05:31 DEBUG }, 20:05:31 DEBUG "value": [ 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG [...] 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG "pod": "idm-b85f9d978-wnhxb" 20:05:31 DEBUG }, 20:05:31 DEBUG "value": [ 20:05:31 DEBUG 1713553531, 20:05:31 DEBUG "0" 20:05:31 DEBUG ] 20:05:31 DEBUG } 20:05:31 DEBUG ] 20:05:31 DEBUG } 20:05:31 DEBUG } 20:05:31 INFO [http_cmd]: http status code OK 20:05:31 DEBUG --- status code --- 20:05:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:31 DEBUG --- http response --- 20:05:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:05:31 DEBUG ----- output ----- 20:05:31 DEBUG { 20:05:31 DEBUG "status": "success", 20:05:31 DEBUG "data": { 20:05:31 DEBUG "resultType": "vector", 20:05:31 DEBUG "result": [ 20:05:31 DEBUG { 20:05:31 DEBUG "metric": { 20:05:31 DEBUG "pod": "ds-cts-0" 20:05:31 DEBUG }, 20:05:31 DEBUG "value": [ 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG [...] 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG "pod": "ds-cts-2" 20:05:31 DEBUG }, 20:05:31 DEBUG "value": [ 20:05:31 DEBUG 1713553531, 20:05:31 DEBUG "0" 20:05:31 DEBUG ] 20:05:31 DEBUG } 20:05:31 DEBUG ] 20:05:31 DEBUG } 20:05:31 DEBUG } 20:05:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553531" 20:05:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553531" 20:05:31 INFO [http_cmd]: http status code OK 20:05:31 INFO [http_cmd]: http status code OK 20:05:31 DEBUG --- status code --- 20:05:31 DEBUG --- status code --- 20:05:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:31 DEBUG --- http response --- 20:05:31 DEBUG --- http response --- 20:05:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:05:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 20:05:31 DEBUG ----- output ----- 20:05:31 DEBUG ----- output ----- 20:05:31 DEBUG { 20:05:31 DEBUG { 20:05:31 DEBUG "status": "success", 20:05:31 DEBUG "status": "success", 20:05:31 DEBUG "data": { 20:05:31 DEBUG "data": { 20:05:31 DEBUG "resultType": "vector", 20:05:31 DEBUG "resultType": "vector", 20:05:31 DEBUG "result": [ 20:05:31 DEBUG "result": [ 20:05:31 DEBUG { 20:05:31 DEBUG { 20:05:31 DEBUG "metric": { 20:05:31 DEBUG "metric": { 20:05:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:05:31 DEBUG "pod": "ds-idrepo-0" 20:05:31 DEBUG }, 20:05:31 DEBUG }, 20:05:31 DEBUG "value": [ 20:05:31 DEBUG "value": [ 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG [...] 20:05:31 DEBUG [...] 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG 20:05:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:05:31 DEBUG "pod": "ds-idrepo-1" 20:05:31 DEBUG }, 20:05:31 DEBUG }, 20:05:31 DEBUG "value": [ 20:05:31 DEBUG "value": [ 20:05:31 DEBUG 1713553531, 20:05:31 DEBUG 1713553531, 20:05:31 DEBUG "2499.9666666666667" 20:05:31 DEBUG "0" 20:05:31 DEBUG ] 20:05:31 DEBUG ] 20:05:31 DEBUG } 20:05:31 DEBUG } 20:05:31 DEBUG ] 20:05:31 DEBUG ] 20:05:31 DEBUG } 20:05:31 DEBUG } 20:05:31 DEBUG } 20:05:31 DEBUG } 20:05:33 INFO 20:05:33 INFO [loop_until]: kubectl --namespace=xlou top pods 20:05:33 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:05:33 INFO [loop_until]: OK (rc = 0) 20:05:33 DEBUG --- stdout --- 20:05:33 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 16227m 9004Mi am-767f6b67b5-qcqz2 16653m 8813Mi am-767f6b67b5-vbltz 17969m 9001Mi ds-cts-0 17648m 10965Mi ds-cts-1 15916m 11218Mi ds-cts-2 17684m 10985Mi ds-idrepo-0 7112m 3686Mi ds-idrepo-1 7015m 3537Mi ds-idrepo-2 7474m 3492Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 4m 1217Mi lodemon-f8459c597-6tj6f 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7949m 3405Mi 20:05:33 DEBUG --- stderr --- 20:05:33 DEBUG 20:05:34 INFO 20:05:34 INFO [loop_until]: kubectl --namespace=xlou top node 20:05:34 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:05:34 INFO [loop_until]: OK (rc = 0) 20:05:34 DEBUG --- stdout --- 20:05:34 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 69m 0% 6244Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7393m 24% 7288Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16147m 54% 11719Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 85m 0% 3349Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 17470m 58% 13942Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 164m 0% 5205Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 16594m 55% 11691Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15717m 52% 2992Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 18207m 60% 11214Mi 10% gke-xlou-cdm-ds-0f601751-j488 7630m 25% 6738Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 15777m 52% 15967Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 7483m 25% 7431Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 17397m 58% 13418Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 8195m 51% 7135Mi 12% 20:05:34 DEBUG --- stderr --- 20:05:34 DEBUG 20:06:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713553591" 20:06:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553591" 20:06:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553591" 20:06:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553591" 20:06:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553591" 20:06:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553591" 20:06:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553591" 20:06:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553591" 20:06:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553591" 20:06:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553591" 20:06:31 ERROR [http_cmd]: ERROR 20:06:31 ERROR --- status code --- 20:06:31 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:06:31 ERROR --- http response --- 20:06:31 INFO [http_cmd]: http status code OK 20:06:31 INFO [http_cmd]: http status code OK 20:06:31 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:06:31 DEBUG --- status code --- 20:06:31 DEBUG --- status code --- 20:06:31 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:06:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:31 DEBUG --- http response --- 20:06:31 DEBUG --- http response --- 20:06:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:06:31 DEBUG ----- output ----- 20:06:31 DEBUG { 20:06:31 DEBUG "status": "success", 20:06:31 DEBUG "data": { 20:06:31 DEBUG "resultType": "vector", 20:06:31 DEBUG "result": [ 20:06:31 DEBUG { 20:06:31 DEBUG "metric": { 20:06:31 DEBUG "pod": "ds-idrepo-0" 20:06:31 DEBUG }, 20:06:31 DEBUG "value": [ 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG [...] 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG "pod": "ds-idrepo-2" 20:06:31 DEBUG }, 20:06:31 DEBUG "value": [ 20:06:31 DEBUG 1713553591, 20:06:31 DEBUG "0" 20:06:31 DEBUG ] 20:06:31 DEBUG } 20:06:31 DEBUG ] 20:06:31 DEBUG } 20:06:31 DEBUG } 20:06:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:06:31 DEBUG ----- output ----- 20:06:31 DEBUG { 20:06:31 DEBUG "status": "success", 20:06:31 DEBUG "data": { 20:06:31 DEBUG "resultType": "vector", 20:06:31 DEBUG "result": [ 20:06:31 DEBUG { 20:06:31 DEBUG "metric": { 20:06:31 DEBUG "pod": "ds-idrepo-0" 20:06:31 DEBUG }, 20:06:31 DEBUG "value": [ 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG [...] 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG "pod": "ds-idrepo-2" 20:06:31 DEBUG }, 20:06:31 DEBUG "value": [ 20:06:31 DEBUG 1713553591, 20:06:31 DEBUG "0" 20:06:31 DEBUG ] 20:06:31 DEBUG } 20:06:31 DEBUG ] 20:06:31 DEBUG } 20:06:31 DEBUG } 20:06:31 INFO [http_cmd]: http status code OK 20:06:31 DEBUG --- status code --- 20:06:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:31 DEBUG --- http response --- 20:06:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:06:31 DEBUG ----- output ----- 20:06:31 DEBUG { 20:06:31 DEBUG "status": "success", 20:06:31 DEBUG "data": { 20:06:31 DEBUG "resultType": "vector", 20:06:31 DEBUG "result": [ 20:06:31 DEBUG { 20:06:31 DEBUG "metric": { 20:06:31 DEBUG "instance": "10.142.0.107:9100" 20:06:31 DEBUG }, 20:06:31 DEBUG "value": [ 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG [...] 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG "instance": "10.142.0.43:9100" 20:06:31 DEBUG }, 20:06:31 DEBUG "value": [ 20:06:31 DEBUG 1713553591, 20:06:31 DEBUG "0.27433333333325055" 20:06:31 DEBUG ] 20:06:31 DEBUG } 20:06:31 DEBUG ] 20:06:31 DEBUG } 20:06:31 DEBUG } 20:06:31 INFO [http_cmd]: http status code OK 20:06:31 DEBUG --- status code --- 20:06:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:31 DEBUG --- http response --- 20:06:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:06:31 DEBUG ----- output ----- 20:06:31 DEBUG { 20:06:31 DEBUG "status": "success", 20:06:31 DEBUG "data": { 20:06:31 DEBUG "resultType": "vector", 20:06:31 DEBUG "result": [ 20:06:31 DEBUG { 20:06:31 DEBUG "metric": { 20:06:31 DEBUG "instance": "10.142.0.107:9100" 20:06:31 DEBUG }, 20:06:31 DEBUG "value": [ 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG [...] 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG "instance": "10.142.0.43:9100" 20:06:31 DEBUG }, 20:06:31 DEBUG "value": [ 20:06:31 DEBUG 1713553591, 20:06:31 DEBUG "0.11366666666666561" 20:06:31 DEBUG ] 20:06:31 DEBUG } 20:06:31 DEBUG ] 20:06:31 DEBUG } 20:06:31 DEBUG } 20:06:31 INFO [http_cmd]: http status code OK 20:06:31 DEBUG --- status code --- 20:06:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:31 DEBUG --- http response --- 20:06:31 INFO [http_cmd]: http status code OK 20:06:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:06:31 INFO [http_cmd]: http status code OK 20:06:31 INFO [http_cmd]: http status code OK 20:06:31 DEBUG --- status code --- 20:06:31 INFO [http_cmd]: http status code OK 20:06:31 DEBUG ----- output ----- 20:06:31 DEBUG --- status code --- 20:06:31 DEBUG --- status code --- 20:06:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:31 DEBUG --- status code --- 20:06:31 DEBUG { 20:06:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:31 DEBUG --- http response --- 20:06:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:31 DEBUG "status": "success", 20:06:31 DEBUG --- http response --- 20:06:31 DEBUG --- http response --- 20:06:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:06:31 DEBUG --- http response --- 20:06:31 DEBUG "data": { 20:06:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:06:31 DEBUG ----- output ----- 20:06:31 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:06:31 DEBUG "resultType": "vector", 20:06:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:06:31 DEBUG ----- output ----- 20:06:31 DEBUG { 20:06:31 DEBUG "result": [ 20:06:31 DEBUG ----- output ----- 20:06:31 DEBUG { 20:06:31 DEBUG "status": "success", 20:06:31 DEBUG { 20:06:31 DEBUG { 20:06:31 DEBUG "status": "success", 20:06:31 DEBUG "data": { 20:06:31 DEBUG "metric": { 20:06:31 DEBUG "status": "success", 20:06:31 DEBUG "data": { 20:06:31 DEBUG "resultType": "vector", 20:06:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:06:31 DEBUG "data": { 20:06:31 DEBUG "resultType": "vector", 20:06:31 DEBUG "result": [ 20:06:31 DEBUG }, 20:06:31 DEBUG "resultType": "vector", 20:06:31 DEBUG "result": [ 20:06:31 DEBUG { 20:06:31 DEBUG "value": [ 20:06:31 DEBUG "result": [ 20:06:31 DEBUG { 20:06:31 DEBUG "metric": { 20:06:31 DEBUG 20:06:31 DEBUG { 20:06:31 DEBUG "metric": { 20:06:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:06:31 DEBUG 20:06:31 DEBUG "metric": { 20:06:31 DEBUG "pod": "ds-cts-0" 20:06:31 DEBUG }, 20:06:31 DEBUG 20:06:31 DEBUG "pod": "ds-idrepo-0" 20:06:31 DEBUG }, 20:06:31 DEBUG "value": [ 20:06:31 DEBUG [...] 20:06:31 DEBUG }, 20:06:31 DEBUG "value": [ 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG "value": [ 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG [...] 20:06:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:06:31 DEBUG 20:06:31 DEBUG [...] 20:06:31 DEBUG 20:06:31 DEBUG }, 20:06:31 DEBUG [...] 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG "value": [ 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 1713553591, 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG "449915.5" 20:06:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:06:31 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 20:06:31 DEBUG "pod": "ds-cts-2" 20:06:31 DEBUG ] 20:06:31 DEBUG }, 20:06:31 DEBUG }, 20:06:31 DEBUG }, 20:06:31 DEBUG } 20:06:31 DEBUG "value": [ 20:06:31 DEBUG "value": [ 20:06:31 DEBUG "value": [ 20:06:31 DEBUG ] 20:06:31 DEBUG 1713553591, 20:06:31 DEBUG 1713553591, 20:06:31 DEBUG 1713553591, 20:06:31 DEBUG } 20:06:31 DEBUG "0" 20:06:31 DEBUG "0" 20:06:31 DEBUG "0.0005495133333333334" 20:06:31 DEBUG } 20:06:31 DEBUG ] 20:06:31 DEBUG ] 20:06:31 DEBUG ] 20:06:31 DEBUG } 20:06:31 DEBUG } 20:06:31 DEBUG } 20:06:31 DEBUG ] 20:06:31 DEBUG ] 20:06:31 DEBUG ] 20:06:31 DEBUG } 20:06:31 DEBUG } 20:06:31 DEBUG } 20:06:31 DEBUG } 20:06:31 DEBUG } 20:06:31 DEBUG } 20:06:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553591" 20:06:31 INFO [http_cmd]: http status code OK 20:06:31 DEBUG --- status code --- 20:06:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:31 DEBUG --- http response --- 20:06:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:06:31 DEBUG ----- output ----- 20:06:31 DEBUG { 20:06:31 DEBUG "status": "success", 20:06:31 DEBUG "data": { 20:06:31 DEBUG "resultType": "vector", 20:06:31 DEBUG "result": [ 20:06:31 DEBUG { 20:06:31 DEBUG "metric": { 20:06:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:06:31 DEBUG }, 20:06:31 DEBUG "value": [ 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG [...] 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:06:31 DEBUG }, 20:06:31 DEBUG "value": [ 20:06:31 DEBUG 1713553591, 20:06:31 DEBUG "2499.9" 20:06:31 DEBUG ] 20:06:31 DEBUG } 20:06:31 DEBUG ] 20:06:31 DEBUG } 20:06:31 DEBUG } 20:06:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713553591" 20:06:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553591" 20:06:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713553591" 20:06:31 INFO [http_cmd]: http status code OK 20:06:31 DEBUG --- status code --- 20:06:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:31 DEBUG --- http response --- 20:06:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:06:31 DEBUG ----- output ----- 20:06:31 DEBUG { 20:06:31 DEBUG "status": "success", 20:06:31 DEBUG "data": { 20:06:31 DEBUG "resultType": "vector", 20:06:31 DEBUG "result": [ 20:06:31 DEBUG { 20:06:31 DEBUG "metric": { 20:06:31 DEBUG "pod": "ds-idrepo-0" 20:06:31 DEBUG }, 20:06:31 DEBUG "value": [ 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG [...] 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG "pod": "ds-cts-2" 20:06:31 DEBUG }, 20:06:31 DEBUG "value": [ 20:06:31 DEBUG 1713553591, 20:06:31 DEBUG "0" 20:06:31 DEBUG ] 20:06:31 DEBUG } 20:06:31 DEBUG ] 20:06:31 DEBUG } 20:06:31 DEBUG } 20:06:31 INFO [http_cmd]: http status code OK 20:06:31 INFO [http_cmd]: http status code OK 20:06:31 DEBUG --- status code --- 20:06:31 DEBUG --- status code --- 20:06:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:31 DEBUG --- http response --- 20:06:31 DEBUG --- http response --- 20:06:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:06:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:06:31 DEBUG ----- output ----- 20:06:31 DEBUG ----- output ----- 20:06:31 DEBUG { 20:06:31 DEBUG { 20:06:31 DEBUG "status": "success", 20:06:31 DEBUG "status": "success", 20:06:31 DEBUG "data": { 20:06:31 DEBUG "data": { 20:06:31 DEBUG "resultType": "vector", 20:06:31 DEBUG "resultType": "vector", 20:06:31 DEBUG "result": [ 20:06:31 DEBUG "result": [ 20:06:31 DEBUG { 20:06:31 DEBUG { 20:06:31 DEBUG "metric": { 20:06:31 DEBUG "metric": { 20:06:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:06:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:06:31 DEBUG }, 20:06:31 DEBUG }, 20:06:31 DEBUG "value": [ 20:06:31 DEBUG "value": [ 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG [...] 20:06:31 DEBUG [...] 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:06:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:06:31 DEBUG }, 20:06:31 DEBUG }, 20:06:31 DEBUG "value": [ 20:06:31 DEBUG "value": [ 20:06:31 DEBUG 1713553591, 20:06:31 DEBUG 1713553591, 20:06:31 DEBUG "0.015" 20:06:31 DEBUG "2490.5333333333333" 20:06:31 DEBUG ] 20:06:31 DEBUG ] 20:06:31 DEBUG } 20:06:31 DEBUG } 20:06:31 DEBUG ] 20:06:31 DEBUG ] 20:06:31 DEBUG } 20:06:31 DEBUG } 20:06:31 DEBUG } 20:06:31 DEBUG } 20:06:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553591" 20:06:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713553591" 20:06:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553591" 20:06:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553591" 20:06:31 INFO [http_cmd]: http status code OK 20:06:31 DEBUG --- status code --- 20:06:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:31 DEBUG --- http response --- 20:06:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:06:31 DEBUG ----- output ----- 20:06:31 DEBUG { 20:06:31 DEBUG "status": "success", 20:06:31 DEBUG "data": { 20:06:31 DEBUG "resultType": "vector", 20:06:31 DEBUG "result": [ 20:06:31 DEBUG { 20:06:31 DEBUG "metric": { 20:06:31 DEBUG "pod": "ds-idrepo-0" 20:06:31 DEBUG }, 20:06:31 DEBUG "value": [ 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG [...] 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG "pod": "ds-idrepo-2" 20:06:31 DEBUG }, 20:06:31 DEBUG "value": [ 20:06:31 DEBUG 1713553591, 20:06:31 DEBUG "0" 20:06:31 DEBUG ] 20:06:31 DEBUG } 20:06:31 DEBUG ] 20:06:31 DEBUG } 20:06:31 DEBUG } 20:06:31 INFO [http_cmd]: http status code OK 20:06:31 DEBUG --- status code --- 20:06:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:31 DEBUG --- http response --- 20:06:31 INFO [http_cmd]: http status code OK 20:06:31 DEBUG --- status code --- 20:06:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:31 DEBUG --- http response --- 20:06:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:06:31 DEBUG ----- output ----- 20:06:31 DEBUG { 20:06:31 DEBUG "status": "success", 20:06:31 DEBUG "data": { 20:06:31 DEBUG "resultType": "vector", 20:06:31 DEBUG "result": [ 20:06:31 DEBUG { 20:06:31 DEBUG "metric": { 20:06:31 DEBUG "instance": "10.142.0.107:9100" 20:06:31 DEBUG }, 20:06:31 DEBUG "value": [ 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG [...] 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG "instance": "10.142.0.43:9100" 20:06:31 DEBUG }, 20:06:31 DEBUG "value": [ 20:06:31 DEBUG 1713553591, 20:06:31 DEBUG "2021908640768" 20:06:31 DEBUG ] 20:06:31 DEBUG } 20:06:31 DEBUG ] 20:06:31 DEBUG } 20:06:31 DEBUG } 20:06:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:06:31 DEBUG ----- output ----- 20:06:31 DEBUG { 20:06:31 DEBUG "status": "success", 20:06:31 DEBUG "data": { 20:06:31 DEBUG "resultType": "vector", 20:06:31 DEBUG "result": [ 20:06:31 DEBUG { 20:06:31 DEBUG "metric": { 20:06:31 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:06:31 DEBUG }, 20:06:31 DEBUG "value": [ 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG [...] 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:06:31 DEBUG }, 20:06:31 DEBUG "value": [ 20:06:31 DEBUG 1713553591, 20:06:31 DEBUG "3668299776" 20:06:31 DEBUG ] 20:06:31 DEBUG } 20:06:31 DEBUG ] 20:06:31 DEBUG } 20:06:31 DEBUG } 20:06:31 INFO [http_cmd]: http status code OK 20:06:31 DEBUG --- status code --- 20:06:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:31 DEBUG --- http response --- 20:06:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:06:31 DEBUG ----- output ----- 20:06:31 DEBUG { 20:06:31 DEBUG "status": "success", 20:06:31 DEBUG "data": { 20:06:31 DEBUG "resultType": "vector", 20:06:31 DEBUG "result": [ 20:06:31 DEBUG { 20:06:31 DEBUG "metric": { 20:06:31 DEBUG "pod": "ds-idrepo-0" 20:06:31 DEBUG }, 20:06:31 DEBUG "value": [ 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG [...] 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 20:06:31 DEBUG }, 20:06:31 DEBUG "value": [ 20:06:31 DEBUG 1713553591, 20:06:31 DEBUG "0" 20:06:31 DEBUG ] 20:06:31 DEBUG } 20:06:31 DEBUG ] 20:06:31 DEBUG } 20:06:31 DEBUG } 20:06:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553591" 20:06:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553591" 20:06:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553591" 20:06:31 INFO [http_cmd]: http status code OK 20:06:31 DEBUG --- status code --- 20:06:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:31 DEBUG --- http response --- 20:06:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:06:31 INFO [http_cmd]: http status code OK 20:06:31 DEBUG ----- output ----- 20:06:31 DEBUG --- status code --- 20:06:31 DEBUG { 20:06:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:31 DEBUG "status": "success", 20:06:31 DEBUG --- http response --- 20:06:31 DEBUG "data": { 20:06:31 DEBUG "resultType": "vector", 20:06:31 DEBUG "result": [ 20:06:31 DEBUG { 20:06:31 DEBUG "metric": { 20:06:31 DEBUG "pod": "ds-cts-0" 20:06:31 DEBUG }, 20:06:31 DEBUG "value": [ 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG [...] 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG "pod": "ds-cts-2" 20:06:31 DEBUG }, 20:06:31 DEBUG "value": [ 20:06:31 DEBUG 1713553591, 20:06:31 DEBUG "0" 20:06:31 DEBUG ] 20:06:31 DEBUG } 20:06:31 DEBUG ] 20:06:31 DEBUG } 20:06:31 DEBUG } 20:06:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:06:31 DEBUG ----- output ----- 20:06:31 DEBUG { 20:06:31 DEBUG "status": "success", 20:06:31 DEBUG "data": { 20:06:31 DEBUG "resultType": "vector", 20:06:31 DEBUG "result": [ 20:06:31 DEBUG { 20:06:31 DEBUG "metric": { 20:06:31 DEBUG "instance": "10.142.0.107:9100" 20:06:31 DEBUG }, 20:06:31 DEBUG "value": [ 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG [...] 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG "instance": "10.142.0.43:9100" 20:06:31 DEBUG }, 20:06:31 DEBUG "value": [ 20:06:31 DEBUG 1713553591, 20:06:31 DEBUG "75395931.36666667" 20:06:31 DEBUG ] 20:06:31 DEBUG } 20:06:31 DEBUG ] 20:06:31 DEBUG } 20:06:31 DEBUG } 20:06:31 INFO [http_cmd]: http status code OK 20:06:31 DEBUG --- status code --- 20:06:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:31 DEBUG --- http response --- 20:06:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:06:31 DEBUG ----- output ----- 20:06:31 DEBUG { 20:06:31 DEBUG "status": "success", 20:06:31 DEBUG "data": { 20:06:31 DEBUG "resultType": "vector", 20:06:31 DEBUG "result": [ 20:06:31 DEBUG { 20:06:31 DEBUG "metric": { 20:06:31 DEBUG "pod": "ds-cts-0" 20:06:31 DEBUG }, 20:06:31 DEBUG "value": [ 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG [...] 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG "pod": "ds-cts-2" 20:06:31 DEBUG }, 20:06:31 DEBUG "value": [ 20:06:31 DEBUG 1713553591, 20:06:31 DEBUG "30" 20:06:31 DEBUG ] 20:06:31 DEBUG } 20:06:31 DEBUG ] 20:06:31 DEBUG } 20:06:31 DEBUG } 20:06:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553591" 20:06:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713553591" 20:06:31 INFO [http_cmd]: http status code OK 20:06:31 DEBUG --- status code --- 20:06:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:31 DEBUG --- http response --- 20:06:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:06:31 DEBUG ----- output ----- 20:06:31 DEBUG { 20:06:31 DEBUG "status": "success", 20:06:31 DEBUG "data": { 20:06:31 DEBUG "resultType": "vector", 20:06:31 DEBUG "result": [ 20:06:31 DEBUG { 20:06:31 DEBUG "metric": { 20:06:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:06:31 DEBUG }, 20:06:31 DEBUG "value": [ 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG [...] 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:06:31 DEBUG }, 20:06:31 DEBUG "value": [ 20:06:31 DEBUG 1713553591, 20:06:31 DEBUG "2498.5666666666666" 20:06:31 DEBUG ] 20:06:31 DEBUG } 20:06:31 DEBUG ] 20:06:31 DEBUG } 20:06:31 DEBUG } 20:06:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553591" 20:06:31 INFO [http_cmd]: http status code OK 20:06:31 DEBUG --- status code --- 20:06:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:31 DEBUG --- http response --- 20:06:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:06:31 DEBUG ----- output ----- 20:06:31 DEBUG { 20:06:31 DEBUG "status": "success", 20:06:31 DEBUG "data": { 20:06:31 DEBUG "resultType": "vector", 20:06:31 DEBUG "result": [ 20:06:31 DEBUG { 20:06:31 DEBUG "metric": { 20:06:31 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:06:31 DEBUG }, 20:06:31 DEBUG "value": [ 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG [...] 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:06:31 DEBUG }, 20:06:31 DEBUG "value": [ 20:06:31 DEBUG 1713553591, 20:06:31 DEBUG "8.050040169198548" 20:06:31 DEBUG ] 20:06:31 DEBUG } 20:06:31 DEBUG ] 20:06:31 DEBUG } 20:06:31 DEBUG } 20:06:31 INFO [http_cmd]: http status code OK 20:06:31 DEBUG --- status code --- 20:06:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:31 DEBUG --- http response --- 20:06:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:06:31 DEBUG ----- output ----- 20:06:31 DEBUG { 20:06:31 DEBUG "status": "success", 20:06:31 DEBUG "data": { 20:06:31 DEBUG "resultType": "vector", 20:06:31 DEBUG "result": [ 20:06:31 DEBUG { 20:06:31 DEBUG "metric": { 20:06:31 DEBUG "instance": "10.142.0.107:9100" 20:06:31 DEBUG }, 20:06:31 DEBUG "value": [ 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG [...] 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG "instance": "10.142.0.43:9100" 20:06:31 DEBUG }, 20:06:31 DEBUG "value": [ 20:06:31 DEBUG 1713553591, 20:06:31 DEBUG "75398218.96666667" 20:06:31 DEBUG ] 20:06:31 DEBUG } 20:06:31 DEBUG ] 20:06:31 DEBUG } 20:06:31 DEBUG } 20:06:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553591" 20:06:31 INFO [http_cmd]: http status code OK 20:06:31 DEBUG --- status code --- 20:06:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:31 DEBUG --- http response --- 20:06:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:06:31 DEBUG ----- output ----- 20:06:31 DEBUG { 20:06:31 DEBUG "status": "success", 20:06:31 DEBUG "data": { 20:06:31 DEBUG "resultType": "vector", 20:06:31 DEBUG "result": [ 20:06:31 DEBUG { 20:06:31 DEBUG "metric": { 20:06:31 DEBUG "pod": "ds-idrepo-0" 20:06:31 DEBUG }, 20:06:31 DEBUG "value": [ 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG [...] 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 20:06:31 DEBUG }, 20:06:31 DEBUG "value": [ 20:06:31 DEBUG 1713553591, 20:06:31 DEBUG "0.00003129422865229234" 20:06:31 DEBUG ] 20:06:31 DEBUG } 20:06:31 DEBUG ] 20:06:31 DEBUG } 20:06:31 DEBUG } 20:06:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553591" 20:06:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553591" 20:06:31 INFO [http_cmd]: http status code OK 20:06:31 DEBUG --- status code --- 20:06:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:31 DEBUG --- http response --- 20:06:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:06:31 DEBUG ----- output ----- 20:06:31 DEBUG { 20:06:31 DEBUG "status": "success", 20:06:31 DEBUG "data": { 20:06:31 DEBUG "resultType": "vector", 20:06:31 DEBUG "result": [ 20:06:31 DEBUG { 20:06:31 DEBUG "metric": { 20:06:31 DEBUG "pod": "ds-idrepo-0" 20:06:31 DEBUG }, 20:06:31 DEBUG "value": [ 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG [...] 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 20:06:31 DEBUG }, 20:06:31 DEBUG "value": [ 20:06:31 DEBUG 1713553591, 20:06:31 DEBUG "0" 20:06:31 DEBUG ] 20:06:31 DEBUG } 20:06:31 DEBUG ] 20:06:31 DEBUG } 20:06:31 DEBUG } 20:06:31 INFO [http_cmd]: http status code OK 20:06:31 DEBUG --- status code --- 20:06:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:31 DEBUG --- http response --- 20:06:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:06:31 DEBUG ----- output ----- 20:06:31 DEBUG { 20:06:31 DEBUG "status": "success", 20:06:31 DEBUG "data": { 20:06:31 DEBUG "resultType": "vector", 20:06:31 DEBUG "result": [ 20:06:31 DEBUG { 20:06:31 DEBUG "metric": { 20:06:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:06:31 DEBUG }, 20:06:31 DEBUG "value": [ 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG [...] 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG 20:06:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:06:31 DEBUG }, 20:06:31 DEBUG "value": [ 20:06:31 DEBUG 1713553591, 20:06:31 DEBUG "2500.366666666667" 20:06:31 DEBUG ] 20:06:31 DEBUG } 20:06:31 DEBUG ] 20:06:31 DEBUG } 20:06:31 DEBUG } 20:06:33 INFO 20:06:33 INFO [loop_until]: kubectl --namespace=xlou top pods 20:06:33 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:06:33 INFO [loop_until]: OK (rc = 0) 20:06:33 DEBUG --- stdout --- 20:06:33 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 16178m 9009Mi am-767f6b67b5-qcqz2 16632m 8812Mi am-767f6b67b5-vbltz 17523m 9000Mi ds-cts-0 17406m 10970Mi ds-cts-1 15668m 11217Mi ds-cts-2 18213m 10974Mi ds-idrepo-0 7144m 3675Mi ds-idrepo-1 7176m 3537Mi ds-idrepo-2 7442m 3492Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 1m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8034m 3406Mi 20:06:33 DEBUG --- stderr --- 20:06:33 DEBUG 20:06:34 INFO 20:06:34 INFO [loop_until]: kubectl --namespace=xlou top node 20:06:34 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:06:34 INFO [loop_until]: OK (rc = 0) 20:06:34 DEBUG --- stdout --- 20:06:34 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 69m 0% 6239Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7472m 25% 7286Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 15624m 52% 11718Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 86m 0% 3336Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 16771m 56% 13926Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 163m 0% 5186Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 16513m 55% 11696Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15402m 51% 2991Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 18257m 61% 11202Mi 10% gke-xlou-cdm-ds-0f601751-j488 7653m 25% 6746Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 15395m 51% 15969Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 7211m 24% 7438Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 17896m 59% 13417Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 8191m 51% 7131Mi 12% 20:06:34 DEBUG --- stderr --- 20:06:34 DEBUG 20:07:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553651" 20:07:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553651" 20:07:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553651" 20:07:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713553651" 20:07:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553651" 20:07:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553651" 20:07:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553651" 20:07:31 INFO [http_cmd]: http status code OK 20:07:31 DEBUG --- status code --- 20:07:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:31 DEBUG --- http response --- 20:07:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:07:31 DEBUG ----- output ----- 20:07:31 DEBUG { 20:07:31 DEBUG "status": "success", 20:07:31 DEBUG "data": { 20:07:31 DEBUG "resultType": "vector", 20:07:31 DEBUG "result": [ 20:07:31 DEBUG { 20:07:31 DEBUG "metric": { 20:07:31 DEBUG "pod": "ds-idrepo-0" 20:07:31 DEBUG }, 20:07:31 DEBUG "value": [ 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG [...] 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG "pod": "ds-idrepo-2" 20:07:31 DEBUG }, 20:07:31 DEBUG "value": [ 20:07:31 DEBUG 1713553651, 20:07:31 DEBUG "0" 20:07:31 DEBUG ] 20:07:31 DEBUG } 20:07:31 DEBUG ] 20:07:31 DEBUG } 20:07:31 DEBUG } 20:07:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553651" 20:07:31 ERROR [http_cmd]: ERROR 20:07:31 ERROR --- status code --- 20:07:31 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:07:31 ERROR --- http response --- 20:07:31 INFO [http_cmd]: http status code OK 20:07:31 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:07:31 DEBUG --- status code --- 20:07:31 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:07:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:31 DEBUG --- http response --- 20:07:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:07:31 DEBUG ----- output ----- 20:07:31 DEBUG { 20:07:31 DEBUG "status": "success", 20:07:31 DEBUG "data": { 20:07:31 DEBUG "resultType": "vector", 20:07:31 DEBUG "result": [ 20:07:31 DEBUG { 20:07:31 DEBUG "metric": { 20:07:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:07:31 DEBUG }, 20:07:31 DEBUG "value": [ 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG [...] 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:07:31 DEBUG }, 20:07:31 DEBUG "value": [ 20:07:31 DEBUG 1713553651, 20:07:31 DEBUG "450110.5" 20:07:31 DEBUG ] 20:07:31 DEBUG } 20:07:31 DEBUG ] 20:07:31 DEBUG } 20:07:31 DEBUG } 20:07:31 INFO [http_cmd]: http status code OK 20:07:31 DEBUG --- status code --- 20:07:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:31 DEBUG --- http response --- 20:07:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:07:31 DEBUG ----- output ----- 20:07:31 DEBUG { 20:07:31 DEBUG "status": "success", 20:07:31 DEBUG "data": { 20:07:31 DEBUG "resultType": "vector", 20:07:31 DEBUG "result": [ 20:07:31 DEBUG { 20:07:31 DEBUG "metric": { 20:07:31 DEBUG "instance": "10.142.0.107:9100" 20:07:31 DEBUG }, 20:07:31 DEBUG "value": [ 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG [...] 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG "instance": "10.142.0.43:9100" 20:07:31 DEBUG }, 20:07:31 DEBUG "value": [ 20:07:31 DEBUG 1713553651, 20:07:31 DEBUG "0.06466666666666471" 20:07:31 DEBUG ] 20:07:31 DEBUG } 20:07:31 DEBUG ] 20:07:31 DEBUG } 20:07:31 DEBUG } 20:07:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553651" 20:07:31 INFO [http_cmd]: http status code OK 20:07:31 INFO [http_cmd]: http status code OK 20:07:31 INFO [http_cmd]: http status code OK 20:07:31 DEBUG --- status code --- 20:07:31 DEBUG --- status code --- 20:07:31 DEBUG --- status code --- 20:07:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:31 DEBUG --- http response --- 20:07:31 DEBUG --- http response --- 20:07:31 DEBUG --- http response --- 20:07:31 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:07:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:07:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:07:31 DEBUG ----- output ----- 20:07:31 DEBUG ----- output ----- 20:07:31 DEBUG { 20:07:31 DEBUG { 20:07:31 DEBUG "status": "success", 20:07:31 DEBUG "status": "success", 20:07:31 DEBUG "data": { 20:07:31 DEBUG "data": { 20:07:31 DEBUG "resultType": "vector", 20:07:31 DEBUG "resultType": "vector", 20:07:31 DEBUG "result": [ 20:07:31 DEBUG "result": [ 20:07:31 INFO [http_cmd]: http status code OK 20:07:31 DEBUG { 20:07:31 DEBUG { 20:07:31 DEBUG --- status code --- 20:07:31 DEBUG "metric": { 20:07:31 DEBUG "metric": { 20:07:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:31 DEBUG "pod": "ds-idrepo-1" 20:07:31 DEBUG "instance": "10.142.0.107:9100" 20:07:31 DEBUG --- http response --- 20:07:31 DEBUG }, 20:07:31 DEBUG }, 20:07:31 DEBUG "value": [ 20:07:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:07:31 DEBUG "value": [ 20:07:31 DEBUG 20:07:31 DEBUG ----- output ----- 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG { 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG "status": "success", 20:07:31 DEBUG 20:07:31 DEBUG [...] 20:07:31 DEBUG "data": { 20:07:31 DEBUG [...] 20:07:31 DEBUG 20:07:31 DEBUG "resultType": "vector", 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG "result": [ 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG { 20:07:31 DEBUG 20:07:31 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:07:31 DEBUG "metric": { 20:07:31 DEBUG "instance": "10.142.0.43:9100" 20:07:31 DEBUG }, 20:07:31 DEBUG "pod": "ds-idrepo-0" 20:07:31 DEBUG }, 20:07:31 DEBUG "value": [ 20:07:31 DEBUG }, 20:07:31 DEBUG "value": [ 20:07:31 DEBUG 1713553651, 20:07:31 DEBUG "value": [ 20:07:31 DEBUG 1713553651, 20:07:31 DEBUG "0" 20:07:31 DEBUG 20:07:31 DEBUG "0.1664666666666856" 20:07:31 DEBUG ] 20:07:31 DEBUG 20:07:31 DEBUG ] 20:07:31 DEBUG } 20:07:31 DEBUG 20:07:31 DEBUG } 20:07:31 DEBUG ] 20:07:31 DEBUG [...] 20:07:31 DEBUG ] 20:07:31 DEBUG } 20:07:31 DEBUG 20:07:31 DEBUG } 20:07:31 DEBUG } 20:07:31 DEBUG 20:07:31 DEBUG } 20:07:31 DEBUG 20:07:31 DEBUG "pod": "ds-idrepo-2" 20:07:31 DEBUG }, 20:07:31 DEBUG "value": [ 20:07:31 DEBUG 1713553651, 20:07:31 DEBUG "0" 20:07:31 DEBUG ] 20:07:31 DEBUG } 20:07:31 DEBUG ] 20:07:31 DEBUG } 20:07:31 DEBUG } 20:07:31 INFO [http_cmd]: http status code OK 20:07:31 DEBUG --- status code --- 20:07:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:31 DEBUG --- http response --- 20:07:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:07:31 DEBUG ----- output ----- 20:07:31 DEBUG { 20:07:31 DEBUG "status": "success", 20:07:31 DEBUG "data": { 20:07:31 DEBUG "resultType": "vector", 20:07:31 DEBUG "result": [ 20:07:31 DEBUG { 20:07:31 DEBUG "metric": { 20:07:31 DEBUG "pod": "ds-cts-0" 20:07:31 DEBUG }, 20:07:31 DEBUG "value": [ 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG [...] 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG "pod": "ds-cts-2" 20:07:31 DEBUG }, 20:07:31 DEBUG "value": [ 20:07:31 DEBUG 1713553651, 20:07:31 DEBUG "0.00006666666666666672" 20:07:31 DEBUG ] 20:07:31 DEBUG } 20:07:31 DEBUG ] 20:07:31 DEBUG } 20:07:31 DEBUG } 20:07:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553651" 20:07:31 INFO [http_cmd]: http status code OK 20:07:31 DEBUG --- status code --- 20:07:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:31 DEBUG --- http response --- 20:07:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 20:07:31 DEBUG ----- output ----- 20:07:31 DEBUG { 20:07:31 DEBUG "status": "success", 20:07:31 DEBUG "data": { 20:07:31 DEBUG "resultType": "vector", 20:07:31 DEBUG "result": [ 20:07:31 DEBUG { 20:07:31 DEBUG "metric": { 20:07:31 DEBUG "pod": "am-767f6b67b5-qcqz2" 20:07:31 DEBUG }, 20:07:31 DEBUG "value": [ 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG [...] 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:07:31 DEBUG }, 20:07:31 DEBUG "value": [ 20:07:31 DEBUG 1713553651, 20:07:31 DEBUG "0" 20:07:31 DEBUG ] 20:07:31 DEBUG } 20:07:31 DEBUG ] 20:07:31 DEBUG } 20:07:31 DEBUG } 20:07:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553651" 20:07:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713553651" 20:07:31 INFO [http_cmd]: http status code OK 20:07:31 DEBUG --- status code --- 20:07:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:31 DEBUG --- http response --- 20:07:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 20:07:31 DEBUG ----- output ----- 20:07:31 DEBUG { 20:07:31 DEBUG "status": "success", 20:07:31 DEBUG "data": { 20:07:31 DEBUG "resultType": "vector", 20:07:31 DEBUG "result": [ 20:07:31 DEBUG { 20:07:31 DEBUG "metric": { 20:07:31 DEBUG "pod": "am-767f6b67b5-qcqz2" 20:07:31 DEBUG }, 20:07:31 DEBUG "value": [ 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG [...] 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:07:31 DEBUG }, 20:07:31 DEBUG "value": [ 20:07:31 DEBUG 1713553651, 20:07:31 DEBUG "2504.2" 20:07:31 DEBUG ] 20:07:31 DEBUG } 20:07:31 DEBUG ] 20:07:31 DEBUG } 20:07:31 DEBUG } 20:07:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553651" 20:07:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713553651" 20:07:31 INFO [http_cmd]: http status code OK 20:07:31 DEBUG --- status code --- 20:07:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:31 DEBUG --- http response --- 20:07:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:07:31 DEBUG ----- output ----- 20:07:31 DEBUG { 20:07:31 DEBUG "status": "success", 20:07:31 DEBUG "data": { 20:07:31 DEBUG "resultType": "vector", 20:07:31 DEBUG "result": [ 20:07:31 DEBUG { 20:07:31 DEBUG "metric": { 20:07:31 DEBUG "pod": "ds-idrepo-0" 20:07:31 DEBUG }, 20:07:31 DEBUG "value": [ 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG [...] 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG "pod": "ds-cts-2" 20:07:31 DEBUG }, 20:07:31 DEBUG "value": [ 20:07:31 DEBUG 1713553651, 20:07:31 DEBUG "0" 20:07:31 DEBUG ] 20:07:31 DEBUG } 20:07:31 DEBUG ] 20:07:31 DEBUG } 20:07:31 DEBUG } 20:07:31 INFO [http_cmd]: http status code OK 20:07:31 DEBUG --- status code --- 20:07:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:31 DEBUG --- http response --- 20:07:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:07:31 DEBUG ----- output ----- 20:07:31 INFO [http_cmd]: http status code OK 20:07:31 DEBUG { 20:07:31 DEBUG --- status code --- 20:07:31 DEBUG "status": "success", 20:07:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:31 DEBUG "data": { 20:07:31 DEBUG --- http response --- 20:07:31 DEBUG "resultType": "vector", 20:07:31 DEBUG "result": [ 20:07:31 DEBUG { 20:07:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 20:07:31 DEBUG "metric": { 20:07:31 DEBUG ----- output ----- 20:07:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:07:31 DEBUG { 20:07:31 DEBUG }, 20:07:31 DEBUG "status": "success", 20:07:31 DEBUG "value": [ 20:07:31 DEBUG "data": { 20:07:31 DEBUG 20:07:31 DEBUG "resultType": "vector", 20:07:31 DEBUG 20:07:31 DEBUG "result": [ 20:07:31 DEBUG 20:07:31 DEBUG { 20:07:31 DEBUG [...] 20:07:31 DEBUG "metric": { 20:07:31 DEBUG 20:07:31 DEBUG "pod": "am-767f6b67b5-qcqz2" 20:07:31 DEBUG 20:07:31 DEBUG }, 20:07:31 DEBUG 20:07:31 DEBUG "value": [ 20:07:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:07:31 DEBUG 20:07:31 DEBUG }, 20:07:31 DEBUG 20:07:31 DEBUG "value": [ 20:07:31 DEBUG 20:07:31 DEBUG 1713553651, 20:07:31 DEBUG [...] 20:07:31 DEBUG "0.015" 20:07:31 DEBUG 20:07:31 DEBUG ] 20:07:31 DEBUG 20:07:31 DEBUG } 20:07:31 DEBUG 20:07:31 DEBUG ] 20:07:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:07:31 DEBUG } 20:07:31 DEBUG }, 20:07:31 DEBUG } 20:07:31 DEBUG "value": [ 20:07:31 DEBUG 1713553651, 20:07:31 DEBUG "2498.7666666666664" 20:07:31 DEBUG ] 20:07:31 DEBUG } 20:07:31 DEBUG ] 20:07:31 DEBUG } 20:07:31 DEBUG } 20:07:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553651" 20:07:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553651" 20:07:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713553651" 20:07:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553651" 20:07:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553651" 20:07:31 INFO [http_cmd]: http status code OK 20:07:31 INFO [http_cmd]: http status code OK 20:07:31 DEBUG --- status code --- 20:07:31 DEBUG --- status code --- 20:07:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:31 DEBUG --- http response --- 20:07:31 DEBUG --- http response --- 20:07:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:07:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:07:31 DEBUG ----- output ----- 20:07:31 DEBUG ----- output ----- 20:07:31 DEBUG { 20:07:31 DEBUG { 20:07:31 DEBUG "status": "success", 20:07:31 DEBUG "status": "success", 20:07:31 DEBUG "data": { 20:07:31 DEBUG "data": { 20:07:31 DEBUG "resultType": "vector", 20:07:31 DEBUG "resultType": "vector", 20:07:31 DEBUG "result": [ 20:07:31 DEBUG "result": [ 20:07:31 DEBUG { 20:07:31 INFO [http_cmd]: http status code OK 20:07:31 DEBUG { 20:07:31 DEBUG "metric": { 20:07:31 DEBUG --- status code --- 20:07:31 DEBUG "metric": { 20:07:31 DEBUG "instance": "10.142.0.107:9100" 20:07:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:31 DEBUG "pod": "ds-idrepo-0" 20:07:31 DEBUG }, 20:07:31 DEBUG --- http response --- 20:07:31 DEBUG }, 20:07:31 DEBUG "value": [ 20:07:31 DEBUG "value": [ 20:07:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG ----- output ----- 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG { 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG "status": "success", 20:07:31 DEBUG [...] 20:07:31 INFO [http_cmd]: http status code OK 20:07:31 DEBUG [...] 20:07:31 DEBUG "data": { 20:07:31 DEBUG 20:07:31 DEBUG --- status code --- 20:07:31 DEBUG 20:07:31 DEBUG "resultType": "vector", 20:07:31 DEBUG 20:07:31 INFO [http_cmd]: http status code OK 20:07:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:31 DEBUG 20:07:31 DEBUG "result": [ 20:07:31 DEBUG 20:07:31 DEBUG --- status code --- 20:07:31 DEBUG --- http response --- 20:07:31 DEBUG 20:07:31 DEBUG { 20:07:31 DEBUG "instance": "10.142.0.43:9100" 20:07:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:31 DEBUG "pod": "ds-idrepo-2" 20:07:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:07:31 DEBUG "metric": { 20:07:31 DEBUG }, 20:07:31 DEBUG --- http response --- 20:07:31 DEBUG }, 20:07:31 DEBUG ----- output ----- 20:07:31 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:07:31 DEBUG "value": [ 20:07:31 DEBUG "value": [ 20:07:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:07:31 DEBUG { 20:07:31 DEBUG }, 20:07:31 DEBUG 1713553651, 20:07:31 DEBUG 1713553651, 20:07:31 DEBUG ----- output ----- 20:07:31 DEBUG "status": "success", 20:07:31 DEBUG "value": [ 20:07:31 DEBUG "2025280068608" 20:07:31 DEBUG "0" 20:07:31 DEBUG { 20:07:31 DEBUG "data": { 20:07:31 DEBUG 20:07:31 DEBUG ] 20:07:31 DEBUG ] 20:07:31 DEBUG "status": "success", 20:07:31 DEBUG "resultType": "vector", 20:07:31 DEBUG 20:07:31 DEBUG } 20:07:31 DEBUG } 20:07:31 DEBUG "data": { 20:07:31 DEBUG "result": [ 20:07:31 DEBUG 20:07:31 DEBUG ] 20:07:31 DEBUG ] 20:07:31 DEBUG "resultType": "vector", 20:07:31 DEBUG { 20:07:31 DEBUG [...] 20:07:31 DEBUG } 20:07:31 DEBUG } 20:07:31 DEBUG "result": [ 20:07:31 DEBUG "metric": { 20:07:31 DEBUG 20:07:31 DEBUG } 20:07:31 DEBUG } 20:07:31 DEBUG { 20:07:31 DEBUG "pod": "ds-idrepo-1" 20:07:31 DEBUG 20:07:31 DEBUG "metric": { 20:07:31 DEBUG }, 20:07:31 DEBUG 20:07:31 DEBUG "instance": "10.142.0.107:9100" 20:07:31 DEBUG "value": [ 20:07:31 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:07:31 DEBUG }, 20:07:31 DEBUG 20:07:31 DEBUG }, 20:07:31 DEBUG "value": [ 20:07:31 DEBUG 20:07:31 DEBUG "value": [ 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG 1713553651, 20:07:31 DEBUG 20:07:31 DEBUG [...] 20:07:31 DEBUG "3672354816" 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG ] 20:07:31 DEBUG [...] 20:07:31 DEBUG 20:07:31 DEBUG } 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG ] 20:07:31 DEBUG 20:07:31 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:07:31 DEBUG } 20:07:31 DEBUG 20:07:31 DEBUG }, 20:07:31 DEBUG } 20:07:31 DEBUG "instance": "10.142.0.43:9100" 20:07:31 DEBUG "value": [ 20:07:31 DEBUG }, 20:07:31 DEBUG 1713553651, 20:07:31 DEBUG "value": [ 20:07:31 DEBUG "0" 20:07:31 DEBUG 1713553651, 20:07:31 DEBUG ] 20:07:31 DEBUG "75652468.16666667" 20:07:31 DEBUG } 20:07:31 DEBUG ] 20:07:31 DEBUG ] 20:07:31 DEBUG } 20:07:31 DEBUG } 20:07:31 DEBUG ] 20:07:31 DEBUG } 20:07:31 DEBUG } 20:07:31 DEBUG } 20:07:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553651" 20:07:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553651" 20:07:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553651" 20:07:31 INFO [http_cmd]: http status code OK 20:07:31 DEBUG --- status code --- 20:07:31 INFO [http_cmd]: http status code OK 20:07:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:31 DEBUG --- status code --- 20:07:31 DEBUG --- http response --- 20:07:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:31 DEBUG --- http response --- 20:07:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:07:31 DEBUG ----- output ----- 20:07:31 DEBUG { 20:07:31 DEBUG "status": "success", 20:07:31 DEBUG "data": { 20:07:31 DEBUG "resultType": "vector", 20:07:31 DEBUG "result": [ 20:07:31 DEBUG { 20:07:31 DEBUG "metric": { 20:07:31 DEBUG "pod": "ds-cts-0" 20:07:31 DEBUG }, 20:07:31 DEBUG "value": [ 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG [...] 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG "pod": "ds-cts-2" 20:07:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:07:31 DEBUG }, 20:07:31 DEBUG ----- output ----- 20:07:31 DEBUG "value": [ 20:07:31 DEBUG { 20:07:31 DEBUG 1713553651, 20:07:31 DEBUG "status": "success", 20:07:31 DEBUG "0" 20:07:31 DEBUG "data": { 20:07:31 DEBUG ] 20:07:31 DEBUG "resultType": "vector", 20:07:31 DEBUG } 20:07:31 DEBUG "result": [ 20:07:31 DEBUG ] 20:07:31 DEBUG { 20:07:31 DEBUG } 20:07:31 DEBUG "metric": { 20:07:31 DEBUG } 20:07:31 DEBUG "instance": "10.142.0.107:9100" 20:07:31 DEBUG }, 20:07:31 DEBUG "value": [ 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG [...] 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG "instance": "10.142.0.43:9100" 20:07:31 DEBUG }, 20:07:31 DEBUG "value": [ 20:07:31 DEBUG 1713553651, 20:07:31 DEBUG "75649737.69999999" 20:07:31 DEBUG ] 20:07:31 DEBUG } 20:07:31 DEBUG ] 20:07:31 DEBUG } 20:07:31 DEBUG } 20:07:31 INFO [http_cmd]: http status code OK 20:07:31 DEBUG --- status code --- 20:07:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:31 DEBUG --- http response --- 20:07:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:07:31 DEBUG ----- output ----- 20:07:31 DEBUG { 20:07:31 DEBUG "status": "success", 20:07:31 DEBUG "data": { 20:07:31 DEBUG "resultType": "vector", 20:07:31 DEBUG "result": [ 20:07:31 DEBUG { 20:07:31 DEBUG "metric": { 20:07:31 DEBUG "pod": "ds-cts-0" 20:07:31 DEBUG }, 20:07:31 DEBUG "value": [ 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG [...] 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG "pod": "ds-cts-2" 20:07:31 DEBUG }, 20:07:31 DEBUG "value": [ 20:07:31 DEBUG 1713553651, 20:07:31 DEBUG "30" 20:07:31 DEBUG ] 20:07:31 DEBUG } 20:07:31 DEBUG ] 20:07:31 DEBUG } 20:07:31 DEBUG } 20:07:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553651" 20:07:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713553651" 20:07:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553651" 20:07:31 INFO [http_cmd]: http status code OK 20:07:31 DEBUG --- status code --- 20:07:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:31 DEBUG --- http response --- 20:07:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 20:07:31 DEBUG ----- output ----- 20:07:31 DEBUG { 20:07:31 DEBUG "status": "success", 20:07:31 DEBUG "data": { 20:07:31 DEBUG "resultType": "vector", 20:07:31 DEBUG "result": [ 20:07:31 DEBUG { 20:07:31 DEBUG "metric": { 20:07:31 DEBUG "pod": "am-767f6b67b5-qcqz2" 20:07:31 DEBUG }, 20:07:31 DEBUG "value": [ 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG [...] 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:07:31 DEBUG }, 20:07:31 DEBUG "value": [ 20:07:31 DEBUG 1713553651, 20:07:31 DEBUG "2503.4333333333334" 20:07:31 DEBUG ] 20:07:31 DEBUG } 20:07:31 DEBUG ] 20:07:31 DEBUG } 20:07:31 DEBUG } 20:07:31 INFO [http_cmd]: http status code OK 20:07:31 DEBUG --- status code --- 20:07:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:31 DEBUG --- http response --- 20:07:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:07:31 DEBUG ----- output ----- 20:07:31 DEBUG { 20:07:31 DEBUG "status": "success", 20:07:31 DEBUG "data": { 20:07:31 DEBUG "resultType": "vector", 20:07:31 DEBUG "result": [ 20:07:31 DEBUG { 20:07:31 DEBUG "metric": { 20:07:31 DEBUG "pod": "ds-idrepo-1" 20:07:31 DEBUG }, 20:07:31 DEBUG "value": [ 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG [...] 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:07:31 DEBUG }, 20:07:31 DEBUG "value": [ 20:07:31 DEBUG 1713553651, 20:07:31 DEBUG "0" 20:07:31 DEBUG ] 20:07:31 DEBUG } 20:07:31 DEBUG ] 20:07:31 DEBUG } 20:07:31 DEBUG } 20:07:31 INFO [http_cmd]: http status code OK 20:07:31 DEBUG --- status code --- 20:07:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:31 DEBUG --- http response --- 20:07:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:07:31 DEBUG ----- output ----- 20:07:31 DEBUG { 20:07:31 DEBUG "status": "success", 20:07:31 DEBUG "data": { 20:07:31 DEBUG "resultType": "vector", 20:07:31 DEBUG "result": [ 20:07:31 DEBUG { 20:07:31 DEBUG "metric": { 20:07:31 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:07:31 DEBUG }, 20:07:31 DEBUG "value": [ 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG [...] 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:07:31 DEBUG }, 20:07:31 DEBUG "value": [ 20:07:31 DEBUG 1713553651, 20:07:31 DEBUG "8.042545509557435" 20:07:31 DEBUG ] 20:07:31 DEBUG } 20:07:31 DEBUG ] 20:07:31 DEBUG } 20:07:31 DEBUG } 20:07:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553651" 20:07:31 INFO [http_cmd]: http status code OK 20:07:31 DEBUG --- status code --- 20:07:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:31 DEBUG --- http response --- 20:07:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:07:31 DEBUG ----- output ----- 20:07:31 DEBUG { 20:07:31 DEBUG "status": "success", 20:07:31 DEBUG "data": { 20:07:31 DEBUG "resultType": "vector", 20:07:31 DEBUG "result": [ 20:07:31 DEBUG { 20:07:31 DEBUG "metric": { 20:07:31 DEBUG "pod": "ds-idrepo-1" 20:07:31 DEBUG }, 20:07:31 DEBUG "value": [ 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG [...] 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:07:31 DEBUG }, 20:07:31 DEBUG "value": [ 20:07:31 DEBUG 1713553651, 20:07:31 DEBUG "0.000019932958712889033" 20:07:31 DEBUG ] 20:07:31 DEBUG } 20:07:31 DEBUG ] 20:07:31 DEBUG } 20:07:31 DEBUG } 20:07:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553651" 20:07:31 INFO [http_cmd]: http status code OK 20:07:31 DEBUG --- status code --- 20:07:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:31 DEBUG --- http response --- 20:07:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 20:07:31 DEBUG ----- output ----- 20:07:31 DEBUG { 20:07:31 DEBUG "status": "success", 20:07:31 DEBUG "data": { 20:07:31 DEBUG "resultType": "vector", 20:07:31 DEBUG "result": [ 20:07:31 DEBUG { 20:07:31 DEBUG "metric": { 20:07:31 DEBUG "pod": "am-767f6b67b5-qcqz2" 20:07:31 DEBUG }, 20:07:31 DEBUG "value": [ 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG [...] 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG 20:07:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:07:31 DEBUG }, 20:07:31 DEBUG "value": [ 20:07:31 DEBUG 1713553651, 20:07:31 DEBUG "2505.0666666666666" 20:07:31 DEBUG ] 20:07:31 DEBUG } 20:07:31 DEBUG ] 20:07:31 DEBUG } 20:07:31 DEBUG } 20:07:33 INFO 20:07:33 INFO [loop_until]: kubectl --namespace=xlou top pods 20:07:33 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:07:33 INFO [loop_until]: OK (rc = 0) 20:07:33 DEBUG --- stdout --- 20:07:33 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 15865m 9009Mi am-767f6b67b5-qcqz2 16108m 8812Mi am-767f6b67b5-vbltz 17544m 8997Mi ds-cts-0 17353m 10973Mi ds-cts-1 15449m 11219Mi ds-cts-2 17923m 10977Mi ds-idrepo-0 7104m 3692Mi ds-idrepo-1 7016m 3537Mi ds-idrepo-2 7189m 3493Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 4m 1217Mi lodemon-f8459c597-6tj6f 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8001m 3408Mi 20:07:33 DEBUG --- stderr --- 20:07:33 DEBUG 20:07:34 INFO 20:07:34 INFO [loop_until]: kubectl --namespace=xlou top node 20:07:34 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:07:34 INFO [loop_until]: OK (rc = 0) 20:07:34 DEBUG --- stdout --- 20:07:34 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 69m 0% 6240Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7410m 24% 7280Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16176m 54% 11712Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 85m 0% 3331Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 17510m 58% 13955Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 164m 0% 5208Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 16507m 55% 11695Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15539m 52% 2982Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 18117m 60% 11210Mi 10% gke-xlou-cdm-ds-0f601751-j488 7684m 25% 6734Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 15663m 52% 15977Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 7532m 25% 7434Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 16917m 56% 13436Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 8198m 51% 7136Mi 12% 20:07:34 DEBUG --- stderr --- 20:07:34 DEBUG 20:08:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553711" 20:08:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553711" 20:08:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553711" 20:08:31 INFO [http_cmd]: http status code OK 20:08:31 DEBUG --- status code --- 20:08:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:31 DEBUG --- http response --- 20:08:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:08:31 DEBUG ----- output ----- 20:08:31 DEBUG { 20:08:31 DEBUG "status": "success", 20:08:31 DEBUG "data": { 20:08:31 DEBUG "resultType": "vector", 20:08:31 DEBUG "result": [ 20:08:31 DEBUG { 20:08:31 DEBUG "metric": { 20:08:31 DEBUG "pod": "ds-idrepo-0" 20:08:31 DEBUG }, 20:08:31 DEBUG "value": [ 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG [...] 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG "pod": "ds-idrepo-2" 20:08:31 DEBUG }, 20:08:31 DEBUG "value": [ 20:08:31 DEBUG 1713553711, 20:08:31 DEBUG "0" 20:08:31 DEBUG ] 20:08:31 DEBUG } 20:08:31 DEBUG ] 20:08:31 DEBUG } 20:08:31 DEBUG } 20:08:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713553711" 20:08:31 INFO [http_cmd]: http status code OK 20:08:31 DEBUG --- status code --- 20:08:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:31 DEBUG --- http response --- 20:08:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:08:31 DEBUG ----- output ----- 20:08:31 DEBUG { 20:08:31 DEBUG "status": "success", 20:08:31 DEBUG "data": { 20:08:31 DEBUG "resultType": "vector", 20:08:31 DEBUG "result": [ 20:08:31 DEBUG { 20:08:31 DEBUG "metric": { 20:08:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:08:31 DEBUG }, 20:08:31 DEBUG "value": [ 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG [...] 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:08:31 DEBUG }, 20:08:31 DEBUG "value": [ 20:08:31 DEBUG 1713553711, 20:08:31 DEBUG "450362" 20:08:31 DEBUG ] 20:08:31 DEBUG } 20:08:31 DEBUG ] 20:08:31 DEBUG } 20:08:31 DEBUG } 20:08:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553711" 20:08:31 INFO [http_cmd]: http status code OK 20:08:31 DEBUG --- status code --- 20:08:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:31 DEBUG --- http response --- 20:08:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:08:31 DEBUG ----- output ----- 20:08:31 DEBUG { 20:08:31 DEBUG "status": "success", 20:08:31 DEBUG "data": { 20:08:31 DEBUG "resultType": "vector", 20:08:31 DEBUG "result": [ 20:08:31 DEBUG { 20:08:31 DEBUG "metric": { 20:08:31 DEBUG "instance": "10.142.0.107:9100" 20:08:31 DEBUG }, 20:08:31 DEBUG "value": [ 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG [...] 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG "instance": "10.142.0.43:9100" 20:08:31 DEBUG }, 20:08:31 DEBUG "value": [ 20:08:31 DEBUG 1713553711, 20:08:31 DEBUG "0.11166666666667122" 20:08:31 DEBUG ] 20:08:31 DEBUG } 20:08:31 DEBUG ] 20:08:31 DEBUG } 20:08:31 DEBUG } 20:08:31 ERROR [http_cmd]: ERROR 20:08:31 ERROR --- status code --- 20:08:31 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:08:31 ERROR --- http response --- 20:08:31 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:08:31 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:08:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553711" 20:08:31 INFO [http_cmd]: http status code OK 20:08:31 DEBUG --- status code --- 20:08:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:31 DEBUG --- http response --- 20:08:31 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:08:31 INFO [http_cmd]: http status code OK 20:08:31 DEBUG --- status code --- 20:08:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:31 DEBUG --- http response --- 20:08:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:08:31 DEBUG ----- output ----- 20:08:31 DEBUG { 20:08:31 DEBUG "status": "success", 20:08:31 DEBUG "data": { 20:08:31 DEBUG "resultType": "vector", 20:08:31 DEBUG "result": [ 20:08:31 DEBUG { 20:08:31 DEBUG "metric": { 20:08:31 DEBUG "pod": "ds-idrepo-0" 20:08:31 DEBUG }, 20:08:31 DEBUG "value": [ 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG [...] 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG "pod": "ds-idrepo-2" 20:08:31 DEBUG }, 20:08:31 DEBUG "value": [ 20:08:31 DEBUG 1713553711, 20:08:31 DEBUG "0" 20:08:31 DEBUG ] 20:08:31 DEBUG } 20:08:31 DEBUG ] 20:08:31 DEBUG } 20:08:31 DEBUG } 20:08:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553711" 20:08:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553711" 20:08:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553711" 20:08:31 INFO [http_cmd]: http status code OK 20:08:31 DEBUG --- status code --- 20:08:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:31 DEBUG --- http response --- 20:08:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:08:31 DEBUG ----- output ----- 20:08:31 DEBUG { 20:08:31 DEBUG "status": "success", 20:08:31 DEBUG "data": { 20:08:31 DEBUG "resultType": "vector", 20:08:31 DEBUG "result": [ 20:08:31 DEBUG { 20:08:31 DEBUG "metric": { 20:08:31 DEBUG "pod": "ds-cts-0" 20:08:31 DEBUG }, 20:08:31 DEBUG "value": [ 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG [...] 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG "pod": "ds-cts-2" 20:08:31 DEBUG }, 20:08:31 DEBUG "value": [ 20:08:31 DEBUG 1713553711, 20:08:31 DEBUG "0.0009982333333333332" 20:08:31 DEBUG ] 20:08:31 DEBUG } 20:08:31 DEBUG ] 20:08:31 DEBUG } 20:08:31 DEBUG } 20:08:31 INFO [http_cmd]: http status code OK 20:08:31 INFO [http_cmd]: http status code OK 20:08:31 DEBUG --- status code --- 20:08:31 DEBUG --- status code --- 20:08:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:31 DEBUG --- http response --- 20:08:31 DEBUG --- http response --- 20:08:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:08:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 20:08:31 DEBUG ----- output ----- 20:08:31 DEBUG ----- output ----- 20:08:31 DEBUG { 20:08:31 DEBUG { 20:08:31 DEBUG "status": "success", 20:08:31 DEBUG "status": "success", 20:08:31 DEBUG "data": { 20:08:31 DEBUG "data": { 20:08:31 DEBUG "resultType": "vector", 20:08:31 DEBUG "resultType": "vector", 20:08:31 DEBUG "result": [ 20:08:31 DEBUG "result": [ 20:08:31 DEBUG { 20:08:31 DEBUG { 20:08:31 DEBUG "metric": { 20:08:31 DEBUG "metric": { 20:08:31 DEBUG "instance": "10.142.0.107:9100" 20:08:31 DEBUG "pod": "ds-idrepo-0" 20:08:31 DEBUG }, 20:08:31 DEBUG }, 20:08:31 DEBUG "value": [ 20:08:31 DEBUG "value": [ 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG [...] 20:08:31 DEBUG [...] 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG "instance": "10.142.0.43:9100" 20:08:31 DEBUG "pod": "idm-b85f9d978-wnhxb" 20:08:31 DEBUG }, 20:08:31 DEBUG }, 20:08:31 DEBUG "value": [ 20:08:31 DEBUG "value": [ 20:08:31 DEBUG 1713553711, 20:08:31 DEBUG 1713553711, 20:08:31 DEBUG "0.28473333333332107" 20:08:31 DEBUG "31729069.04319841" 20:08:31 DEBUG ] 20:08:31 DEBUG ] 20:08:31 DEBUG } 20:08:31 DEBUG } 20:08:31 DEBUG ] 20:08:31 DEBUG ] 20:08:31 DEBUG } 20:08:31 DEBUG } 20:08:31 DEBUG } 20:08:31 DEBUG } 20:08:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553711" 20:08:31 INFO [http_cmd]: http status code OK 20:08:31 DEBUG --- status code --- 20:08:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:31 DEBUG --- http response --- 20:08:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:08:31 DEBUG ----- output ----- 20:08:31 DEBUG { 20:08:31 DEBUG "status": "success", 20:08:31 DEBUG "data": { 20:08:31 DEBUG "resultType": "vector", 20:08:31 DEBUG "result": [ 20:08:31 DEBUG { 20:08:31 DEBUG "metric": { 20:08:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:08:31 DEBUG }, 20:08:31 DEBUG "value": [ 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG [...] 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:08:31 DEBUG }, 20:08:31 DEBUG "value": [ 20:08:31 DEBUG 1713553711, 20:08:31 DEBUG "0" 20:08:31 DEBUG ] 20:08:31 DEBUG } 20:08:31 DEBUG ] 20:08:31 DEBUG } 20:08:31 DEBUG } 20:08:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553711" 20:08:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713553711" 20:08:31 INFO [http_cmd]: http status code OK 20:08:31 DEBUG --- status code --- 20:08:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:31 DEBUG --- http response --- 20:08:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:08:31 DEBUG ----- output ----- 20:08:31 DEBUG { 20:08:31 DEBUG "status": "success", 20:08:31 DEBUG "data": { 20:08:31 DEBUG "resultType": "vector", 20:08:31 DEBUG "result": [ 20:08:31 DEBUG { 20:08:31 DEBUG "metric": { 20:08:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:08:31 DEBUG }, 20:08:31 DEBUG "value": [ 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG [...] 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:08:31 DEBUG }, 20:08:31 DEBUG "value": [ 20:08:31 DEBUG 1713553711, 20:08:31 DEBUG "2498.633333333333" 20:08:31 DEBUG ] 20:08:31 DEBUG } 20:08:31 DEBUG ] 20:08:31 DEBUG } 20:08:31 DEBUG } 20:08:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713553711" 20:08:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553711" 20:08:31 INFO [http_cmd]: http status code OK 20:08:31 DEBUG --- status code --- 20:08:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:31 DEBUG --- http response --- 20:08:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:08:31 DEBUG ----- output ----- 20:08:31 DEBUG { 20:08:31 DEBUG "status": "success", 20:08:31 DEBUG "data": { 20:08:31 DEBUG "resultType": "vector", 20:08:31 DEBUG "result": [ 20:08:31 DEBUG { 20:08:31 DEBUG "metric": { 20:08:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:08:31 DEBUG }, 20:08:31 DEBUG "value": [ 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG [...] 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:08:31 DEBUG }, 20:08:31 DEBUG "value": [ 20:08:31 DEBUG 1713553711, 20:08:31 DEBUG "0.015" 20:08:31 DEBUG ] 20:08:31 DEBUG } 20:08:31 DEBUG ] 20:08:31 DEBUG } 20:08:31 DEBUG } 20:08:31 INFO [http_cmd]: http status code OK 20:08:31 DEBUG --- status code --- 20:08:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:31 DEBUG --- http response --- 20:08:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:08:31 DEBUG ----- output ----- 20:08:31 DEBUG { 20:08:31 DEBUG "status": "success", 20:08:31 DEBUG "data": { 20:08:31 DEBUG "resultType": "vector", 20:08:31 DEBUG "result": [ 20:08:31 DEBUG { 20:08:31 DEBUG "metric": { 20:08:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:08:31 DEBUG }, 20:08:31 DEBUG "value": [ 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG [...] 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:08:31 DEBUG }, 20:08:31 DEBUG "value": [ 20:08:31 DEBUG 1713553711, 20:08:31 DEBUG "2498.8333333333335" 20:08:31 DEBUG ] 20:08:31 DEBUG } 20:08:31 DEBUG ] 20:08:31 DEBUG } 20:08:31 DEBUG } 20:08:31 INFO [http_cmd]: http status code OK 20:08:31 DEBUG --- status code --- 20:08:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:31 DEBUG --- http response --- 20:08:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:08:31 DEBUG ----- output ----- 20:08:31 DEBUG { 20:08:31 DEBUG "status": "success", 20:08:31 DEBUG "data": { 20:08:31 DEBUG "resultType": "vector", 20:08:31 DEBUG "result": [ 20:08:31 DEBUG { 20:08:31 DEBUG "metric": { 20:08:31 DEBUG "pod": "ds-idrepo-0" 20:08:31 DEBUG }, 20:08:31 DEBUG "value": [ 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG [...] 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG "pod": "ds-cts-2" 20:08:31 DEBUG }, 20:08:31 DEBUG "value": [ 20:08:31 DEBUG 1713553711, 20:08:31 DEBUG "0" 20:08:31 DEBUG ] 20:08:31 DEBUG } 20:08:31 DEBUG ] 20:08:31 DEBUG } 20:08:31 DEBUG } 20:08:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553711" 20:08:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553711" 20:08:31 INFO [http_cmd]: http status code OK 20:08:31 DEBUG --- status code --- 20:08:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:31 DEBUG --- http response --- 20:08:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:08:31 DEBUG ----- output ----- 20:08:31 DEBUG { 20:08:31 DEBUG "status": "success", 20:08:31 DEBUG "data": { 20:08:31 DEBUG "resultType": "vector", 20:08:31 DEBUG "result": [ 20:08:31 DEBUG { 20:08:31 DEBUG "metric": { 20:08:31 DEBUG "instance": "10.142.0.107:9100" 20:08:31 DEBUG }, 20:08:31 DEBUG "value": [ 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG [...] 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG "instance": "10.142.0.43:9100" 20:08:31 DEBUG }, 20:08:31 DEBUG "value": [ 20:08:31 DEBUG 1713553711, 20:08:31 DEBUG "75567279.66666667" 20:08:31 DEBUG ] 20:08:31 DEBUG } 20:08:31 DEBUG ] 20:08:31 DEBUG } 20:08:31 DEBUG } 20:08:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553711" 20:08:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553711" 20:08:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713553711" 20:08:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553711" 20:08:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553711" 20:08:31 INFO [http_cmd]: http status code OK 20:08:31 DEBUG --- status code --- 20:08:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:31 DEBUG --- http response --- 20:08:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:08:31 INFO [http_cmd]: http status code OK 20:08:31 INFO [http_cmd]: http status code OK 20:08:31 DEBUG ----- output ----- 20:08:31 DEBUG --- status code --- 20:08:31 DEBUG --- status code --- 20:08:31 DEBUG { 20:08:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:31 DEBUG "status": "success", 20:08:31 DEBUG --- http response --- 20:08:31 DEBUG --- http response --- 20:08:31 DEBUG "data": { 20:08:31 INFO [http_cmd]: http status code OK 20:08:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:08:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:08:31 DEBUG "resultType": "vector", 20:08:31 DEBUG --- status code --- 20:08:31 DEBUG ----- output ----- 20:08:31 DEBUG ----- output ----- 20:08:31 DEBUG "result": [ 20:08:31 INFO [http_cmd]: http status code OK 20:08:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:31 DEBUG { 20:08:31 DEBUG { 20:08:31 DEBUG { 20:08:31 DEBUG --- status code --- 20:08:31 DEBUG --- http response --- 20:08:31 DEBUG "status": "success", 20:08:31 DEBUG "status": "success", 20:08:31 DEBUG "metric": { 20:08:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:31 DEBUG "data": { 20:08:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:08:31 DEBUG "data": { 20:08:31 DEBUG "pod": "ds-idrepo-0" 20:08:31 DEBUG --- http response --- 20:08:31 DEBUG "resultType": "vector", 20:08:31 DEBUG ----- output ----- 20:08:31 DEBUG "resultType": "vector", 20:08:31 DEBUG }, 20:08:31 DEBUG "result": [ 20:08:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 20:08:31 DEBUG { 20:08:31 DEBUG "result": [ 20:08:31 DEBUG "value": [ 20:08:31 DEBUG { 20:08:31 INFO [http_cmd]: http status code OK 20:08:31 DEBUG ----- output ----- 20:08:31 DEBUG "status": "success", 20:08:31 DEBUG { 20:08:31 DEBUG 20:08:31 DEBUG "metric": { 20:08:31 DEBUG --- status code --- 20:08:31 DEBUG { 20:08:31 DEBUG "data": { 20:08:31 DEBUG "metric": { 20:08:31 DEBUG 20:08:31 DEBUG "pod": "ds-cts-0" 20:08:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:31 DEBUG "status": "success", 20:08:31 DEBUG "resultType": "vector", 20:08:31 DEBUG "instance": "10.142.0.107:9100" 20:08:31 DEBUG 20:08:31 DEBUG }, 20:08:31 DEBUG --- http response --- 20:08:31 DEBUG "data": { 20:08:31 DEBUG "result": [ 20:08:31 DEBUG }, 20:08:31 DEBUG [...] 20:08:31 DEBUG "value": [ 20:08:31 DEBUG "resultType": "vector", 20:08:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:08:31 DEBUG { 20:08:31 DEBUG "value": [ 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG "result": [ 20:08:31 DEBUG ----- output ----- 20:08:31 DEBUG "metric": { 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG { 20:08:31 DEBUG { 20:08:31 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG "metric": { 20:08:31 DEBUG "status": "success", 20:08:31 DEBUG }, 20:08:31 DEBUG 20:08:31 DEBUG "pod": "ds-idrepo-2" 20:08:31 DEBUG [...] 20:08:31 DEBUG "pod": "ds-idrepo-0" 20:08:31 DEBUG "data": { 20:08:31 DEBUG "value": [ 20:08:31 DEBUG [...] 20:08:31 DEBUG }, 20:08:31 DEBUG 20:08:31 DEBUG }, 20:08:31 DEBUG "resultType": "vector", 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG "value": [ 20:08:31 DEBUG 20:08:31 DEBUG "value": [ 20:08:31 DEBUG "result": [ 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG 1713553711, 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG { 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG "0" 20:08:31 DEBUG "pod": "ds-cts-2" 20:08:31 DEBUG 20:08:31 DEBUG "metric": { 20:08:31 DEBUG [...] 20:08:31 DEBUG "instance": "10.142.0.43:9100" 20:08:31 DEBUG ] 20:08:31 DEBUG }, 20:08:31 DEBUG 20:08:31 DEBUG "instance": "10.142.0.107:9100" 20:08:31 DEBUG 20:08:31 DEBUG }, 20:08:31 DEBUG } 20:08:31 DEBUG "value": [ 20:08:31 DEBUG [...] 20:08:31 DEBUG }, 20:08:31 DEBUG 20:08:31 DEBUG "value": [ 20:08:31 DEBUG ] 20:08:31 DEBUG 1713553711, 20:08:31 DEBUG 20:08:31 DEBUG "value": [ 20:08:31 DEBUG 20:08:31 DEBUG 1713553711, 20:08:31 DEBUG } 20:08:31 DEBUG "0" 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:08:31 DEBUG "2028744694784" 20:08:31 DEBUG } 20:08:31 DEBUG ] 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG }, 20:08:31 DEBUG ] 20:08:31 DEBUG } 20:08:31 DEBUG "pod": "idm-b85f9d978-wnhxb" 20:08:31 DEBUG 20:08:31 DEBUG "value": [ 20:08:31 DEBUG } 20:08:31 DEBUG ] 20:08:31 DEBUG }, 20:08:31 DEBUG [...] 20:08:31 DEBUG 1713553711, 20:08:31 DEBUG ] 20:08:31 DEBUG } 20:08:31 DEBUG "value": [ 20:08:31 DEBUG 20:08:31 DEBUG "3675209728" 20:08:31 DEBUG } 20:08:31 DEBUG } 20:08:31 DEBUG 1713553711, 20:08:31 DEBUG 20:08:31 DEBUG ] 20:08:31 DEBUG } 20:08:31 DEBUG "405.59475182572095" 20:08:31 DEBUG 20:08:31 DEBUG } 20:08:31 DEBUG ] 20:08:31 DEBUG "instance": "10.142.0.43:9100" 20:08:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553711" 20:08:31 DEBUG ] 20:08:31 DEBUG } 20:08:31 DEBUG }, 20:08:31 DEBUG } 20:08:31 DEBUG ] 20:08:31 DEBUG "value": [ 20:08:31 DEBUG } 20:08:31 DEBUG } 20:08:31 DEBUG 1713553711, 20:08:31 DEBUG } 20:08:31 DEBUG "75571215.63333333" 20:08:31 DEBUG ] 20:08:31 DEBUG } 20:08:31 DEBUG ] 20:08:31 DEBUG } 20:08:31 DEBUG } 20:08:31 INFO [http_cmd]: http status code OK 20:08:31 DEBUG --- status code --- 20:08:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:31 DEBUG --- http response --- 20:08:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:08:31 DEBUG ----- output ----- 20:08:31 DEBUG { 20:08:31 DEBUG "status": "success", 20:08:31 DEBUG "data": { 20:08:31 DEBUG "resultType": "vector", 20:08:31 DEBUG "result": [ 20:08:31 DEBUG { 20:08:31 DEBUG "metric": { 20:08:31 DEBUG "pod": "ds-cts-0" 20:08:31 DEBUG }, 20:08:31 DEBUG "value": [ 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG [...] 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG "pod": "ds-cts-2" 20:08:31 DEBUG }, 20:08:31 DEBUG "value": [ 20:08:31 DEBUG 1713553711, 20:08:31 DEBUG "30" 20:08:31 DEBUG ] 20:08:31 DEBUG } 20:08:31 DEBUG ] 20:08:31 DEBUG } 20:08:31 DEBUG } 20:08:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553711" 20:08:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553711" 20:08:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713553711" 20:08:31 INFO [http_cmd]: http status code OK 20:08:31 DEBUG --- status code --- 20:08:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:31 DEBUG --- http response --- 20:08:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:08:31 DEBUG ----- output ----- 20:08:31 DEBUG { 20:08:31 DEBUG "status": "success", 20:08:31 DEBUG "data": { 20:08:31 DEBUG "resultType": "vector", 20:08:31 DEBUG "result": [ 20:08:31 DEBUG { 20:08:31 DEBUG "metric": { 20:08:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:08:31 DEBUG }, 20:08:31 DEBUG "value": [ 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG [...] 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:08:31 DEBUG }, 20:08:31 DEBUG "value": [ 20:08:31 DEBUG 1713553711, 20:08:31 DEBUG "2497" 20:08:31 DEBUG ] 20:08:31 DEBUG } 20:08:31 DEBUG ] 20:08:31 DEBUG } 20:08:31 DEBUG } 20:08:31 INFO [http_cmd]: http status code OK 20:08:31 DEBUG --- status code --- 20:08:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:31 DEBUG --- http response --- 20:08:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 20:08:31 DEBUG ----- output ----- 20:08:31 DEBUG { 20:08:31 DEBUG "status": "success", 20:08:31 DEBUG "data": { 20:08:31 DEBUG "resultType": "vector", 20:08:31 DEBUG "result": [ 20:08:31 DEBUG { 20:08:31 DEBUG "metric": { 20:08:31 DEBUG "pod": "ds-idrepo-0" 20:08:31 DEBUG }, 20:08:31 DEBUG "value": [ 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG [...] 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG "pod": "idm-b85f9d978-wnhxb" 20:08:31 DEBUG }, 20:08:31 DEBUG "value": [ 20:08:31 DEBUG 1713553711, 20:08:31 DEBUG "0" 20:08:31 DEBUG ] 20:08:31 DEBUG } 20:08:31 DEBUG ] 20:08:31 DEBUG } 20:08:31 DEBUG } 20:08:31 INFO [http_cmd]: http status code OK 20:08:31 DEBUG --- status code --- 20:08:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:31 DEBUG --- http response --- 20:08:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:08:31 DEBUG ----- output ----- 20:08:31 DEBUG { 20:08:31 DEBUG "status": "success", 20:08:31 DEBUG "data": { 20:08:31 DEBUG "resultType": "vector", 20:08:31 DEBUG "result": [ 20:08:31 DEBUG { 20:08:31 DEBUG "metric": { 20:08:31 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:08:31 DEBUG }, 20:08:31 DEBUG "value": [ 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG [...] 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:08:31 DEBUG }, 20:08:31 DEBUG "value": [ 20:08:31 DEBUG 1713553711, 20:08:31 DEBUG "8.039264053145246" 20:08:31 DEBUG ] 20:08:31 DEBUG } 20:08:31 DEBUG ] 20:08:31 DEBUG } 20:08:31 DEBUG } 20:08:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553711" 20:08:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553711" 20:08:31 INFO [http_cmd]: http status code OK 20:08:31 DEBUG --- status code --- 20:08:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:31 DEBUG --- http response --- 20:08:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 20:08:31 DEBUG ----- output ----- 20:08:31 DEBUG { 20:08:31 DEBUG "status": "success", 20:08:31 DEBUG "data": { 20:08:31 DEBUG "resultType": "vector", 20:08:31 DEBUG "result": [ 20:08:31 DEBUG { 20:08:31 DEBUG "metric": { 20:08:31 DEBUG "pod": "ds-idrepo-0" 20:08:31 DEBUG }, 20:08:31 DEBUG "value": [ 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG [...] 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG "pod": "idm-b85f9d978-wnhxb" 20:08:31 DEBUG }, 20:08:31 DEBUG "value": [ 20:08:31 DEBUG 1713553711, 20:08:31 DEBUG "0.002926278004703155" 20:08:31 DEBUG ] 20:08:31 DEBUG } 20:08:31 DEBUG ] 20:08:31 DEBUG } 20:08:31 DEBUG } 20:08:31 INFO [http_cmd]: http status code OK 20:08:31 DEBUG --- status code --- 20:08:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:31 DEBUG --- http response --- 20:08:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:08:31 DEBUG ----- output ----- 20:08:31 DEBUG { 20:08:31 DEBUG "status": "success", 20:08:31 DEBUG "data": { 20:08:31 DEBUG "resultType": "vector", 20:08:31 DEBUG "result": [ 20:08:31 DEBUG { 20:08:31 DEBUG "metric": { 20:08:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:08:31 DEBUG }, 20:08:31 DEBUG "value": [ 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG [...] 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG 20:08:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:08:31 DEBUG }, 20:08:31 DEBUG "value": [ 20:08:31 DEBUG 1713553711, 20:08:31 DEBUG "2498.9" 20:08:31 DEBUG ] 20:08:31 DEBUG } 20:08:31 DEBUG ] 20:08:31 DEBUG } 20:08:31 DEBUG } 20:08:33 INFO 20:08:33 INFO [loop_until]: kubectl --namespace=xlou top pods 20:08:33 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:08:33 INFO [loop_until]: OK (rc = 0) 20:08:33 DEBUG --- stdout --- 20:08:33 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 16033m 9013Mi am-767f6b67b5-qcqz2 16106m 8813Mi am-767f6b67b5-vbltz 17749m 8998Mi ds-cts-0 17302m 10975Mi ds-cts-1 15894m 11224Mi ds-cts-2 17749m 10987Mi ds-idrepo-0 7288m 3692Mi ds-idrepo-1 7007m 3537Mi ds-idrepo-2 7356m 3492Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7967m 3410Mi 20:08:33 DEBUG --- stderr --- 20:08:33 DEBUG 20:08:34 INFO 20:08:34 INFO [loop_until]: kubectl --namespace=xlou top node 20:08:34 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:08:34 INFO [loop_until]: OK (rc = 0) 20:08:34 DEBUG --- stdout --- 20:08:34 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 70m 0% 6237Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7381m 24% 7290Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16060m 53% 11717Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 82m 0% 3343Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 17375m 58% 13955Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 148m 0% 5181Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 16521m 55% 11700Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15538m 52% 2994Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 18217m 61% 11219Mi 10% gke-xlou-cdm-ds-0f601751-j488 7699m 25% 6742Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 15665m 52% 15976Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 7483m 25% 7433Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 17314m 57% 13419Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 8199m 51% 7135Mi 12% 20:08:34 DEBUG --- stderr --- 20:08:34 DEBUG 20:09:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553771" 20:09:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713553771" 20:09:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553771" 20:09:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553771" 20:09:31 INFO [http_cmd]: http status code OK 20:09:31 DEBUG --- status code --- 20:09:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:31 DEBUG --- http response --- 20:09:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:09:31 DEBUG ----- output ----- 20:09:31 DEBUG { 20:09:31 DEBUG "status": "success", 20:09:31 DEBUG "data": { 20:09:31 DEBUG "resultType": "vector", 20:09:31 DEBUG "result": [ 20:09:31 DEBUG { 20:09:31 DEBUG "metric": { 20:09:31 DEBUG "pod": "ds-idrepo-0" 20:09:31 DEBUG }, 20:09:31 DEBUG "value": [ 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG [...] 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG "pod": "ds-idrepo-2" 20:09:31 DEBUG }, 20:09:31 DEBUG "value": [ 20:09:31 DEBUG 1713553771, 20:09:31 DEBUG "0" 20:09:31 DEBUG ] 20:09:31 DEBUG } 20:09:31 DEBUG ] 20:09:31 DEBUG } 20:09:31 DEBUG } 20:09:31 INFO [http_cmd]: http status code OK 20:09:31 INFO [http_cmd]: http status code OK 20:09:31 ERROR [http_cmd]: ERROR 20:09:31 DEBUG --- status code --- 20:09:31 DEBUG --- status code --- 20:09:31 ERROR --- status code --- 20:09:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:31 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:09:31 DEBUG --- http response --- 20:09:31 DEBUG --- http response --- 20:09:31 ERROR --- http response --- 20:09:31 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:09:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:09:31 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:09:31 DEBUG ----- output ----- 20:09:31 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:09:31 DEBUG { 20:09:31 DEBUG "status": "success", 20:09:31 DEBUG "data": { 20:09:31 DEBUG "resultType": "vector", 20:09:31 DEBUG "result": [ 20:09:31 DEBUG { 20:09:31 DEBUG "metric": { 20:09:31 DEBUG "pod": "ds-idrepo-0" 20:09:31 DEBUG }, 20:09:31 DEBUG "value": [ 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG [...] 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG "pod": "ds-idrepo-2" 20:09:31 DEBUG }, 20:09:31 DEBUG "value": [ 20:09:31 DEBUG 1713553771, 20:09:31 DEBUG "0" 20:09:31 DEBUG ] 20:09:31 DEBUG } 20:09:31 DEBUG ] 20:09:31 DEBUG } 20:09:31 DEBUG } 20:09:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553771" 20:09:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553771" 20:09:31 INFO [http_cmd]: http status code OK 20:09:31 DEBUG --- status code --- 20:09:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:31 DEBUG --- http response --- 20:09:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:09:31 DEBUG ----- output ----- 20:09:31 DEBUG { 20:09:31 DEBUG "status": "success", 20:09:31 DEBUG "data": { 20:09:31 DEBUG "resultType": "vector", 20:09:31 DEBUG "result": [ 20:09:31 DEBUG { 20:09:31 DEBUG "metric": { 20:09:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:09:31 DEBUG }, 20:09:31 DEBUG "value": [ 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG [...] 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:09:31 DEBUG }, 20:09:31 DEBUG "value": [ 20:09:31 DEBUG 1713553771, 20:09:31 DEBUG "450463.5" 20:09:31 DEBUG ] 20:09:31 DEBUG } 20:09:31 DEBUG ] 20:09:31 DEBUG } 20:09:31 DEBUG } 20:09:31 INFO [http_cmd]: http status code OK 20:09:31 DEBUG --- status code --- 20:09:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:31 DEBUG --- http response --- 20:09:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:09:31 DEBUG ----- output ----- 20:09:31 DEBUG { 20:09:31 DEBUG "status": "success", 20:09:31 DEBUG "data": { 20:09:31 DEBUG "resultType": "vector", 20:09:31 DEBUG "result": [ 20:09:31 DEBUG { 20:09:31 DEBUG "metric": { 20:09:31 DEBUG "instance": "10.142.0.107:9100" 20:09:31 DEBUG }, 20:09:31 DEBUG "value": [ 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG [...] 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG "instance": "10.142.0.43:9100" 20:09:31 DEBUG }, 20:09:31 DEBUG "value": [ 20:09:31 DEBUG 1713553771, 20:09:31 DEBUG "0.12766666666666424" 20:09:31 DEBUG ] 20:09:31 DEBUG } 20:09:31 DEBUG ] 20:09:31 DEBUG } 20:09:31 DEBUG } 20:09:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553771" 20:09:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553771" 20:09:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553771" 20:09:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713553771" 20:09:31 INFO [http_cmd]: http status code OK 20:09:31 DEBUG --- status code --- 20:09:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:31 DEBUG --- http response --- 20:09:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:09:31 DEBUG ----- output ----- 20:09:31 DEBUG { 20:09:31 DEBUG "status": "success", 20:09:31 DEBUG "data": { 20:09:31 DEBUG "resultType": "vector", 20:09:31 DEBUG "result": [ 20:09:31 DEBUG { 20:09:31 DEBUG "metric": { 20:09:31 DEBUG "pod": "ds-cts-0" 20:09:31 DEBUG }, 20:09:31 DEBUG "value": [ 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG [...] 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG "pod": "ds-cts-2" 20:09:31 DEBUG }, 20:09:31 DEBUG "value": [ 20:09:31 DEBUG 1713553771, 20:09:31 DEBUG "0.0007666666666666666" 20:09:31 DEBUG ] 20:09:31 DEBUG } 20:09:31 DEBUG ] 20:09:31 DEBUG } 20:09:31 DEBUG } 20:09:31 INFO [http_cmd]: http status code OK 20:09:31 DEBUG --- status code --- 20:09:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:31 DEBUG --- http response --- 20:09:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:09:31 DEBUG ----- output ----- 20:09:31 DEBUG { 20:09:31 DEBUG "status": "success", 20:09:31 DEBUG "data": { 20:09:31 DEBUG "resultType": "vector", 20:09:31 DEBUG "result": [ 20:09:31 DEBUG { 20:09:31 DEBUG "metric": { 20:09:31 DEBUG "instance": "10.142.0.107:9100" 20:09:31 DEBUG }, 20:09:31 DEBUG "value": [ 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG [...] 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG "instance": "10.142.0.43:9100" 20:09:31 DEBUG }, 20:09:31 DEBUG "value": [ 20:09:31 DEBUG 1713553771, 20:09:31 DEBUG "0.29116666666663205" 20:09:31 DEBUG ] 20:09:31 DEBUG } 20:09:31 DEBUG ] 20:09:31 DEBUG } 20:09:31 DEBUG } 20:09:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553771" 20:09:31 INFO [http_cmd]: http status code OK 20:09:31 DEBUG --- status code --- 20:09:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:31 DEBUG --- http response --- 20:09:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:09:31 DEBUG ----- output ----- 20:09:31 DEBUG { 20:09:31 DEBUG "status": "success", 20:09:31 DEBUG "data": { 20:09:31 DEBUG "resultType": "vector", 20:09:31 DEBUG "result": [ 20:09:31 DEBUG { 20:09:31 DEBUG "metric": { 20:09:31 DEBUG "pod": "ds-idrepo-0" 20:09:31 DEBUG }, 20:09:31 DEBUG "value": [ 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG [...] 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 20:09:31 DEBUG }, 20:09:31 DEBUG "value": [ 20:09:31 DEBUG 1713553771, 20:09:31 DEBUG "0" 20:09:31 DEBUG ] 20:09:31 DEBUG } 20:09:31 DEBUG ] 20:09:31 DEBUG } 20:09:31 DEBUG } 20:09:31 INFO [http_cmd]: http status code OK 20:09:31 DEBUG --- status code --- 20:09:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:31 DEBUG --- http response --- 20:09:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:09:31 DEBUG ----- output ----- 20:09:31 DEBUG { 20:09:31 DEBUG "status": "success", 20:09:31 DEBUG "data": { 20:09:31 DEBUG "resultType": "vector", 20:09:31 DEBUG "result": [ 20:09:31 DEBUG { 20:09:31 DEBUG "metric": { 20:09:31 DEBUG "pod": "ds-idrepo-0" 20:09:31 DEBUG }, 20:09:31 DEBUG "value": [ 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG [...] 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG "pod": "ds-cts-2" 20:09:31 DEBUG }, 20:09:31 DEBUG "value": [ 20:09:31 DEBUG 1713553771, 20:09:31 DEBUG "0" 20:09:31 DEBUG ] 20:09:31 DEBUG } 20:09:31 DEBUG ] 20:09:31 DEBUG } 20:09:31 DEBUG } 20:09:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553771" 20:09:31 INFO [http_cmd]: http status code OK 20:09:31 DEBUG --- status code --- 20:09:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:31 DEBUG --- http response --- 20:09:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:09:31 DEBUG ----- output ----- 20:09:31 DEBUG { 20:09:31 DEBUG "status": "success", 20:09:31 DEBUG "data": { 20:09:31 DEBUG "resultType": "vector", 20:09:31 DEBUG "result": [ 20:09:31 DEBUG { 20:09:31 DEBUG "metric": { 20:09:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:09:31 DEBUG }, 20:09:31 DEBUG "value": [ 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG [...] 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:09:31 DEBUG }, 20:09:31 DEBUG "value": [ 20:09:31 DEBUG 1713553771, 20:09:31 DEBUG "0" 20:09:31 DEBUG ] 20:09:31 DEBUG } 20:09:31 DEBUG ] 20:09:31 DEBUG } 20:09:31 DEBUG } 20:09:31 INFO [http_cmd]: http status code OK 20:09:31 DEBUG --- status code --- 20:09:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:31 DEBUG --- http response --- 20:09:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:09:31 DEBUG ----- output ----- 20:09:31 DEBUG { 20:09:31 DEBUG "status": "success", 20:09:31 DEBUG "data": { 20:09:31 DEBUG "resultType": "vector", 20:09:31 DEBUG "result": [ 20:09:31 DEBUG { 20:09:31 DEBUG "metric": { 20:09:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:09:31 DEBUG }, 20:09:31 DEBUG "value": [ 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG [...] 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:09:31 DEBUG }, 20:09:31 DEBUG "value": [ 20:09:31 DEBUG 1713553771, 20:09:31 DEBUG "2506.866666666667" 20:09:31 DEBUG ] 20:09:31 DEBUG } 20:09:31 DEBUG ] 20:09:31 DEBUG } 20:09:31 DEBUG } 20:09:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713553771" 20:09:31 INFO [http_cmd]: http status code OK 20:09:31 DEBUG --- status code --- 20:09:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:31 DEBUG --- http response --- 20:09:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:09:31 DEBUG ----- output ----- 20:09:31 DEBUG { 20:09:31 DEBUG "status": "success", 20:09:31 DEBUG "data": { 20:09:31 DEBUG "resultType": "vector", 20:09:31 DEBUG "result": [ 20:09:31 DEBUG { 20:09:31 DEBUG "metric": { 20:09:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:09:31 DEBUG }, 20:09:31 DEBUG "value": [ 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG [...] 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:09:31 DEBUG }, 20:09:31 DEBUG "value": [ 20:09:31 DEBUG 1713553771, 20:09:31 DEBUG "0.015" 20:09:31 DEBUG ] 20:09:31 DEBUG } 20:09:31 DEBUG ] 20:09:31 DEBUG } 20:09:31 DEBUG } 20:09:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553771" 20:09:31 INFO [http_cmd]: http status code OK 20:09:31 DEBUG --- status code --- 20:09:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:31 DEBUG --- http response --- 20:09:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:09:31 DEBUG ----- output ----- 20:09:31 DEBUG { 20:09:31 DEBUG "status": "success", 20:09:31 DEBUG "data": { 20:09:31 DEBUG "resultType": "vector", 20:09:31 DEBUG "result": [ 20:09:31 DEBUG { 20:09:31 DEBUG "metric": { 20:09:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:09:31 DEBUG }, 20:09:31 DEBUG "value": [ 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG [...] 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:09:31 DEBUG }, 20:09:31 DEBUG "value": [ 20:09:31 DEBUG 1713553771, 20:09:31 DEBUG "2500.3333333333335" 20:09:31 DEBUG ] 20:09:31 DEBUG } 20:09:31 DEBUG ] 20:09:31 DEBUG } 20:09:31 DEBUG } 20:09:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553771" 20:09:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713553771" 20:09:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553771" 20:09:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553771" 20:09:31 INFO [http_cmd]: http status code OK 20:09:31 DEBUG --- status code --- 20:09:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:31 DEBUG --- http response --- 20:09:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 20:09:31 INFO [http_cmd]: http status code OK 20:09:31 DEBUG ----- output ----- 20:09:31 DEBUG --- status code --- 20:09:31 DEBUG { 20:09:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:31 DEBUG "status": "success", 20:09:31 DEBUG --- http response --- 20:09:31 DEBUG "data": { 20:09:31 DEBUG "resultType": "vector", 20:09:31 DEBUG "result": [ 20:09:31 DEBUG { 20:09:31 DEBUG "metric": { 20:09:31 DEBUG "pod": "ds-idrepo-0" 20:09:31 DEBUG }, 20:09:31 DEBUG "value": [ 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG [...] 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG "pod": "idm-b85f9d978-wnhxb" 20:09:31 DEBUG }, 20:09:31 DEBUG "value": [ 20:09:31 DEBUG 1713553771, 20:09:31 DEBUG "980.7006943096538" 20:09:31 DEBUG ] 20:09:31 DEBUG } 20:09:31 DEBUG ] 20:09:31 DEBUG } 20:09:31 DEBUG } 20:09:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:09:31 DEBUG ----- output ----- 20:09:31 DEBUG { 20:09:31 DEBUG "status": "success", 20:09:31 DEBUG "data": { 20:09:31 DEBUG "resultType": "vector", 20:09:31 DEBUG "result": [ 20:09:31 DEBUG { 20:09:31 DEBUG "metric": { 20:09:31 DEBUG "instance": "10.142.0.107:9100" 20:09:31 DEBUG }, 20:09:31 DEBUG "value": [ 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG 20:09:31 DEBUG [...] 20:09:31 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG "instance": "10.142.0.43:9100" 20:09:32 DEBUG }, 20:09:32 DEBUG "value": [ 20:09:32 DEBUG 1713553771, 20:09:32 DEBUG "75436155" 20:09:32 DEBUG ] 20:09:32 DEBUG } 20:09:32 DEBUG ] 20:09:32 DEBUG } 20:09:32 DEBUG } 20:09:32 INFO [http_cmd]: http status code OK 20:09:32 DEBUG --- status code --- 20:09:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:32 DEBUG --- http response --- 20:09:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:09:32 DEBUG ----- output ----- 20:09:32 DEBUG { 20:09:32 DEBUG "status": "success", 20:09:32 DEBUG "data": { 20:09:32 DEBUG "resultType": "vector", 20:09:32 DEBUG "result": [ 20:09:32 DEBUG { 20:09:32 DEBUG "metric": { 20:09:32 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:09:32 DEBUG }, 20:09:32 DEBUG "value": [ 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG [...] 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:09:32 DEBUG }, 20:09:32 DEBUG "value": [ 20:09:32 DEBUG 1713553771, 20:09:32 DEBUG "8.047677859236629" 20:09:32 DEBUG ] 20:09:32 DEBUG } 20:09:32 DEBUG ] 20:09:32 DEBUG } 20:09:32 DEBUG } 20:09:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553772" 20:09:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553772" 20:09:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553772" 20:09:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713553772" 20:09:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553772" 20:09:32 INFO [http_cmd]: http status code OK 20:09:32 DEBUG --- status code --- 20:09:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:32 DEBUG --- http response --- 20:09:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 20:09:32 DEBUG ----- output ----- 20:09:32 DEBUG { 20:09:32 DEBUG "status": "success", 20:09:32 DEBUG "data": { 20:09:32 DEBUG "resultType": "vector", 20:09:32 DEBUG "result": [ 20:09:32 DEBUG { 20:09:32 DEBUG "metric": { 20:09:32 DEBUG "pod": "ds-idrepo-0" 20:09:32 DEBUG }, 20:09:32 DEBUG "value": [ 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG [...] 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG "pod": "idm-b85f9d978-wnhxb" 20:09:32 DEBUG }, 20:09:32 DEBUG "value": [ 20:09:32 DEBUG 1713553771, 20:09:32 DEBUG "0" 20:09:32 DEBUG ] 20:09:32 DEBUG } 20:09:32 DEBUG ] 20:09:32 DEBUG } 20:09:32 DEBUG } 20:09:32 INFO [http_cmd]: http status code OK 20:09:32 DEBUG --- status code --- 20:09:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:32 DEBUG --- http response --- 20:09:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:09:32 DEBUG ----- output ----- 20:09:32 DEBUG { 20:09:32 DEBUG "status": "success", 20:09:32 DEBUG "data": { 20:09:32 DEBUG "resultType": "vector", 20:09:32 DEBUG "result": [ 20:09:32 DEBUG { 20:09:32 DEBUG "metric": { 20:09:32 DEBUG "pod": "ds-idrepo-0" 20:09:32 DEBUG }, 20:09:32 DEBUG "value": [ 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG [...] 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG "pod": "ds-idrepo-2" 20:09:32 DEBUG }, 20:09:32 DEBUG "value": [ 20:09:32 DEBUG 1713553772, 20:09:32 DEBUG "0" 20:09:32 DEBUG ] 20:09:32 DEBUG } 20:09:32 DEBUG ] 20:09:32 DEBUG } 20:09:32 DEBUG } 20:09:32 INFO [http_cmd]: http status code OK 20:09:32 INFO [http_cmd]: http status code OK 20:09:32 DEBUG --- status code --- 20:09:32 DEBUG --- status code --- 20:09:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:32 DEBUG --- http response --- 20:09:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553772" 20:09:32 DEBUG --- http response --- 20:09:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:09:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:09:32 DEBUG ----- output ----- 20:09:32 DEBUG ----- output ----- 20:09:32 DEBUG { 20:09:32 DEBUG { 20:09:32 DEBUG "status": "success", 20:09:32 DEBUG "status": "success", 20:09:32 DEBUG "data": { 20:09:32 DEBUG "data": { 20:09:32 DEBUG "resultType": "vector", 20:09:32 DEBUG "resultType": "vector", 20:09:32 DEBUG "result": [ 20:09:32 DEBUG "result": [ 20:09:32 DEBUG { 20:09:32 DEBUG { 20:09:32 DEBUG "metric": { 20:09:32 DEBUG "metric": { 20:09:32 DEBUG "instance": "10.142.0.107:9100" 20:09:32 DEBUG "pod": "ds-cts-0" 20:09:32 DEBUG }, 20:09:32 DEBUG }, 20:09:32 DEBUG "value": [ 20:09:32 DEBUG "value": [ 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG [...] 20:09:32 DEBUG [...] 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 INFO [http_cmd]: http status code OK 20:09:32 INFO [http_cmd]: http status code OK 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG --- status code --- 20:09:32 DEBUG --- status code --- 20:09:32 DEBUG "instance": "10.142.0.43:9100" 20:09:32 DEBUG "pod": "ds-cts-2" 20:09:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:32 DEBUG }, 20:09:32 DEBUG }, 20:09:32 DEBUG --- http response --- 20:09:32 DEBUG --- http response --- 20:09:32 DEBUG "value": [ 20:09:32 DEBUG "value": [ 20:09:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:09:32 DEBUG 1713553772, 20:09:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:09:32 DEBUG 1713553772, 20:09:32 DEBUG ----- output ----- 20:09:32 DEBUG "2032875621376" 20:09:32 DEBUG ----- output ----- 20:09:32 DEBUG "0" 20:09:32 DEBUG { 20:09:32 DEBUG ] 20:09:32 DEBUG { 20:09:32 DEBUG ] 20:09:32 DEBUG "status": "success", 20:09:32 DEBUG } 20:09:32 DEBUG "status": "success", 20:09:32 DEBUG } 20:09:32 DEBUG "data": { 20:09:32 DEBUG ] 20:09:32 DEBUG "data": { 20:09:32 DEBUG ] 20:09:32 DEBUG "resultType": "vector", 20:09:32 DEBUG } 20:09:32 DEBUG "resultType": "vector", 20:09:32 DEBUG } 20:09:32 DEBUG "result": [ 20:09:32 DEBUG } 20:09:32 DEBUG "result": [ 20:09:32 DEBUG } 20:09:32 DEBUG { 20:09:32 DEBUG { 20:09:32 DEBUG "metric": { 20:09:32 DEBUG "metric": { 20:09:32 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:09:32 DEBUG "instance": "10.142.0.107:9100" 20:09:32 DEBUG }, 20:09:32 DEBUG }, 20:09:32 DEBUG "value": [ 20:09:32 DEBUG "value": [ 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG [...] 20:09:32 DEBUG [...] 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:09:32 DEBUG "instance": "10.142.0.43:9100" 20:09:32 DEBUG }, 20:09:32 DEBUG }, 20:09:32 DEBUG "value": [ 20:09:32 DEBUG "value": [ 20:09:32 DEBUG 1713553772, 20:09:32 DEBUG 1713553772, 20:09:32 DEBUG "3672428544" 20:09:32 DEBUG "75438638.93333334" 20:09:32 DEBUG ] 20:09:32 DEBUG ] 20:09:32 DEBUG } 20:09:32 DEBUG } 20:09:32 DEBUG ] 20:09:32 DEBUG ] 20:09:32 DEBUG } 20:09:32 DEBUG } 20:09:32 DEBUG } 20:09:32 DEBUG } 20:09:32 INFO [http_cmd]: http status code OK 20:09:32 DEBUG --- status code --- 20:09:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:32 DEBUG --- http response --- 20:09:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:09:32 DEBUG ----- output ----- 20:09:32 DEBUG { 20:09:32 DEBUG "status": "success", 20:09:32 DEBUG "data": { 20:09:32 DEBUG "resultType": "vector", 20:09:32 DEBUG "result": [ 20:09:32 DEBUG { 20:09:32 DEBUG "metric": { 20:09:32 DEBUG "pod": "ds-cts-0" 20:09:32 DEBUG }, 20:09:32 DEBUG "value": [ 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG [...] 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG "pod": "ds-cts-2" 20:09:32 DEBUG }, 20:09:32 DEBUG "value": [ 20:09:32 DEBUG 1713553772, 20:09:32 DEBUG "29.966666666666665" 20:09:32 DEBUG ] 20:09:32 DEBUG } 20:09:32 DEBUG ] 20:09:32 DEBUG } 20:09:32 DEBUG } 20:09:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553772" 20:09:32 INFO [http_cmd]: http status code OK 20:09:32 DEBUG --- status code --- 20:09:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:32 DEBUG --- http response --- 20:09:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:09:32 DEBUG ----- output ----- 20:09:32 DEBUG { 20:09:32 DEBUG "status": "success", 20:09:32 DEBUG "data": { 20:09:32 DEBUG "resultType": "vector", 20:09:32 DEBUG "result": [ 20:09:32 DEBUG { 20:09:32 DEBUG "metric": { 20:09:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:09:32 DEBUG }, 20:09:32 DEBUG "value": [ 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG [...] 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:09:32 DEBUG }, 20:09:32 DEBUG "value": [ 20:09:32 DEBUG 1713553772, 20:09:32 DEBUG "2508.0333333333333" 20:09:32 DEBUG ] 20:09:32 DEBUG } 20:09:32 DEBUG ] 20:09:32 DEBUG } 20:09:32 DEBUG } 20:09:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553772" 20:09:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553772" 20:09:32 INFO [http_cmd]: http status code OK 20:09:32 DEBUG --- status code --- 20:09:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:32 DEBUG --- http response --- 20:09:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:09:32 DEBUG ----- output ----- 20:09:32 DEBUG { 20:09:32 DEBUG "status": "success", 20:09:32 DEBUG "data": { 20:09:32 DEBUG "resultType": "vector", 20:09:32 DEBUG "result": [ 20:09:32 DEBUG { 20:09:32 DEBUG "metric": { 20:09:32 DEBUG "pod": "ds-idrepo-0" 20:09:32 DEBUG }, 20:09:32 DEBUG "value": [ 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG [...] 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG "pod": "idm-b85f9d978-txpwv" 20:09:32 DEBUG }, 20:09:32 DEBUG "value": [ 20:09:32 DEBUG 1713553772, 20:09:32 DEBUG "0" 20:09:32 DEBUG ] 20:09:32 DEBUG } 20:09:32 DEBUG ] 20:09:32 DEBUG } 20:09:32 DEBUG } 20:09:32 INFO [http_cmd]: http status code OK 20:09:32 DEBUG --- status code --- 20:09:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:32 DEBUG --- http response --- 20:09:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:09:32 DEBUG ----- output ----- 20:09:32 DEBUG { 20:09:32 DEBUG "status": "success", 20:09:32 DEBUG "data": { 20:09:32 DEBUG "resultType": "vector", 20:09:32 DEBUG "result": [ 20:09:32 DEBUG { 20:09:32 DEBUG "metric": { 20:09:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:09:32 DEBUG }, 20:09:32 DEBUG "value": [ 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG [...] 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG 20:09:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:09:32 DEBUG }, 20:09:32 DEBUG "value": [ 20:09:32 DEBUG 1713553772, 20:09:32 DEBUG "2506.4" 20:09:32 DEBUG ] 20:09:32 DEBUG } 20:09:32 DEBUG ] 20:09:32 DEBUG } 20:09:32 DEBUG } 20:09:33 INFO 20:09:33 INFO [loop_until]: kubectl --namespace=xlou top pods 20:09:33 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:09:33 INFO [loop_until]: OK (rc = 0) 20:09:33 DEBUG --- stdout --- 20:09:33 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 16220m 9011Mi am-767f6b67b5-qcqz2 16374m 8812Mi am-767f6b67b5-vbltz 18168m 8997Mi ds-cts-0 17362m 10976Mi ds-cts-1 15578m 11221Mi ds-cts-2 17997m 10984Mi ds-idrepo-0 7036m 3692Mi ds-idrepo-1 7170m 3536Mi ds-idrepo-2 7362m 3493Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8071m 3412Mi 20:09:33 DEBUG --- stderr --- 20:09:33 DEBUG 20:09:34 INFO 20:09:34 INFO [loop_until]: kubectl --namespace=xlou top node 20:09:34 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:09:35 INFO [loop_until]: OK (rc = 0) 20:09:35 DEBUG --- stdout --- 20:09:35 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 70m 0% 6243Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7404m 24% 7293Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16137m 54% 11713Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 85m 0% 3355Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 17223m 57% 13958Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 160m 0% 5183Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 16451m 55% 11695Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15510m 51% 2991Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 18116m 60% 11207Mi 10% gke-xlou-cdm-ds-0f601751-j488 7647m 25% 6728Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 15542m 52% 15977Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 7474m 25% 7439Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 17831m 59% 13420Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 8198m 51% 7146Mi 12% 20:09:35 DEBUG --- stderr --- 20:09:35 DEBUG 20:10:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553831" 20:10:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553831" 20:10:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713553831" 20:10:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553831" 20:10:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553831" 20:10:31 INFO [http_cmd]: http status code OK 20:10:31 INFO [http_cmd]: http status code OK 20:10:31 DEBUG --- status code --- 20:10:31 DEBUG --- status code --- 20:10:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:31 DEBUG --- http response --- 20:10:31 DEBUG --- http response --- 20:10:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:10:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:10:31 DEBUG ----- output ----- 20:10:31 DEBUG ----- output ----- 20:10:31 DEBUG { 20:10:31 DEBUG { 20:10:31 DEBUG "status": "success", 20:10:31 DEBUG "status": "success", 20:10:31 DEBUG "data": { 20:10:31 DEBUG "data": { 20:10:31 DEBUG "resultType": "vector", 20:10:31 DEBUG "resultType": "vector", 20:10:31 DEBUG "result": [ 20:10:31 DEBUG "result": [ 20:10:31 DEBUG { 20:10:31 DEBUG { 20:10:31 DEBUG "metric": { 20:10:31 DEBUG "metric": { 20:10:31 DEBUG "pod": "ds-idrepo-0" 20:10:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:10:31 DEBUG }, 20:10:31 DEBUG }, 20:10:31 DEBUG "value": [ 20:10:31 DEBUG "value": [ 20:10:31 DEBUG 20:10:31 DEBUG 20:10:31 DEBUG 20:10:31 DEBUG 20:10:31 DEBUG 20:10:31 DEBUG 20:10:31 DEBUG [...] 20:10:31 DEBUG [...] 20:10:31 DEBUG 20:10:31 DEBUG 20:10:31 DEBUG 20:10:31 DEBUG 20:10:31 DEBUG 20:10:31 DEBUG 20:10:31 DEBUG "pod": "ds-idrepo-2" 20:10:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:10:31 DEBUG }, 20:10:31 DEBUG }, 20:10:31 DEBUG "value": [ 20:10:31 DEBUG "value": [ 20:10:31 DEBUG 1713553831, 20:10:31 DEBUG 1713553831, 20:10:31 DEBUG "0" 20:10:31 DEBUG "450688" 20:10:31 DEBUG ] 20:10:31 DEBUG ] 20:10:31 DEBUG } 20:10:31 DEBUG } 20:10:31 DEBUG ] 20:10:31 DEBUG ] 20:10:31 DEBUG } 20:10:31 DEBUG } 20:10:31 DEBUG } 20:10:31 DEBUG } 20:10:31 INFO [http_cmd]: http status code OK 20:10:31 ERROR [http_cmd]: ERROR 20:10:31 DEBUG --- status code --- 20:10:31 ERROR --- status code --- 20:10:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:31 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:10:31 DEBUG --- http response --- 20:10:31 ERROR --- http response --- 20:10:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:10:31 DEBUG ----- output ----- 20:10:31 DEBUG { 20:10:31 DEBUG "status": "success", 20:10:31 DEBUG "data": { 20:10:31 DEBUG "resultType": "vector", 20:10:31 DEBUG "result": [ 20:10:31 DEBUG { 20:10:31 DEBUG "metric": { 20:10:31 DEBUG "pod": "ds-idrepo-0" 20:10:31 DEBUG }, 20:10:31 DEBUG "value": [ 20:10:31 DEBUG 20:10:31 DEBUG 20:10:31 DEBUG 20:10:31 DEBUG [...] 20:10:31 DEBUG 20:10:31 DEBUG 20:10:31 DEBUG 20:10:31 DEBUG "pod": "ds-idrepo-2" 20:10:31 DEBUG }, 20:10:31 DEBUG "value": [ 20:10:31 DEBUG 1713553831, 20:10:31 DEBUG "0" 20:10:31 DEBUG ] 20:10:31 DEBUG } 20:10:31 DEBUG ] 20:10:31 DEBUG } 20:10:31 DEBUG } 20:10:31 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:10:31 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:10:31 INFO [http_cmd]: http status code OK 20:10:31 DEBUG --- status code --- 20:10:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:31 DEBUG --- http response --- 20:10:31 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:10:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553831" 20:10:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553831" 20:10:31 INFO [http_cmd]: http status code OK 20:10:31 DEBUG --- status code --- 20:10:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:31 DEBUG --- http response --- 20:10:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:10:31 DEBUG ----- output ----- 20:10:31 DEBUG { 20:10:31 DEBUG "status": "success", 20:10:31 DEBUG "data": { 20:10:31 DEBUG "resultType": "vector", 20:10:31 DEBUG "result": [ 20:10:31 DEBUG { 20:10:31 DEBUG "metric": { 20:10:31 DEBUG "instance": "10.142.0.107:9100" 20:10:31 DEBUG }, 20:10:31 DEBUG "value": [ 20:10:31 DEBUG 20:10:31 DEBUG 20:10:31 DEBUG 20:10:31 DEBUG [...] 20:10:31 DEBUG 20:10:31 DEBUG 20:10:31 DEBUG 20:10:31 DEBUG "instance": "10.142.0.43:9100" 20:10:31 DEBUG }, 20:10:31 DEBUG "value": [ 20:10:31 DEBUG 1713553831, 20:10:31 DEBUG "0.24366666666663547" 20:10:31 DEBUG ] 20:10:31 DEBUG } 20:10:31 DEBUG ] 20:10:31 DEBUG } 20:10:31 DEBUG } 20:10:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553831" 20:10:31 INFO [http_cmd]: http status code OK 20:10:31 DEBUG --- status code --- 20:10:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:31 DEBUG --- http response --- 20:10:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:10:31 DEBUG ----- output ----- 20:10:31 DEBUG { 20:10:31 DEBUG "status": "success", 20:10:31 DEBUG "data": { 20:10:31 DEBUG "resultType": "vector", 20:10:31 DEBUG "result": [ 20:10:31 DEBUG { 20:10:31 DEBUG "metric": { 20:10:31 DEBUG "pod": "ds-cts-0" 20:10:31 DEBUG }, 20:10:31 DEBUG "value": [ 20:10:31 DEBUG 20:10:31 DEBUG 20:10:31 DEBUG 20:10:31 DEBUG [...] 20:10:31 DEBUG 20:10:31 DEBUG 20:10:31 DEBUG 20:10:31 DEBUG "pod": "ds-cts-2" 20:10:31 DEBUG }, 20:10:31 DEBUG "value": [ 20:10:31 DEBUG 1713553831, 20:10:31 DEBUG "0.0004945466666666666" 20:10:31 DEBUG ] 20:10:31 DEBUG } 20:10:31 DEBUG ] 20:10:31 DEBUG } 20:10:31 DEBUG } 20:10:31 INFO [http_cmd]: http status code OK 20:10:31 DEBUG --- status code --- 20:10:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:31 DEBUG --- http response --- 20:10:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:10:31 DEBUG ----- output ----- 20:10:31 DEBUG { 20:10:31 DEBUG "status": "success", 20:10:31 DEBUG "data": { 20:10:31 DEBUG "resultType": "vector", 20:10:31 DEBUG "result": [ 20:10:31 DEBUG { 20:10:31 DEBUG "metric": { 20:10:31 DEBUG "instance": "10.142.0.107:9100" 20:10:31 DEBUG }, 20:10:31 DEBUG "value": [ 20:10:31 DEBUG 20:10:31 DEBUG 20:10:31 DEBUG 20:10:31 DEBUG [...] 20:10:31 DEBUG 20:10:31 DEBUG 20:10:31 DEBUG 20:10:31 DEBUG "instance": "10.142.0.43:9100" 20:10:31 DEBUG }, 20:10:31 DEBUG "value": [ 20:10:31 DEBUG 1713553831, 20:10:31 DEBUG "0.10333333333333786" 20:10:31 DEBUG ] 20:10:31 DEBUG } 20:10:31 DEBUG ] 20:10:31 DEBUG } 20:10:31 DEBUG } 20:10:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713553831" 20:10:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553831" 20:10:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713553831" 20:10:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553831" 20:10:31 INFO [http_cmd]: http status code OK 20:10:31 DEBUG --- status code --- 20:10:31 INFO [http_cmd]: http status code OK 20:10:31 INFO [http_cmd]: http status code OK 20:10:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:31 DEBUG --- status code --- 20:10:31 DEBUG --- status code --- 20:10:31 DEBUG --- http response --- 20:10:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:31 DEBUG --- http response --- 20:10:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:10:31 DEBUG --- http response --- 20:10:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:10:31 DEBUG ----- output ----- 20:10:31 DEBUG ----- output ----- 20:10:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 20:10:31 DEBUG { 20:10:31 DEBUG { 20:10:31 DEBUG ----- output ----- 20:10:31 DEBUG "status": "success", 20:10:31 DEBUG "status": "success", 20:10:31 DEBUG { 20:10:31 DEBUG "data": { 20:10:31 DEBUG "data": { 20:10:31 DEBUG "status": "success", 20:10:31 DEBUG "resultType": "vector", 20:10:31 DEBUG "resultType": "vector", 20:10:31 DEBUG "data": { 20:10:31 DEBUG "result": [ 20:10:31 DEBUG "result": [ 20:10:31 DEBUG "resultType": "vector", 20:10:31 DEBUG { 20:10:31 DEBUG { 20:10:31 DEBUG "result": [ 20:10:31 DEBUG "metric": { 20:10:31 DEBUG "metric": { 20:10:31 DEBUG { 20:10:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:10:31 DEBUG "pod": "ds-idrepo-0" 20:10:31 DEBUG "metric": { 20:10:31 DEBUG }, 20:10:31 DEBUG }, 20:10:31 DEBUG "pod": "ds-idrepo-0" 20:10:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553831" 20:10:31 DEBUG "value": [ 20:10:31 DEBUG "value": [ 20:10:31 DEBUG }, 20:10:31 DEBUG 20:10:31 DEBUG 20:10:31 DEBUG "value": [ 20:10:31 DEBUG 20:10:31 DEBUG 20:10:31 DEBUG 20:10:31 DEBUG 20:10:31 DEBUG 20:10:31 DEBUG 20:10:31 DEBUG [...] 20:10:31 DEBUG [...] 20:10:31 DEBUG 20:10:31 DEBUG 20:10:31 DEBUG 20:10:31 DEBUG [...] 20:10:31 DEBUG 20:10:31 DEBUG 20:10:31 DEBUG 20:10:31 DEBUG 20:10:31 DEBUG 20:10:31 DEBUG 20:10:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:10:31 DEBUG "pod": "ds-cts-2" 20:10:31 DEBUG 20:10:31 DEBUG }, 20:10:31 DEBUG }, 20:10:31 DEBUG "pod": "am-767f6b67b5-qcqz2" 20:10:31 DEBUG "value": [ 20:10:31 DEBUG "value": [ 20:10:31 DEBUG }, 20:10:31 DEBUG 1713553831, 20:10:31 DEBUG 1713553831, 20:10:31 DEBUG "value": [ 20:10:31 DEBUG "0.015" 20:10:31 DEBUG "0" 20:10:31 DEBUG 1713553831, 20:10:31 DEBUG ] 20:10:31 DEBUG ] 20:10:31 DEBUG "337449939.0132564" 20:10:31 DEBUG } 20:10:31 DEBUG } 20:10:31 DEBUG ] 20:10:31 DEBUG ] 20:10:31 DEBUG ] 20:10:31 DEBUG } 20:10:31 DEBUG } 20:10:31 DEBUG } 20:10:31 DEBUG ] 20:10:31 DEBUG } 20:10:31 DEBUG } 20:10:31 DEBUG } 20:10:31 DEBUG } 20:10:31 INFO [http_cmd]: http status code OK 20:10:31 DEBUG --- status code --- 20:10:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:31 DEBUG --- http response --- 20:10:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:10:31 DEBUG ----- output ----- 20:10:31 DEBUG { 20:10:31 DEBUG "status": "success", 20:10:31 DEBUG "data": { 20:10:31 DEBUG "resultType": "vector", 20:10:31 DEBUG "result": [ 20:10:31 DEBUG { 20:10:31 DEBUG "metric": { 20:10:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:10:31 DEBUG }, 20:10:31 DEBUG "value": [ 20:10:31 DEBUG 20:10:31 DEBUG 20:10:31 DEBUG 20:10:31 DEBUG [...] 20:10:31 DEBUG 20:10:31 DEBUG 20:10:31 DEBUG 20:10:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:10:31 DEBUG }, 20:10:31 DEBUG "value": [ 20:10:31 DEBUG 1713553831, 20:10:31 DEBUG "0" 20:10:31 DEBUG ] 20:10:31 DEBUG } 20:10:31 DEBUG ] 20:10:31 DEBUG } 20:10:31 DEBUG } 20:10:31 INFO [http_cmd]: http status code OK 20:10:31 DEBUG --- status code --- 20:10:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:31 DEBUG --- http response --- 20:10:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:10:31 DEBUG ----- output ----- 20:10:31 DEBUG { 20:10:31 DEBUG "status": "success", 20:10:31 DEBUG "data": { 20:10:31 DEBUG "resultType": "vector", 20:10:31 DEBUG "result": [ 20:10:31 DEBUG { 20:10:31 DEBUG "metric": { 20:10:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:10:31 DEBUG }, 20:10:31 DEBUG "value": [ 20:10:31 DEBUG 20:10:31 DEBUG 20:10:31 DEBUG 20:10:31 DEBUG [...] 20:10:31 DEBUG 20:10:31 DEBUG 20:10:31 DEBUG 20:10:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:10:31 DEBUG }, 20:10:31 DEBUG "value": [ 20:10:31 DEBUG 1713553831, 20:10:31 DEBUG "2498" 20:10:31 DEBUG ] 20:10:31 DEBUG } 20:10:31 DEBUG ] 20:10:31 DEBUG } 20:10:31 DEBUG } 20:10:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553832" 20:10:32 INFO [http_cmd]: http status code OK 20:10:32 DEBUG --- status code --- 20:10:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:32 DEBUG --- http response --- 20:10:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:10:32 DEBUG ----- output ----- 20:10:32 DEBUG { 20:10:32 DEBUG "status": "success", 20:10:32 DEBUG "data": { 20:10:32 DEBUG "resultType": "vector", 20:10:32 DEBUG "result": [ 20:10:32 DEBUG { 20:10:32 DEBUG "metric": { 20:10:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:10:32 DEBUG }, 20:10:32 DEBUG "value": [ 20:10:32 DEBUG 20:10:32 DEBUG 20:10:32 DEBUG 20:10:32 DEBUG [...] 20:10:32 DEBUG 20:10:32 DEBUG 20:10:32 DEBUG 20:10:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:10:32 DEBUG }, 20:10:32 DEBUG "value": [ 20:10:32 DEBUG 1713553832, 20:10:32 DEBUG "2505.733333333333" 20:10:32 DEBUG ] 20:10:32 DEBUG } 20:10:32 DEBUG ] 20:10:32 DEBUG } 20:10:32 DEBUG } 20:10:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553832" 20:10:32 INFO [http_cmd]: http status code OK 20:10:32 DEBUG --- status code --- 20:10:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:32 DEBUG --- http response --- 20:10:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:10:32 DEBUG ----- output ----- 20:10:32 DEBUG { 20:10:32 DEBUG "status": "success", 20:10:32 DEBUG "data": { 20:10:32 DEBUG "resultType": "vector", 20:10:32 DEBUG "result": [ 20:10:32 DEBUG { 20:10:32 DEBUG "metric": { 20:10:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:10:32 DEBUG }, 20:10:32 DEBUG "value": [ 20:10:32 DEBUG 20:10:32 DEBUG 20:10:32 DEBUG 20:10:32 DEBUG [...] 20:10:32 DEBUG 20:10:32 DEBUG 20:10:32 DEBUG 20:10:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:10:32 DEBUG }, 20:10:32 DEBUG "value": [ 20:10:32 DEBUG 1713553832, 20:10:32 DEBUG "2496.4" 20:10:32 DEBUG ] 20:10:32 DEBUG } 20:10:32 DEBUG ] 20:10:32 DEBUG } 20:10:32 DEBUG } 20:10:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553832" 20:10:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553832" 20:10:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713553832" 20:10:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553832" 20:10:32 INFO [http_cmd]: http status code OK 20:10:32 INFO [http_cmd]: http status code OK 20:10:32 DEBUG --- status code --- 20:10:32 DEBUG --- status code --- 20:10:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:32 DEBUG --- http response --- 20:10:32 DEBUG --- http response --- 20:10:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553832" 20:10:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:10:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553832" 20:10:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 20:10:32 INFO [http_cmd]: http status code OK 20:10:32 DEBUG ----- output ----- 20:10:32 INFO [http_cmd]: http status code OK 20:10:32 DEBUG ----- output ----- 20:10:32 DEBUG --- status code --- 20:10:32 DEBUG { 20:10:32 DEBUG --- status code --- 20:10:32 DEBUG { 20:10:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:32 DEBUG "status": "success", 20:10:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553832" 20:10:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:32 DEBUG "status": "success", 20:10:32 DEBUG --- http response --- 20:10:32 DEBUG "data": { 20:10:32 DEBUG --- http response --- 20:10:32 DEBUG "data": { 20:10:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:10:32 DEBUG "resultType": "vector", 20:10:32 DEBUG "resultType": "vector", 20:10:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:10:32 DEBUG ----- output ----- 20:10:32 DEBUG "result": [ 20:10:32 DEBUG "result": [ 20:10:32 DEBUG ----- output ----- 20:10:32 DEBUG { 20:10:32 DEBUG { 20:10:32 DEBUG { 20:10:32 DEBUG { 20:10:32 DEBUG "status": "success", 20:10:32 DEBUG "metric": { 20:10:32 DEBUG "metric": { 20:10:32 DEBUG "status": "success", 20:10:32 DEBUG "data": { 20:10:32 DEBUG "instance": "10.142.0.107:9100" 20:10:32 DEBUG "pod": "ds-idrepo-0" 20:10:32 DEBUG "data": { 20:10:32 DEBUG "resultType": "vector", 20:10:32 DEBUG }, 20:10:32 DEBUG }, 20:10:32 DEBUG "resultType": "vector", 20:10:32 DEBUG "result": [ 20:10:32 DEBUG "value": [ 20:10:32 DEBUG "value": [ 20:10:32 DEBUG "result": [ 20:10:32 DEBUG { 20:10:32 DEBUG 20:10:32 DEBUG 20:10:32 DEBUG { 20:10:32 DEBUG "metric": { 20:10:32 DEBUG 20:10:32 DEBUG 20:10:32 DEBUG "metric": { 20:10:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553832" 20:10:32 DEBUG "instance": "10.142.0.107:9100" 20:10:32 DEBUG 20:10:32 DEBUG 20:10:32 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:10:32 DEBUG }, 20:10:32 DEBUG [...] 20:10:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553832" 20:10:32 DEBUG [...] 20:10:32 DEBUG }, 20:10:32 DEBUG "value": [ 20:10:32 DEBUG 20:10:32 DEBUG 20:10:32 DEBUG "value": [ 20:10:32 DEBUG 20:10:32 INFO [http_cmd]: http status code OK 20:10:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713553832" 20:10:32 DEBUG 20:10:32 DEBUG 20:10:32 DEBUG 20:10:32 DEBUG 20:10:32 DEBUG --- status code --- 20:10:32 DEBUG 20:10:32 DEBUG 20:10:32 DEBUG 20:10:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553832" 20:10:32 INFO [http_cmd]: http status code OK 20:10:32 DEBUG 20:10:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:32 INFO [http_cmd]: http status code OK 20:10:32 DEBUG "instance": "10.142.0.43:9100" 20:10:32 DEBUG "pod": "am-767f6b67b5-qcqz2" 20:10:32 DEBUG 20:10:32 DEBUG --- status code --- 20:10:32 DEBUG [...] 20:10:32 INFO [http_cmd]: http status code OK 20:10:32 DEBUG --- http response --- 20:10:32 DEBUG --- status code --- 20:10:32 DEBUG }, 20:10:32 DEBUG }, 20:10:32 DEBUG [...] 20:10:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:32 DEBUG 20:10:32 DEBUG --- status code --- 20:10:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 20:10:32 DEBUG "value": [ 20:10:32 DEBUG "value": [ 20:10:32 DEBUG 20:10:32 DEBUG --- http response --- 20:10:32 DEBUG 20:10:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:32 DEBUG --- http response --- 20:10:32 DEBUG ----- output ----- 20:10:32 DEBUG 1713553832, 20:10:32 DEBUG 1713553832, 20:10:32 DEBUG "598.3274294270167" 20:10:32 DEBUG 20:10:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:10:32 DEBUG --- http response --- 20:10:32 DEBUG { 20:10:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:10:32 DEBUG "75663314.13333333" 20:10:32 DEBUG 20:10:32 DEBUG ] 20:10:32 DEBUG "instance": "10.142.0.43:9100" 20:10:32 DEBUG ----- output ----- 20:10:32 DEBUG "status": "success", 20:10:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:10:32 DEBUG ----- output ----- 20:10:32 DEBUG ] 20:10:32 DEBUG 20:10:32 DEBUG } 20:10:32 DEBUG }, 20:10:32 DEBUG { 20:10:32 DEBUG "data": { 20:10:32 DEBUG ----- output ----- 20:10:32 DEBUG { 20:10:32 DEBUG } 20:10:32 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:10:32 DEBUG ] 20:10:32 DEBUG "value": [ 20:10:32 DEBUG "status": "success", 20:10:32 DEBUG "resultType": "vector", 20:10:32 DEBUG { 20:10:32 DEBUG "status": "success", 20:10:32 DEBUG ] 20:10:32 DEBUG }, 20:10:32 DEBUG } 20:10:32 INFO [http_cmd]: http status code OK 20:10:32 DEBUG 1713553832, 20:10:32 DEBUG "data": { 20:10:32 DEBUG "result": [ 20:10:32 DEBUG "status": "success", 20:10:32 DEBUG "data": { 20:10:32 DEBUG } 20:10:32 DEBUG "value": [ 20:10:32 DEBUG } 20:10:32 DEBUG --- status code --- 20:10:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:32 DEBUG "resultType": "vector", 20:10:32 DEBUG { 20:10:32 DEBUG "data": { 20:10:32 DEBUG "resultType": "vector", 20:10:32 DEBUG } 20:10:32 DEBUG 1713553832, 20:10:32 INFO [http_cmd]: http status code OK 20:10:32 DEBUG "75666009.30000001" 20:10:32 DEBUG --- http response --- 20:10:32 DEBUG "result": [ 20:10:32 DEBUG "metric": { 20:10:32 DEBUG "resultType": "vector", 20:10:32 DEBUG "result": [ 20:10:32 DEBUG "8.04369509180729" 20:10:32 DEBUG --- status code --- 20:10:32 DEBUG ] 20:10:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:10:32 DEBUG { 20:10:32 DEBUG "pod": "ds-idrepo-0" 20:10:32 DEBUG "result": [ 20:10:32 DEBUG { 20:10:32 DEBUG ] 20:10:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:32 DEBUG } 20:10:32 DEBUG ----- output ----- 20:10:32 INFO [http_cmd]: http status code OK 20:10:32 DEBUG "metric": { 20:10:32 DEBUG }, 20:10:32 DEBUG { 20:10:32 DEBUG "metric": { 20:10:32 DEBUG } 20:10:32 DEBUG --- http response --- 20:10:32 DEBUG ] 20:10:32 DEBUG { 20:10:32 DEBUG --- status code --- 20:10:32 DEBUG "pod": "ds-idrepo-0" 20:10:32 DEBUG "value": [ 20:10:32 DEBUG "metric": { 20:10:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:10:32 DEBUG ] 20:10:32 DEBUG } 20:10:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:10:32 DEBUG "status": "success", 20:10:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:32 DEBUG }, 20:10:32 DEBUG 20:10:32 DEBUG "instance": "10.142.0.107:9100" 20:10:32 DEBUG }, 20:10:32 DEBUG } 20:10:32 DEBUG } 20:10:32 DEBUG ----- output ----- 20:10:32 DEBUG "data": { 20:10:32 DEBUG --- http response --- 20:10:32 DEBUG "value": [ 20:10:32 DEBUG 20:10:32 DEBUG }, 20:10:32 DEBUG "value": [ 20:10:32 DEBUG } 20:10:32 DEBUG { 20:10:32 DEBUG "resultType": "vector", 20:10:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:10:32 DEBUG 20:10:32 DEBUG 20:10:32 DEBUG "value": [ 20:10:32 DEBUG 20:10:32 DEBUG "status": "success", 20:10:32 DEBUG "result": [ 20:10:32 DEBUG ----- output ----- 20:10:32 DEBUG 20:10:32 DEBUG [...] 20:10:32 DEBUG 20:10:32 DEBUG 20:10:32 DEBUG "data": { 20:10:32 DEBUG { 20:10:32 DEBUG { 20:10:32 DEBUG 20:10:32 DEBUG 20:10:32 DEBUG 20:10:32 DEBUG 20:10:32 DEBUG "resultType": "vector", 20:10:32 DEBUG "metric": { 20:10:32 DEBUG "status": "success", 20:10:32 DEBUG [...] 20:10:32 DEBUG 20:10:32 DEBUG 20:10:32 DEBUG [...] 20:10:32 DEBUG "result": [ 20:10:32 DEBUG "pod": "ds-cts-0" 20:10:32 DEBUG "data": { 20:10:32 DEBUG 20:10:32 DEBUG 20:10:32 DEBUG [...] 20:10:32 DEBUG 20:10:32 DEBUG { 20:10:32 DEBUG }, 20:10:32 DEBUG "resultType": "vector", 20:10:32 DEBUG 20:10:32 DEBUG "pod": "am-767f6b67b5-qcqz2" 20:10:32 DEBUG 20:10:32 DEBUG 20:10:32 DEBUG "metric": { 20:10:32 DEBUG "value": [ 20:10:32 DEBUG "result": [ 20:10:32 DEBUG 20:10:32 DEBUG }, 20:10:32 DEBUG 20:10:32 DEBUG 20:10:32 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:10:32 DEBUG 20:10:32 DEBUG { 20:10:32 DEBUG "pod": "ds-idrepo-2" 20:10:32 DEBUG "value": [ 20:10:32 DEBUG 20:10:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:10:32 DEBUG }, 20:10:32 DEBUG 20:10:32 DEBUG "metric": { 20:10:32 DEBUG }, 20:10:32 DEBUG 1713553832, 20:10:32 DEBUG "instance": "10.142.0.43:9100" 20:10:32 DEBUG }, 20:10:32 DEBUG "value": [ 20:10:32 DEBUG 20:10:32 DEBUG "pod": "ds-cts-0" 20:10:32 DEBUG "value": [ 20:10:32 DEBUG "0" 20:10:32 DEBUG }, 20:10:32 DEBUG "value": [ 20:10:32 DEBUG 20:10:32 DEBUG [...] 20:10:32 DEBUG }, 20:10:32 DEBUG 1713553832, 20:10:32 DEBUG ] 20:10:32 DEBUG "value": [ 20:10:32 DEBUG 1713553832, 20:10:32 DEBUG 20:10:32 DEBUG 20:10:32 DEBUG "value": [ 20:10:32 DEBUG "0" 20:10:32 DEBUG } 20:10:32 DEBUG 1713553832, 20:10:32 DEBUG "2499.0333333333333" 20:10:32 DEBUG 20:10:32 DEBUG 20:10:32 DEBUG 20:10:32 DEBUG ] 20:10:32 DEBUG ] 20:10:32 DEBUG "2036862870528" 20:10:32 DEBUG ] 20:10:32 DEBUG [...] 20:10:32 DEBUG 20:10:32 DEBUG 20:10:32 DEBUG } 20:10:32 DEBUG } 20:10:32 DEBUG ] 20:10:32 DEBUG } 20:10:32 DEBUG 20:10:32 DEBUG "pod": "ds-cts-2" 20:10:32 DEBUG 20:10:32 DEBUG ] 20:10:32 DEBUG } 20:10:32 DEBUG } 20:10:32 DEBUG ] 20:10:32 DEBUG 20:10:32 DEBUG }, 20:10:32 DEBUG [...] 20:10:32 DEBUG } 20:10:32 DEBUG ] 20:10:32 DEBUG } 20:10:32 DEBUG 20:10:32 DEBUG "value": [ 20:10:32 DEBUG 20:10:32 DEBUG } 20:10:32 DEBUG } 20:10:32 DEBUG } 20:10:32 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:10:32 DEBUG 1713553832, 20:10:32 DEBUG 20:10:32 DEBUG } 20:10:32 DEBUG }, 20:10:32 DEBUG "0" 20:10:32 DEBUG 20:10:32 DEBUG "value": [ 20:10:32 DEBUG ] 20:10:32 DEBUG "pod": "ds-cts-2" 20:10:32 DEBUG 1713553832, 20:10:32 DEBUG } 20:10:32 DEBUG }, 20:10:32 DEBUG "3675189248" 20:10:32 DEBUG ] 20:10:32 DEBUG "value": [ 20:10:32 DEBUG ] 20:10:32 DEBUG } 20:10:32 DEBUG 1713553832, 20:10:32 DEBUG } 20:10:32 DEBUG } 20:10:32 DEBUG "30" 20:10:32 DEBUG ] 20:10:32 DEBUG ] 20:10:32 DEBUG } 20:10:32 DEBUG } 20:10:32 DEBUG } 20:10:32 DEBUG ] 20:10:32 DEBUG } 20:10:32 DEBUG } 20:10:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553832" 20:10:32 INFO [http_cmd]: http status code OK 20:10:32 DEBUG --- status code --- 20:10:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:32 DEBUG --- http response --- 20:10:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 20:10:32 DEBUG ----- output ----- 20:10:32 DEBUG { 20:10:32 DEBUG "status": "success", 20:10:32 DEBUG "data": { 20:10:32 DEBUG "resultType": "vector", 20:10:32 DEBUG "result": [ 20:10:32 DEBUG { 20:10:32 DEBUG "metric": { 20:10:32 DEBUG "pod": "ds-idrepo-0" 20:10:32 DEBUG }, 20:10:32 DEBUG "value": [ 20:10:32 DEBUG 20:10:32 DEBUG 20:10:32 DEBUG 20:10:32 DEBUG [...] 20:10:32 DEBUG 20:10:32 DEBUG 20:10:32 DEBUG 20:10:32 DEBUG "pod": "am-767f6b67b5-qcqz2" 20:10:32 DEBUG }, 20:10:32 DEBUG "value": [ 20:10:32 DEBUG 1713553832, 20:10:32 DEBUG "16.341055362816373" 20:10:32 DEBUG ] 20:10:32 DEBUG } 20:10:32 DEBUG ] 20:10:32 DEBUG } 20:10:32 DEBUG } 20:10:33 INFO 20:10:33 INFO [loop_until]: kubectl --namespace=xlou top pods 20:10:33 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:10:33 INFO [loop_until]: OK (rc = 0) 20:10:33 DEBUG --- stdout --- 20:10:33 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 16301m 9010Mi am-767f6b67b5-qcqz2 16031m 8812Mi am-767f6b67b5-vbltz 17760m 8998Mi ds-cts-0 17161m 10982Mi ds-cts-1 15573m 11223Mi ds-cts-2 17615m 10971Mi ds-idrepo-0 7125m 3692Mi ds-idrepo-1 7155m 3537Mi ds-idrepo-2 7144m 3493Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 4m 1217Mi lodemon-f8459c597-6tj6f 1m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7970m 3414Mi 20:10:33 DEBUG --- stderr --- 20:10:33 DEBUG 20:10:35 INFO 20:10:35 INFO [loop_until]: kubectl --namespace=xlou top node 20:10:35 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:10:35 INFO [loop_until]: OK (rc = 0) 20:10:35 DEBUG --- stdout --- 20:10:35 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 71m 0% 6250Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7420m 24% 7290Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16116m 53% 11717Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 87m 0% 3352Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 16956m 56% 13948Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 163m 0% 5175Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 16477m 55% 11699Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15938m 53% 2992Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 18258m 61% 11210Mi 10% gke-xlou-cdm-ds-0f601751-j488 7596m 25% 6735Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 15650m 52% 15971Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 7471m 25% 7427Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 17634m 59% 13428Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 8195m 51% 7145Mi 12% 20:10:35 DEBUG --- stderr --- 20:10:35 DEBUG 20:11:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553891" 20:11:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553891" 20:11:31 INFO [http_cmd]: http status code OK 20:11:31 DEBUG --- status code --- 20:11:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:31 DEBUG --- http response --- 20:11:31 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:11:31 INFO [http_cmd]: http status code OK 20:11:31 DEBUG --- status code --- 20:11:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:31 DEBUG --- http response --- 20:11:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:11:31 DEBUG ----- output ----- 20:11:31 DEBUG { 20:11:31 DEBUG "status": "success", 20:11:31 DEBUG "data": { 20:11:31 DEBUG "resultType": "vector", 20:11:31 DEBUG "result": [ 20:11:31 DEBUG { 20:11:31 DEBUG "metric": { 20:11:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:11:31 DEBUG }, 20:11:31 DEBUG "value": [ 20:11:31 DEBUG 20:11:31 DEBUG 20:11:31 DEBUG 20:11:31 DEBUG [...] 20:11:31 DEBUG 20:11:31 DEBUG 20:11:31 DEBUG 20:11:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:11:31 DEBUG }, 20:11:31 DEBUG "value": [ 20:11:31 DEBUG 1713553891, 20:11:31 DEBUG "450717" 20:11:31 DEBUG ] 20:11:31 DEBUG } 20:11:31 DEBUG ] 20:11:31 DEBUG } 20:11:31 DEBUG } 20:11:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713553891" 20:11:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553891" 20:11:31 ERROR [http_cmd]: ERROR 20:11:31 ERROR --- status code --- 20:11:31 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:11:31 ERROR --- http response --- 20:11:31 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:11:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553891" 20:11:31 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:11:31 INFO [http_cmd]: http status code OK 20:11:31 DEBUG --- status code --- 20:11:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:31 DEBUG --- http response --- 20:11:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:11:31 DEBUG ----- output ----- 20:11:31 DEBUG { 20:11:31 DEBUG "status": "success", 20:11:31 DEBUG "data": { 20:11:31 DEBUG "resultType": "vector", 20:11:31 DEBUG "result": [ 20:11:31 DEBUG { 20:11:31 DEBUG "metric": { 20:11:31 DEBUG "pod": "ds-idrepo-0" 20:11:31 DEBUG }, 20:11:31 DEBUG "value": [ 20:11:31 DEBUG 20:11:31 DEBUG 20:11:31 DEBUG 20:11:31 DEBUG [...] 20:11:31 DEBUG 20:11:31 DEBUG 20:11:31 DEBUG 20:11:31 DEBUG "pod": "ds-idrepo-2" 20:11:31 DEBUG }, 20:11:31 DEBUG "value": [ 20:11:31 DEBUG 1713553891, 20:11:31 DEBUG "0" 20:11:31 DEBUG ] 20:11:31 DEBUG } 20:11:31 DEBUG ] 20:11:31 DEBUG } 20:11:31 DEBUG } 20:11:31 INFO [http_cmd]: http status code OK 20:11:31 DEBUG --- status code --- 20:11:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:31 DEBUG --- http response --- 20:11:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:11:31 DEBUG ----- output ----- 20:11:31 DEBUG { 20:11:31 DEBUG "status": "success", 20:11:31 DEBUG "data": { 20:11:31 DEBUG "resultType": "vector", 20:11:31 DEBUG "result": [ 20:11:31 DEBUG { 20:11:31 DEBUG "metric": { 20:11:31 DEBUG "pod": "ds-idrepo-0" 20:11:31 DEBUG }, 20:11:31 DEBUG "value": [ 20:11:31 DEBUG 20:11:31 DEBUG 20:11:31 DEBUG 20:11:31 DEBUG [...] 20:11:31 DEBUG 20:11:31 DEBUG 20:11:31 DEBUG 20:11:31 DEBUG "pod": "ds-idrepo-2" 20:11:31 DEBUG }, 20:11:31 DEBUG "value": [ 20:11:31 DEBUG 1713553891, 20:11:31 DEBUG "0" 20:11:31 DEBUG ] 20:11:31 DEBUG } 20:11:31 DEBUG ] 20:11:31 DEBUG } 20:11:31 DEBUG } 20:11:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553891" 20:11:31 INFO [http_cmd]: http status code OK 20:11:31 DEBUG --- status code --- 20:11:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:31 DEBUG --- http response --- 20:11:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:11:31 DEBUG ----- output ----- 20:11:31 DEBUG { 20:11:31 DEBUG "status": "success", 20:11:31 DEBUG "data": { 20:11:31 DEBUG "resultType": "vector", 20:11:31 DEBUG "result": [ 20:11:31 DEBUG { 20:11:31 DEBUG "metric": { 20:11:31 DEBUG "instance": "10.142.0.107:9100" 20:11:31 DEBUG }, 20:11:31 DEBUG "value": [ 20:11:31 DEBUG 20:11:31 DEBUG 20:11:31 DEBUG 20:11:31 DEBUG [...] 20:11:31 DEBUG 20:11:31 DEBUG 20:11:31 DEBUG 20:11:31 DEBUG "instance": "10.142.0.43:9100" 20:11:31 DEBUG }, 20:11:31 DEBUG "value": [ 20:11:31 DEBUG 1713553891, 20:11:31 DEBUG "0.17659999999999532" 20:11:31 DEBUG ] 20:11:31 DEBUG } 20:11:31 DEBUG ] 20:11:31 DEBUG } 20:11:31 DEBUG } 20:11:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713553891" 20:11:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553891" 20:11:32 INFO [http_cmd]: http status code OK 20:11:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553892" 20:11:32 INFO [http_cmd]: http status code OK 20:11:32 DEBUG --- status code --- 20:11:32 DEBUG --- status code --- 20:11:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:32 DEBUG --- http response --- 20:11:32 DEBUG --- http response --- 20:11:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:11:32 DEBUG ----- output ----- 20:11:32 DEBUG { 20:11:32 DEBUG "status": "success", 20:11:32 DEBUG "data": { 20:11:32 DEBUG "resultType": "vector", 20:11:32 DEBUG "result": [ 20:11:32 DEBUG { 20:11:32 DEBUG "metric": { 20:11:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:11:32 DEBUG }, 20:11:32 DEBUG "value": [ 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG [...] 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:11:32 DEBUG }, 20:11:32 DEBUG "value": [ 20:11:32 DEBUG 1713553891, 20:11:32 DEBUG "0.015" 20:11:32 DEBUG ] 20:11:32 DEBUG } 20:11:32 DEBUG ] 20:11:32 DEBUG } 20:11:32 DEBUG } 20:11:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:11:32 DEBUG ----- output ----- 20:11:32 DEBUG { 20:11:32 DEBUG "status": "success", 20:11:32 DEBUG "data": { 20:11:32 DEBUG "resultType": "vector", 20:11:32 DEBUG "result": [ 20:11:32 DEBUG { 20:11:32 DEBUG "metric": { 20:11:32 DEBUG "pod": "ds-cts-0" 20:11:32 DEBUG }, 20:11:32 DEBUG "value": [ 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG [...] 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG "pod": "ds-cts-2" 20:11:32 DEBUG }, 20:11:32 DEBUG "value": [ 20:11:32 DEBUG 1713553891, 20:11:32 DEBUG "0.0005155399999999999" 20:11:32 DEBUG ] 20:11:32 DEBUG } 20:11:32 DEBUG ] 20:11:32 DEBUG } 20:11:32 DEBUG } 20:11:32 INFO [http_cmd]: http status code OK 20:11:32 DEBUG --- status code --- 20:11:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:32 DEBUG --- http response --- 20:11:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:11:32 DEBUG ----- output ----- 20:11:32 DEBUG { 20:11:32 DEBUG "status": "success", 20:11:32 DEBUG "data": { 20:11:32 DEBUG "resultType": "vector", 20:11:32 DEBUG "result": [ 20:11:32 DEBUG { 20:11:32 DEBUG "metric": { 20:11:32 DEBUG "instance": "10.142.0.107:9100" 20:11:32 DEBUG }, 20:11:32 DEBUG "value": [ 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG [...] 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG "instance": "10.142.0.43:9100" 20:11:32 DEBUG }, 20:11:32 DEBUG "value": [ 20:11:32 DEBUG 1713553892, 20:11:32 DEBUG "0.0853333333333334" 20:11:32 DEBUG ] 20:11:32 DEBUG } 20:11:32 DEBUG ] 20:11:32 DEBUG } 20:11:32 DEBUG } 20:11:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713553892" 20:11:32 INFO [http_cmd]: http status code OK 20:11:32 DEBUG --- status code --- 20:11:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:32 DEBUG --- http response --- 20:11:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:11:32 DEBUG ----- output ----- 20:11:32 DEBUG { 20:11:32 DEBUG "status": "success", 20:11:32 DEBUG "data": { 20:11:32 DEBUG "resultType": "vector", 20:11:32 DEBUG "result": [ 20:11:32 DEBUG { 20:11:32 DEBUG "metric": { 20:11:32 DEBUG "pod": "ds-idrepo-0" 20:11:32 DEBUG }, 20:11:32 DEBUG "value": [ 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG [...] 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG "pod": "ds-cts-2" 20:11:32 DEBUG }, 20:11:32 DEBUG "value": [ 20:11:32 DEBUG 1713553892, 20:11:32 DEBUG "0" 20:11:32 DEBUG ] 20:11:32 DEBUG } 20:11:32 DEBUG ] 20:11:32 DEBUG } 20:11:32 DEBUG } 20:11:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553892" 20:11:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553892" 20:11:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553892" 20:11:32 INFO [http_cmd]: http status code OK 20:11:32 DEBUG --- status code --- 20:11:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:32 DEBUG --- http response --- 20:11:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:11:32 DEBUG ----- output ----- 20:11:32 DEBUG { 20:11:32 DEBUG "status": "success", 20:11:32 DEBUG "data": { 20:11:32 DEBUG "resultType": "vector", 20:11:32 DEBUG "result": [ 20:11:32 DEBUG { 20:11:32 DEBUG "metric": { 20:11:32 DEBUG "pod": "ds-idrepo-1" 20:11:32 DEBUG }, 20:11:32 DEBUG "value": [ 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG [...] 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:11:32 DEBUG }, 20:11:32 DEBUG "value": [ 20:11:32 DEBUG 1713553892, 20:11:32 DEBUG "0" 20:11:32 DEBUG ] 20:11:32 DEBUG } 20:11:32 DEBUG ] 20:11:32 DEBUG } 20:11:32 DEBUG } 20:11:32 INFO [http_cmd]: http status code OK 20:11:32 INFO [http_cmd]: http status code OK 20:11:32 DEBUG --- status code --- 20:11:32 DEBUG --- status code --- 20:11:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:32 DEBUG --- http response --- 20:11:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:32 DEBUG --- http response --- 20:11:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:11:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:11:32 DEBUG ----- output ----- 20:11:32 DEBUG ----- output ----- 20:11:32 DEBUG { 20:11:32 DEBUG { 20:11:32 DEBUG "status": "success", 20:11:32 DEBUG "status": "success", 20:11:32 DEBUG "data": { 20:11:32 DEBUG "data": { 20:11:32 DEBUG "resultType": "vector", 20:11:32 DEBUG "resultType": "vector", 20:11:32 DEBUG "result": [ 20:11:32 DEBUG "result": [ 20:11:32 DEBUG { 20:11:32 DEBUG { 20:11:32 DEBUG "metric": { 20:11:32 DEBUG "metric": { 20:11:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:11:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:11:32 DEBUG }, 20:11:32 DEBUG }, 20:11:32 DEBUG "value": [ 20:11:32 DEBUG "value": [ 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG [...] 20:11:32 DEBUG [...] 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:11:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:11:32 DEBUG }, 20:11:32 DEBUG }, 20:11:32 DEBUG "value": [ 20:11:32 DEBUG "value": [ 20:11:32 DEBUG 1713553892, 20:11:32 DEBUG 1713553892, 20:11:32 DEBUG "0" 20:11:32 DEBUG "2507.5333333333333" 20:11:32 DEBUG ] 20:11:32 DEBUG ] 20:11:32 DEBUG } 20:11:32 DEBUG } 20:11:32 DEBUG ] 20:11:32 DEBUG ] 20:11:32 DEBUG } 20:11:32 DEBUG } 20:11:32 DEBUG } 20:11:32 DEBUG } 20:11:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553892" 20:11:32 INFO [http_cmd]: http status code OK 20:11:32 DEBUG --- status code --- 20:11:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:32 DEBUG --- http response --- 20:11:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:11:32 DEBUG ----- output ----- 20:11:32 DEBUG { 20:11:32 DEBUG "status": "success", 20:11:32 DEBUG "data": { 20:11:32 DEBUG "resultType": "vector", 20:11:32 DEBUG "result": [ 20:11:32 DEBUG { 20:11:32 DEBUG "metric": { 20:11:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:11:32 DEBUG }, 20:11:32 DEBUG "value": [ 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG [...] 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:11:32 DEBUG }, 20:11:32 DEBUG "value": [ 20:11:32 DEBUG 1713553892, 20:11:32 DEBUG "2498.9666666666667" 20:11:32 DEBUG ] 20:11:32 DEBUG } 20:11:32 DEBUG ] 20:11:32 DEBUG } 20:11:32 DEBUG } 20:11:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553892" 20:11:32 INFO [http_cmd]: http status code OK 20:11:32 DEBUG --- status code --- 20:11:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:32 DEBUG --- http response --- 20:11:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:11:32 DEBUG ----- output ----- 20:11:32 DEBUG { 20:11:32 DEBUG "status": "success", 20:11:32 DEBUG "data": { 20:11:32 DEBUG "resultType": "vector", 20:11:32 DEBUG "result": [ 20:11:32 DEBUG { 20:11:32 DEBUG "metric": { 20:11:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:11:32 DEBUG }, 20:11:32 DEBUG "value": [ 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG [...] 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:11:32 DEBUG }, 20:11:32 DEBUG "value": [ 20:11:32 DEBUG 1713553892, 20:11:32 DEBUG "2506.3" 20:11:32 DEBUG ] 20:11:32 DEBUG } 20:11:32 DEBUG ] 20:11:32 DEBUG } 20:11:32 DEBUG } 20:11:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553892" 20:11:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553892" 20:11:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553892" 20:11:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553892" 20:11:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553892" 20:11:32 INFO [http_cmd]: http status code OK 20:11:32 INFO [http_cmd]: http status code OK 20:11:32 DEBUG --- status code --- 20:11:32 DEBUG --- status code --- 20:11:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:32 DEBUG --- http response --- 20:11:32 DEBUG --- http response --- 20:11:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 20:11:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:11:32 DEBUG ----- output ----- 20:11:32 DEBUG ----- output ----- 20:11:32 DEBUG { 20:11:32 DEBUG { 20:11:32 DEBUG "status": "success", 20:11:32 DEBUG "status": "success", 20:11:32 DEBUG "data": { 20:11:32 DEBUG "data": { 20:11:32 DEBUG "resultType": "vector", 20:11:32 DEBUG "resultType": "vector", 20:11:32 DEBUG "result": [ 20:11:32 DEBUG "result": [ 20:11:32 DEBUG { 20:11:32 DEBUG { 20:11:32 DEBUG "metric": { 20:11:32 DEBUG "metric": { 20:11:32 DEBUG "pod": "ds-idrepo-1" 20:11:32 DEBUG "instance": "10.142.0.107:9100" 20:11:32 DEBUG }, 20:11:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553892" 20:11:32 DEBUG }, 20:11:32 DEBUG "value": [ 20:11:32 DEBUG "value": [ 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG [...] 20:11:32 DEBUG [...] 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG "pod": "idm-b85f9d978-wnhxb" 20:11:32 DEBUG "instance": "10.142.0.43:9100" 20:11:32 DEBUG }, 20:11:32 DEBUG }, 20:11:32 INFO [http_cmd]: http status code OK 20:11:32 INFO [http_cmd]: http status code OK 20:11:32 DEBUG "value": [ 20:11:32 INFO [http_cmd]: http status code OK 20:11:32 DEBUG --- status code --- 20:11:32 DEBUG --- status code --- 20:11:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713553892" 20:11:32 DEBUG --- status code --- 20:11:32 DEBUG 1713553892, 20:11:32 DEBUG "value": [ 20:11:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553892" 20:11:32 DEBUG "642.724839772387" 20:11:32 DEBUG 1713553892, 20:11:32 DEBUG --- http response --- 20:11:32 DEBUG --- http response --- 20:11:32 DEBUG --- http response --- 20:11:32 DEBUG ] 20:11:32 DEBUG "75551484.46666667" 20:11:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:11:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:11:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:11:32 DEBUG } 20:11:32 DEBUG ] 20:11:32 DEBUG ----- output ----- 20:11:32 DEBUG ----- output ----- 20:11:32 DEBUG ----- output ----- 20:11:32 DEBUG ] 20:11:32 DEBUG } 20:11:32 DEBUG { 20:11:32 DEBUG { 20:11:32 DEBUG { 20:11:32 DEBUG } 20:11:32 DEBUG ] 20:11:32 DEBUG "status": "success", 20:11:32 DEBUG "status": "success", 20:11:32 DEBUG "status": "success", 20:11:32 DEBUG } 20:11:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553892" 20:11:32 DEBUG } 20:11:32 DEBUG "data": { 20:11:32 DEBUG "data": { 20:11:32 DEBUG "data": { 20:11:32 DEBUG } 20:11:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553892" 20:11:32 DEBUG "resultType": "vector", 20:11:32 INFO [http_cmd]: http status code OK 20:11:32 DEBUG "resultType": "vector", 20:11:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713553892" 20:11:32 DEBUG "resultType": "vector", 20:11:32 DEBUG "result": [ 20:11:32 INFO [http_cmd]: http status code OK 20:11:32 DEBUG --- status code --- 20:11:32 DEBUG "result": [ 20:11:32 DEBUG "result": [ 20:11:32 DEBUG { 20:11:32 INFO [http_cmd]: http status code OK 20:11:32 DEBUG --- status code --- 20:11:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:32 DEBUG { 20:11:32 DEBUG { 20:11:32 DEBUG "metric": { 20:11:32 DEBUG --- status code --- 20:11:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:32 DEBUG --- http response --- 20:11:32 DEBUG "metric": { 20:11:32 DEBUG "metric": { 20:11:32 DEBUG "pod": "ds-cts-0" 20:11:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:32 DEBUG --- http response --- 20:11:32 DEBUG "pod": "ds-cts-0" 20:11:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:11:32 DEBUG "pod": "ds-idrepo-0" 20:11:32 DEBUG }, 20:11:32 DEBUG --- http response --- 20:11:32 DEBUG }, 20:11:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:11:32 DEBUG ----- output ----- 20:11:32 DEBUG }, 20:11:32 DEBUG "value": [ 20:11:32 DEBUG "value": [ 20:11:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:11:32 DEBUG ----- output ----- 20:11:32 DEBUG { 20:11:32 DEBUG "value": [ 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG ----- output ----- 20:11:32 DEBUG { 20:11:32 DEBUG "status": "success", 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 INFO [http_cmd]: http status code OK 20:11:32 DEBUG { 20:11:32 DEBUG "status": "success", 20:11:32 DEBUG "data": { 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG --- status code --- 20:11:32 DEBUG "status": "success", 20:11:32 DEBUG "data": { 20:11:32 DEBUG "resultType": "vector", 20:11:32 DEBUG "result": [ 20:11:32 DEBUG [...] 20:11:32 DEBUG [...] 20:11:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:32 DEBUG "data": { 20:11:32 DEBUG "resultType": "vector", 20:11:32 DEBUG 20:11:32 DEBUG { 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG --- http response --- 20:11:32 DEBUG "resultType": "vector", 20:11:32 DEBUG "result": [ 20:11:32 DEBUG [...] 20:11:32 DEBUG "metric": { 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 20:11:32 DEBUG "result": [ 20:11:32 INFO [http_cmd]: http status code OK 20:11:32 DEBUG { 20:11:32 DEBUG 20:11:32 DEBUG "instance": "10.142.0.107:9100" 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG ----- output ----- 20:11:32 DEBUG { 20:11:32 DEBUG --- status code --- 20:11:32 DEBUG "metric": { 20:11:32 DEBUG 20:11:32 DEBUG }, 20:11:32 DEBUG "pod": "ds-cts-2" 20:11:32 DEBUG "pod": "ds-cts-2" 20:11:32 DEBUG { 20:11:32 DEBUG "metric": { 20:11:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:32 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:11:32 DEBUG }, 20:11:32 DEBUG "value": [ 20:11:32 DEBUG }, 20:11:32 DEBUG }, 20:11:32 DEBUG "status": "success", 20:11:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:11:32 DEBUG --- http response --- 20:11:32 DEBUG 20:11:32 DEBUG "value": [ 20:11:32 DEBUG 20:11:32 DEBUG "value": [ 20:11:32 DEBUG "value": [ 20:11:32 DEBUG "data": { 20:11:32 DEBUG }, 20:11:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:11:32 DEBUG "pod": "ds-idrepo-2" 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG 1713553892, 20:11:32 DEBUG 1713553892, 20:11:32 DEBUG "resultType": "vector", 20:11:32 DEBUG "value": [ 20:11:32 DEBUG ----- output ----- 20:11:32 DEBUG }, 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG "0" 20:11:32 DEBUG "29.966666666666665" 20:11:32 DEBUG "result": [ 20:11:32 DEBUG 20:11:32 DEBUG { 20:11:32 DEBUG "value": [ 20:11:32 DEBUG 20:11:32 DEBUG [...] 20:11:32 DEBUG ] 20:11:32 DEBUG ] 20:11:32 DEBUG { 20:11:32 DEBUG 20:11:32 DEBUG "status": "success", 20:11:32 DEBUG 1713553892, 20:11:32 DEBUG [...] 20:11:32 DEBUG 20:11:32 DEBUG } 20:11:32 DEBUG } 20:11:32 DEBUG "metric": { 20:11:32 DEBUG 20:11:32 DEBUG "data": { 20:11:32 DEBUG "0" 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG ] 20:11:32 DEBUG } 20:11:32 DEBUG "pod": "ds-idrepo-1" 20:11:32 DEBUG [...] 20:11:32 DEBUG "resultType": "vector", 20:11:32 DEBUG ] 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 INFO [http_cmd]: http status code OK 20:11:32 DEBUG } 20:11:32 DEBUG ] 20:11:32 DEBUG }, 20:11:32 DEBUG 20:11:32 DEBUG "result": [ 20:11:32 DEBUG } 20:11:32 DEBUG 20:11:32 DEBUG "instance": "10.142.0.43:9100" 20:11:32 DEBUG --- status code --- 20:11:32 DEBUG } 20:11:32 DEBUG "value": [ 20:11:32 DEBUG 20:11:32 DEBUG { 20:11:32 DEBUG ] 20:11:32 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:11:32 DEBUG }, 20:11:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:32 DEBUG } 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG "metric": { 20:11:32 DEBUG } 20:11:32 DEBUG }, 20:11:32 DEBUG "value": [ 20:11:32 DEBUG --- http response --- 20:11:32 DEBUG 20:11:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:11:32 DEBUG "instance": "10.142.0.107:9100" 20:11:32 DEBUG } 20:11:32 DEBUG "value": [ 20:11:32 DEBUG 1713553892, 20:11:32 DEBUG 20:11:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:11:32 DEBUG }, 20:11:32 DEBUG }, 20:11:32 DEBUG 1713553892, 20:11:32 DEBUG "75548896.93333334" 20:11:32 DEBUG [...] 20:11:32 DEBUG ----- output ----- 20:11:32 DEBUG "value": [ 20:11:32 DEBUG "value": [ 20:11:32 DEBUG "8.039235900000035" 20:11:32 DEBUG ] 20:11:32 DEBUG 20:11:32 DEBUG { 20:11:32 DEBUG 1713553892, 20:11:32 DEBUG 20:11:32 DEBUG ] 20:11:32 DEBUG } 20:11:32 DEBUG 20:11:32 DEBUG "status": "success", 20:11:32 DEBUG "2508.0666666666666" 20:11:32 DEBUG 20:11:32 DEBUG } 20:11:32 DEBUG ] 20:11:32 DEBUG 20:11:32 DEBUG "data": { 20:11:32 DEBUG ] 20:11:32 DEBUG 20:11:32 DEBUG ] 20:11:32 DEBUG } 20:11:32 DEBUG "pod": "idm-b85f9d978-wnhxb" 20:11:32 DEBUG "resultType": "vector", 20:11:32 DEBUG } 20:11:32 DEBUG [...] 20:11:32 DEBUG } 20:11:32 DEBUG } 20:11:32 DEBUG }, 20:11:32 DEBUG "result": [ 20:11:32 DEBUG ] 20:11:32 DEBUG 20:11:32 DEBUG } 20:11:32 DEBUG "value": [ 20:11:32 DEBUG { 20:11:32 DEBUG } 20:11:32 DEBUG 20:11:32 DEBUG 1713553892, 20:11:32 DEBUG "metric": { 20:11:32 DEBUG } 20:11:32 DEBUG 20:11:32 DEBUG "0" 20:11:32 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:11:32 DEBUG "instance": "10.142.0.43:9100" 20:11:32 DEBUG ] 20:11:32 DEBUG }, 20:11:32 DEBUG }, 20:11:32 DEBUG } 20:11:32 DEBUG "value": [ 20:11:32 DEBUG "value": [ 20:11:32 DEBUG ] 20:11:32 DEBUG 20:11:32 DEBUG 1713553892, 20:11:32 DEBUG } 20:11:32 DEBUG 20:11:32 DEBUG "2040849738752" 20:11:32 DEBUG } 20:11:32 DEBUG 20:11:32 DEBUG ] 20:11:32 DEBUG [...] 20:11:32 DEBUG } 20:11:32 DEBUG 20:11:32 DEBUG ] 20:11:32 DEBUG 20:11:32 DEBUG } 20:11:32 DEBUG 20:11:32 DEBUG } 20:11:32 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:11:32 DEBUG }, 20:11:32 DEBUG "value": [ 20:11:32 DEBUG 1713553892, 20:11:32 DEBUG "3678117888" 20:11:32 DEBUG ] 20:11:32 DEBUG } 20:11:32 DEBUG ] 20:11:32 DEBUG } 20:11:32 DEBUG } 20:11:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553892" 20:11:32 INFO [http_cmd]: http status code OK 20:11:32 DEBUG --- status code --- 20:11:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:32 DEBUG --- http response --- 20:11:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:11:32 DEBUG ----- output ----- 20:11:32 DEBUG { 20:11:32 DEBUG "status": "success", 20:11:32 DEBUG "data": { 20:11:32 DEBUG "resultType": "vector", 20:11:32 DEBUG "result": [ 20:11:32 DEBUG { 20:11:32 DEBUG "metric": { 20:11:32 DEBUG "pod": "ds-idrepo-1" 20:11:32 DEBUG }, 20:11:32 DEBUG "value": [ 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG [...] 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG 20:11:32 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:11:32 DEBUG }, 20:11:32 DEBUG "value": [ 20:11:32 DEBUG 1713553892, 20:11:32 DEBUG "0" 20:11:32 DEBUG ] 20:11:32 DEBUG } 20:11:32 DEBUG ] 20:11:32 DEBUG } 20:11:32 DEBUG } 20:11:33 INFO 20:11:33 INFO [loop_until]: kubectl --namespace=xlou top pods 20:11:33 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:11:33 INFO [loop_until]: OK (rc = 0) 20:11:33 DEBUG --- stdout --- 20:11:33 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 16389m 9010Mi am-767f6b67b5-qcqz2 15954m 8812Mi am-767f6b67b5-vbltz 17572m 8996Mi ds-cts-0 17438m 10975Mi ds-cts-1 16255m 11215Mi ds-cts-2 17556m 10981Mi ds-idrepo-0 7224m 3691Mi ds-idrepo-1 7021m 3537Mi ds-idrepo-2 7220m 3492Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7992m 3416Mi 20:11:33 DEBUG --- stderr --- 20:11:33 DEBUG 20:11:35 INFO 20:11:35 INFO [loop_until]: kubectl --namespace=xlou top node 20:11:35 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:11:35 INFO [loop_until]: OK (rc = 0) 20:11:35 DEBUG --- stdout --- 20:11:35 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 70m 0% 6247Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7471m 25% 7283Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16090m 53% 11711Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 80m 0% 3347Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 17251m 57% 13961Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 163m 0% 5194Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 16497m 55% 11698Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15698m 52% 2996Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 17728m 59% 11217Mi 10% gke-xlou-cdm-ds-0f601751-j488 7707m 25% 6738Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 15993m 53% 15976Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 7504m 25% 7422Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 17900m 59% 13425Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 8195m 51% 7133Mi 12% 20:11:35 DEBUG --- stderr --- 20:11:35 DEBUG 20:12:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553951" 20:12:31 INFO [http_cmd]: http status code OK 20:12:31 DEBUG --- status code --- 20:12:31 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:31 DEBUG --- http response --- 20:12:31 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:12:31 DEBUG ----- output ----- 20:12:31 DEBUG { 20:12:31 DEBUG "status": "success", 20:12:31 DEBUG "data": { 20:12:31 DEBUG "resultType": "vector", 20:12:31 DEBUG "result": [ 20:12:31 DEBUG { 20:12:31 DEBUG "metric": { 20:12:31 DEBUG "pod": "am-767f6b67b5-9zw22" 20:12:31 DEBUG }, 20:12:31 DEBUG "value": [ 20:12:31 DEBUG 20:12:31 DEBUG 20:12:31 DEBUG 20:12:31 DEBUG [...] 20:12:31 DEBUG 20:12:31 DEBUG 20:12:31 DEBUG 20:12:31 DEBUG "pod": "am-767f6b67b5-vbltz" 20:12:31 DEBUG }, 20:12:31 DEBUG "value": [ 20:12:31 DEBUG 1713553951, 20:12:31 DEBUG "450642.5" 20:12:31 DEBUG ] 20:12:31 DEBUG } 20:12:31 DEBUG ] 20:12:31 DEBUG } 20:12:31 DEBUG } 20:12:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553951" 20:12:31 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713553951" 20:12:32 ERROR [http_cmd]: ERROR 20:12:32 INFO [http_cmd]: http status code OK 20:12:32 ERROR --- status code --- 20:12:32 DEBUG --- status code --- 20:12:32 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:12:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:32 ERROR --- http response --- 20:12:32 DEBUG --- http response --- 20:12:32 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:12:32 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:12:32 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:12:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553952" 20:12:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553952" 20:12:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553952" 20:12:32 INFO [http_cmd]: http status code OK 20:12:32 INFO [http_cmd]: http status code OK 20:12:32 DEBUG --- status code --- 20:12:32 DEBUG --- status code --- 20:12:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:32 DEBUG --- http response --- 20:12:32 DEBUG --- http response --- 20:12:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:12:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:12:32 DEBUG ----- output ----- 20:12:32 DEBUG ----- output ----- 20:12:32 DEBUG { 20:12:32 DEBUG { 20:12:32 DEBUG "status": "success", 20:12:32 DEBUG "status": "success", 20:12:32 DEBUG "data": { 20:12:32 DEBUG "data": { 20:12:32 DEBUG "resultType": "vector", 20:12:32 DEBUG "resultType": "vector", 20:12:32 DEBUG "result": [ 20:12:32 DEBUG "result": [ 20:12:32 DEBUG { 20:12:32 DEBUG { 20:12:32 DEBUG "metric": { 20:12:32 DEBUG "metric": { 20:12:32 DEBUG "pod": "ds-idrepo-0" 20:12:32 DEBUG "pod": "ds-idrepo-0" 20:12:32 DEBUG }, 20:12:32 DEBUG }, 20:12:32 DEBUG "value": [ 20:12:32 DEBUG "value": [ 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG [...] 20:12:32 DEBUG [...] 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG "pod": "ds-idrepo-2" 20:12:32 DEBUG "pod": "ds-idrepo-2" 20:12:32 DEBUG }, 20:12:32 DEBUG }, 20:12:32 DEBUG "value": [ 20:12:32 DEBUG "value": [ 20:12:32 DEBUG 1713553952, 20:12:32 DEBUG 1713553952, 20:12:32 DEBUG "0" 20:12:32 DEBUG "0" 20:12:32 DEBUG ] 20:12:32 DEBUG ] 20:12:32 DEBUG } 20:12:32 DEBUG } 20:12:32 DEBUG ] 20:12:32 DEBUG ] 20:12:32 DEBUG } 20:12:32 DEBUG } 20:12:32 DEBUG } 20:12:32 DEBUG } 20:12:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553952" 20:12:32 INFO [http_cmd]: http status code OK 20:12:32 DEBUG --- status code --- 20:12:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:32 DEBUG --- http response --- 20:12:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:12:32 DEBUG ----- output ----- 20:12:32 DEBUG { 20:12:32 DEBUG "status": "success", 20:12:32 DEBUG "data": { 20:12:32 DEBUG "resultType": "vector", 20:12:32 DEBUG "result": [ 20:12:32 DEBUG { 20:12:32 DEBUG "metric": { 20:12:32 DEBUG "instance": "10.142.0.107:9100" 20:12:32 DEBUG }, 20:12:32 DEBUG "value": [ 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG [...] 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG "instance": "10.142.0.43:9100" 20:12:32 DEBUG }, 20:12:32 DEBUG "value": [ 20:12:32 DEBUG 1713553952, 20:12:32 DEBUG "0.09366666666667052" 20:12:32 DEBUG ] 20:12:32 DEBUG } 20:12:32 DEBUG ] 20:12:32 DEBUG } 20:12:32 DEBUG } 20:12:32 INFO [http_cmd]: http status code OK 20:12:32 DEBUG --- status code --- 20:12:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:32 DEBUG --- http response --- 20:12:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:12:32 DEBUG ----- output ----- 20:12:32 DEBUG { 20:12:32 DEBUG "status": "success", 20:12:32 DEBUG "data": { 20:12:32 DEBUG "resultType": "vector", 20:12:32 DEBUG "result": [ 20:12:32 DEBUG { 20:12:32 DEBUG "metric": { 20:12:32 DEBUG "instance": "10.142.0.107:9100" 20:12:32 DEBUG }, 20:12:32 DEBUG "value": [ 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG [...] 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG "instance": "10.142.0.43:9100" 20:12:32 DEBUG }, 20:12:32 DEBUG "value": [ 20:12:32 DEBUG 1713553952, 20:12:32 DEBUG "0.2093666666666678" 20:12:32 DEBUG ] 20:12:32 DEBUG } 20:12:32 DEBUG ] 20:12:32 DEBUG } 20:12:32 DEBUG } 20:12:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713553952" 20:12:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553952" 20:12:32 INFO [http_cmd]: http status code OK 20:12:32 INFO [http_cmd]: http status code OK 20:12:32 DEBUG --- status code --- 20:12:32 DEBUG --- status code --- 20:12:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:32 DEBUG --- http response --- 20:12:32 DEBUG --- http response --- 20:12:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:12:32 DEBUG ----- output ----- 20:12:32 DEBUG { 20:12:32 DEBUG "status": "success", 20:12:32 DEBUG "data": { 20:12:32 DEBUG "resultType": "vector", 20:12:32 DEBUG "result": [ 20:12:32 DEBUG { 20:12:32 DEBUG "metric": { 20:12:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:12:32 DEBUG }, 20:12:32 DEBUG "value": [ 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG [...] 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:12:32 DEBUG }, 20:12:32 DEBUG "value": [ 20:12:32 DEBUG 1713553952, 20:12:32 DEBUG "0.015" 20:12:32 DEBUG ] 20:12:32 DEBUG } 20:12:32 DEBUG ] 20:12:32 DEBUG } 20:12:32 DEBUG } 20:12:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:12:32 DEBUG ----- output ----- 20:12:32 DEBUG { 20:12:32 DEBUG "status": "success", 20:12:32 DEBUG "data": { 20:12:32 DEBUG "resultType": "vector", 20:12:32 DEBUG "result": [ 20:12:32 DEBUG { 20:12:32 DEBUG "metric": { 20:12:32 DEBUG "pod": "ds-cts-0" 20:12:32 DEBUG }, 20:12:32 DEBUG "value": [ 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG [...] 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG "pod": "ds-cts-2" 20:12:32 DEBUG }, 20:12:32 DEBUG "value": [ 20:12:32 DEBUG 1713553952, 20:12:32 DEBUG "0.0010333333333333334" 20:12:32 DEBUG ] 20:12:32 DEBUG } 20:12:32 DEBUG ] 20:12:32 DEBUG } 20:12:32 DEBUG } 20:12:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713553952" 20:12:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553952" 20:12:32 INFO [http_cmd]: http status code OK 20:12:32 DEBUG --- status code --- 20:12:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:32 DEBUG --- http response --- 20:12:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:12:32 DEBUG ----- output ----- 20:12:32 DEBUG { 20:12:32 DEBUG "status": "success", 20:12:32 DEBUG "data": { 20:12:32 DEBUG "resultType": "vector", 20:12:32 DEBUG "result": [ 20:12:32 DEBUG { 20:12:32 DEBUG "metric": { 20:12:32 DEBUG "pod": "ds-idrepo-0" 20:12:32 DEBUG }, 20:12:32 DEBUG "value": [ 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG [...] 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG "pod": "ds-cts-2" 20:12:32 DEBUG }, 20:12:32 DEBUG "value": [ 20:12:32 DEBUG 1713553952, 20:12:32 DEBUG "0" 20:12:32 DEBUG ] 20:12:32 DEBUG } 20:12:32 DEBUG ] 20:12:32 DEBUG } 20:12:32 DEBUG } 20:12:32 INFO [http_cmd]: http status code OK 20:12:32 DEBUG --- status code --- 20:12:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:32 DEBUG --- http response --- 20:12:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:12:32 DEBUG ----- output ----- 20:12:32 DEBUG { 20:12:32 DEBUG "status": "success", 20:12:32 DEBUG "data": { 20:12:32 DEBUG "resultType": "vector", 20:12:32 DEBUG "result": [ 20:12:32 DEBUG { 20:12:32 DEBUG "metric": { 20:12:32 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:12:32 DEBUG }, 20:12:32 DEBUG "value": [ 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG [...] 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:12:32 DEBUG }, 20:12:32 DEBUG "value": [ 20:12:32 DEBUG 1713553952, 20:12:32 DEBUG "0" 20:12:32 DEBUG ] 20:12:32 DEBUG } 20:12:32 DEBUG ] 20:12:32 DEBUG } 20:12:32 DEBUG } 20:12:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553952" 20:12:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553952" 20:12:32 INFO [http_cmd]: http status code OK 20:12:32 DEBUG --- status code --- 20:12:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:32 DEBUG --- http response --- 20:12:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:12:32 DEBUG ----- output ----- 20:12:32 DEBUG { 20:12:32 DEBUG "status": "success", 20:12:32 DEBUG "data": { 20:12:32 DEBUG "resultType": "vector", 20:12:32 DEBUG "result": [ 20:12:32 DEBUG { 20:12:32 DEBUG "metric": { 20:12:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:12:32 DEBUG }, 20:12:32 DEBUG "value": [ 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 INFO [http_cmd]: http status code OK 20:12:32 DEBUG 20:12:32 DEBUG --- status code --- 20:12:32 DEBUG [...] 20:12:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:32 DEBUG 20:12:32 DEBUG --- http response --- 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:12:32 DEBUG }, 20:12:32 DEBUG "value": [ 20:12:32 DEBUG 1713553952, 20:12:32 DEBUG "2509.8326778214787" 20:12:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:12:32 DEBUG ] 20:12:32 DEBUG ----- output ----- 20:12:32 DEBUG } 20:12:32 DEBUG { 20:12:32 DEBUG ] 20:12:32 DEBUG "status": "success", 20:12:32 DEBUG } 20:12:32 DEBUG "data": { 20:12:32 DEBUG } 20:12:32 DEBUG "resultType": "vector", 20:12:32 DEBUG "result": [ 20:12:32 DEBUG { 20:12:32 DEBUG "metric": { 20:12:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:12:32 DEBUG }, 20:12:32 DEBUG "value": [ 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG [...] 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:12:32 DEBUG }, 20:12:32 DEBUG "value": [ 20:12:32 DEBUG 1713553952, 20:12:32 DEBUG "0" 20:12:32 DEBUG ] 20:12:32 DEBUG } 20:12:32 DEBUG ] 20:12:32 DEBUG } 20:12:32 DEBUG } 20:12:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553952" 20:12:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553952" 20:12:32 INFO [http_cmd]: http status code OK 20:12:32 INFO [http_cmd]: http status code OK 20:12:32 DEBUG --- status code --- 20:12:32 DEBUG --- status code --- 20:12:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:32 DEBUG --- http response --- 20:12:32 DEBUG --- http response --- 20:12:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:12:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:12:32 DEBUG ----- output ----- 20:12:32 DEBUG ----- output ----- 20:12:32 DEBUG { 20:12:32 DEBUG { 20:12:32 DEBUG "status": "success", 20:12:32 DEBUG "status": "success", 20:12:32 DEBUG "data": { 20:12:32 DEBUG "data": { 20:12:32 DEBUG "resultType": "vector", 20:12:32 DEBUG "resultType": "vector", 20:12:32 DEBUG "result": [ 20:12:32 DEBUG "result": [ 20:12:32 DEBUG { 20:12:32 DEBUG { 20:12:32 DEBUG "metric": { 20:12:32 DEBUG "metric": { 20:12:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:12:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:12:32 DEBUG }, 20:12:32 DEBUG }, 20:12:32 DEBUG "value": [ 20:12:32 DEBUG "value": [ 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG [...] 20:12:32 DEBUG [...] 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:12:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:12:32 DEBUG }, 20:12:32 DEBUG }, 20:12:32 DEBUG "value": [ 20:12:32 DEBUG "value": [ 20:12:32 DEBUG 1713553952, 20:12:32 DEBUG 1713553952, 20:12:32 DEBUG "2503.766415572295" 20:12:32 DEBUG "2510.0326644890342" 20:12:32 DEBUG ] 20:12:32 DEBUG ] 20:12:32 DEBUG } 20:12:32 DEBUG } 20:12:32 DEBUG ] 20:12:32 DEBUG ] 20:12:32 DEBUG } 20:12:32 DEBUG } 20:12:32 DEBUG } 20:12:32 DEBUG } 20:12:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553952" 20:12:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713553952" 20:12:32 INFO [http_cmd]: http status code OK 20:12:32 DEBUG --- status code --- 20:12:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:32 DEBUG --- http response --- 20:12:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:12:32 DEBUG ----- output ----- 20:12:32 DEBUG { 20:12:32 DEBUG "status": "success", 20:12:32 DEBUG "data": { 20:12:32 DEBUG "resultType": "vector", 20:12:32 DEBUG "result": [ 20:12:32 DEBUG { 20:12:32 DEBUG "metric": { 20:12:32 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:12:32 DEBUG }, 20:12:32 DEBUG "value": [ 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG [...] 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 20:12:32 DEBUG }, 20:12:32 DEBUG "value": [ 20:12:32 DEBUG 1713553952, 20:12:32 DEBUG "0" 20:12:32 DEBUG ] 20:12:32 DEBUG } 20:12:32 DEBUG ] 20:12:32 DEBUG } 20:12:32 DEBUG } 20:12:32 INFO [http_cmd]: http status code OK 20:12:32 DEBUG --- status code --- 20:12:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:32 DEBUG --- http response --- 20:12:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:12:32 DEBUG ----- output ----- 20:12:32 DEBUG { 20:12:32 DEBUG "status": "success", 20:12:32 DEBUG "data": { 20:12:32 DEBUG "resultType": "vector", 20:12:32 DEBUG "result": [ 20:12:32 DEBUG { 20:12:32 DEBUG "metric": { 20:12:32 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:12:32 DEBUG }, 20:12:32 DEBUG "value": [ 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG [...] 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:12:32 DEBUG }, 20:12:32 DEBUG "value": [ 20:12:32 DEBUG 1713553952, 20:12:32 DEBUG "8.039858794255563" 20:12:32 DEBUG ] 20:12:32 DEBUG } 20:12:32 DEBUG ] 20:12:32 DEBUG } 20:12:32 DEBUG } 20:12:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553952" 20:12:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553952" 20:12:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553952" 20:12:32 INFO [http_cmd]: http status code OK 20:12:32 INFO [http_cmd]: http status code OK 20:12:32 DEBUG --- status code --- 20:12:32 DEBUG --- status code --- 20:12:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:32 DEBUG --- http response --- 20:12:32 DEBUG --- http response --- 20:12:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:12:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:12:32 DEBUG ----- output ----- 20:12:32 DEBUG ----- output ----- 20:12:32 DEBUG { 20:12:32 DEBUG { 20:12:32 DEBUG "status": "success", 20:12:32 DEBUG "status": "success", 20:12:32 DEBUG "data": { 20:12:32 DEBUG "data": { 20:12:32 DEBUG "resultType": "vector", 20:12:32 DEBUG "resultType": "vector", 20:12:32 DEBUG "result": [ 20:12:32 DEBUG "result": [ 20:12:32 DEBUG { 20:12:32 DEBUG { 20:12:32 DEBUG "metric": { 20:12:32 DEBUG "metric": { 20:12:32 DEBUG "pod": "ds-cts-0" 20:12:32 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:12:32 DEBUG }, 20:12:32 DEBUG }, 20:12:32 DEBUG "value": [ 20:12:32 DEBUG "value": [ 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG [...] 20:12:32 DEBUG [...] 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG "pod": "ds-cts-2" 20:12:32 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 20:12:32 DEBUG }, 20:12:32 DEBUG }, 20:12:32 DEBUG "value": [ 20:12:32 DEBUG "value": [ 20:12:32 DEBUG 1713553952, 20:12:32 DEBUG 1713553952, 20:12:32 DEBUG "0" 20:12:32 DEBUG "0" 20:12:32 DEBUG ] 20:12:32 DEBUG ] 20:12:32 DEBUG } 20:12:32 DEBUG } 20:12:32 DEBUG ] 20:12:32 DEBUG ] 20:12:32 DEBUG } 20:12:32 DEBUG } 20:12:32 DEBUG } 20:12:32 DEBUG } 20:12:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553952" 20:12:32 INFO [http_cmd]: http status code OK 20:12:32 DEBUG --- status code --- 20:12:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:32 DEBUG --- http response --- 20:12:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:12:32 DEBUG ----- output ----- 20:12:32 DEBUG { 20:12:32 DEBUG "status": "success", 20:12:32 DEBUG "data": { 20:12:32 DEBUG "resultType": "vector", 20:12:32 DEBUG "result": [ 20:12:32 DEBUG { 20:12:32 DEBUG "metric": { 20:12:32 DEBUG "pod": "ds-idrepo-0" 20:12:32 DEBUG }, 20:12:32 DEBUG "value": [ 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG [...] 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG "pod": "ds-idrepo-2" 20:12:32 DEBUG }, 20:12:32 DEBUG "value": [ 20:12:32 DEBUG 1713553952, 20:12:32 DEBUG "0" 20:12:32 DEBUG ] 20:12:32 DEBUG } 20:12:32 DEBUG ] 20:12:32 DEBUG } 20:12:32 DEBUG } 20:12:32 INFO [http_cmd]: http status code OK 20:12:32 DEBUG --- status code --- 20:12:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:32 DEBUG --- http response --- 20:12:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:12:32 DEBUG ----- output ----- 20:12:32 DEBUG { 20:12:32 DEBUG "status": "success", 20:12:32 DEBUG "data": { 20:12:32 DEBUG "resultType": "vector", 20:12:32 DEBUG "result": [ 20:12:32 DEBUG { 20:12:32 DEBUG "metric": { 20:12:32 DEBUG "instance": "10.142.0.107:9100" 20:12:32 DEBUG }, 20:12:32 DEBUG "value": [ 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG [...] 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG "instance": "10.142.0.43:9100" 20:12:32 DEBUG }, 20:12:32 DEBUG "value": [ 20:12:32 DEBUG 1713553952, 20:12:32 DEBUG "75628990.6" 20:12:32 DEBUG ] 20:12:32 DEBUG } 20:12:32 DEBUG ] 20:12:32 DEBUG } 20:12:32 DEBUG } 20:12:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553952" 20:12:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713553952" 20:12:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553952" 20:12:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553952" 20:12:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713553952" 20:12:32 INFO [http_cmd]: http status code OK 20:12:32 DEBUG --- status code --- 20:12:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:32 DEBUG --- http response --- 20:12:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:12:32 DEBUG ----- output ----- 20:12:32 DEBUG { 20:12:32 DEBUG "status": "success", 20:12:32 DEBUG "data": { 20:12:32 DEBUG "resultType": "vector", 20:12:32 DEBUG "result": [ 20:12:32 DEBUG { 20:12:32 DEBUG "metric": { 20:12:32 DEBUG "pod": "ds-cts-0" 20:12:32 DEBUG }, 20:12:32 DEBUG "value": [ 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG [...] 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG "pod": "ds-cts-2" 20:12:32 DEBUG }, 20:12:32 DEBUG "value": [ 20:12:32 DEBUG 1713553952, 20:12:32 DEBUG "29.966666666666665" 20:12:32 DEBUG ] 20:12:32 DEBUG } 20:12:32 DEBUG ] 20:12:32 DEBUG } 20:12:32 DEBUG } 20:12:32 INFO [http_cmd]: http status code OK 20:12:32 DEBUG --- status code --- 20:12:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:32 DEBUG --- http response --- 20:12:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:12:32 DEBUG ----- output ----- 20:12:32 DEBUG { 20:12:32 DEBUG "status": "success", 20:12:32 DEBUG "data": { 20:12:32 DEBUG "resultType": "vector", 20:12:32 DEBUG "result": [ 20:12:32 DEBUG { 20:12:32 DEBUG "metric": { 20:12:32 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:12:32 DEBUG }, 20:12:32 DEBUG "value": [ 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG [...] 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:12:32 DEBUG }, 20:12:32 DEBUG "value": [ 20:12:32 DEBUG 1713553952, 20:12:32 DEBUG "3680051200" 20:12:32 DEBUG ] 20:12:32 DEBUG } 20:12:32 DEBUG ] 20:12:32 DEBUG } 20:12:32 DEBUG } 20:12:32 INFO [http_cmd]: http status code OK 20:12:32 DEBUG --- status code --- 20:12:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:32 DEBUG --- http response --- 20:12:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:12:32 DEBUG ----- output ----- 20:12:32 DEBUG { 20:12:32 DEBUG "status": "success", 20:12:32 DEBUG "data": { 20:12:32 DEBUG "resultType": "vector", 20:12:32 DEBUG "result": [ 20:12:32 DEBUG { 20:12:32 DEBUG "metric": { 20:12:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:12:32 DEBUG }, 20:12:32 DEBUG "value": [ 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG [...] 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:12:32 DEBUG }, 20:12:32 DEBUG "value": [ 20:12:32 DEBUG 1713553952, 20:12:32 DEBUG "2507.366175588294" 20:12:32 DEBUG ] 20:12:32 DEBUG } 20:12:32 DEBUG ] 20:12:32 DEBUG } 20:12:32 DEBUG } 20:12:32 INFO [http_cmd]: http status code OK 20:12:32 DEBUG --- status code --- 20:12:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:32 DEBUG --- http response --- 20:12:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:12:32 DEBUG ----- output ----- 20:12:32 DEBUG { 20:12:32 DEBUG "status": "success", 20:12:32 DEBUG "data": { 20:12:32 DEBUG "resultType": "vector", 20:12:32 DEBUG "result": [ 20:12:32 DEBUG { 20:12:32 DEBUG "metric": { 20:12:32 DEBUG "instance": "10.142.0.107:9100" 20:12:32 DEBUG }, 20:12:32 DEBUG "value": [ 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG [...] 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG "instance": "10.142.0.43:9100" 20:12:32 DEBUG }, 20:12:32 DEBUG "value": [ 20:12:32 DEBUG 1713553952, 20:12:32 DEBUG "75627576.93333334" 20:12:32 DEBUG ] 20:12:32 DEBUG } 20:12:32 DEBUG ] 20:12:32 DEBUG } 20:12:32 DEBUG } 20:12:32 INFO [http_cmd]: http status code OK 20:12:32 DEBUG --- status code --- 20:12:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:32 DEBUG --- http response --- 20:12:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:12:32 DEBUG ----- output ----- 20:12:32 DEBUG { 20:12:32 DEBUG "status": "success", 20:12:32 DEBUG "data": { 20:12:32 DEBUG "resultType": "vector", 20:12:32 DEBUG "result": [ 20:12:32 DEBUG { 20:12:32 DEBUG "metric": { 20:12:32 DEBUG "instance": "10.142.0.107:9100" 20:12:32 DEBUG }, 20:12:32 DEBUG "value": [ 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG [...] 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG "instance": "10.142.0.43:9100" 20:12:32 DEBUG }, 20:12:32 DEBUG "value": [ 20:12:32 DEBUG 1713553952, 20:12:32 DEBUG "2045081580544" 20:12:32 DEBUG ] 20:12:32 DEBUG } 20:12:32 DEBUG ] 20:12:32 DEBUG } 20:12:32 DEBUG } 20:12:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713553952" 20:12:32 INFO [http_cmd]: http status code OK 20:12:32 DEBUG --- status code --- 20:12:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:32 DEBUG --- http response --- 20:12:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:12:32 DEBUG ----- output ----- 20:12:32 DEBUG { 20:12:32 DEBUG "status": "success", 20:12:32 DEBUG "data": { 20:12:32 DEBUG "resultType": "vector", 20:12:32 DEBUG "result": [ 20:12:32 DEBUG { 20:12:32 DEBUG "metric": { 20:12:32 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:12:32 DEBUG }, 20:12:32 DEBUG "value": [ 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG [...] 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG 20:12:32 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:12:32 DEBUG }, 20:12:32 DEBUG "value": [ 20:12:32 DEBUG 1713553952, 20:12:32 DEBUG "0" 20:12:32 DEBUG ] 20:12:32 DEBUG } 20:12:32 DEBUG ] 20:12:32 DEBUG } 20:12:32 DEBUG } 20:12:34 INFO 20:12:34 INFO [loop_until]: kubectl --namespace=xlou top pods 20:12:34 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:12:34 INFO [loop_until]: OK (rc = 0) 20:12:34 DEBUG --- stdout --- 20:12:34 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 16363m 9009Mi am-767f6b67b5-qcqz2 16260m 8812Mi am-767f6b67b5-vbltz 17809m 8997Mi ds-cts-0 17123m 10976Mi ds-cts-1 16389m 11225Mi ds-cts-2 16970m 10985Mi ds-idrepo-0 7284m 3692Mi ds-idrepo-1 7219m 3537Mi ds-idrepo-2 7445m 3493Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 4m 1217Mi lodemon-f8459c597-6tj6f 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8101m 3418Mi 20:12:34 DEBUG --- stderr --- 20:12:34 DEBUG 20:12:35 INFO 20:12:35 INFO [loop_until]: kubectl --namespace=xlou top node 20:12:35 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:12:35 INFO [loop_until]: OK (rc = 0) 20:12:35 DEBUG --- stdout --- 20:12:35 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 68m 0% 6246Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7450m 24% 7298Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16222m 54% 11711Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 80m 0% 3335Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 17136m 57% 13957Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 161m 0% 5174Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 16541m 55% 11699Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15802m 52% 2986Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 18117m 60% 11213Mi 10% gke-xlou-cdm-ds-0f601751-j488 7715m 25% 6734Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 15984m 53% 15991Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 7507m 25% 7424Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 17486m 58% 13427Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 8198m 51% 7132Mi 12% 20:12:35 DEBUG --- stderr --- 20:12:35 DEBUG 20:13:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554012" 20:13:32 INFO [http_cmd]: http status code OK 20:13:32 DEBUG --- status code --- 20:13:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:32 DEBUG --- http response --- 20:13:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:13:32 DEBUG ----- output ----- 20:13:32 DEBUG { 20:13:32 DEBUG "status": "success", 20:13:32 DEBUG "data": { 20:13:32 DEBUG "resultType": "vector", 20:13:32 DEBUG "result": [ 20:13:32 DEBUG { 20:13:32 DEBUG "metric": { 20:13:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:13:32 DEBUG }, 20:13:32 DEBUG "value": [ 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG [...] 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:13:32 DEBUG }, 20:13:32 DEBUG "value": [ 20:13:32 DEBUG 1713554012, 20:13:32 DEBUG "450695" 20:13:32 DEBUG ] 20:13:32 DEBUG } 20:13:32 DEBUG ] 20:13:32 DEBUG } 20:13:32 DEBUG } 20:13:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713554012" 20:13:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554012" 20:13:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554012" 20:13:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554012" 20:13:32 INFO [http_cmd]: http status code OK 20:13:32 DEBUG --- status code --- 20:13:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:32 DEBUG --- http response --- 20:13:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:13:32 DEBUG ----- output ----- 20:13:32 DEBUG { 20:13:32 DEBUG "status": "success", 20:13:32 DEBUG "data": { 20:13:32 DEBUG "resultType": "vector", 20:13:32 DEBUG "result": [ 20:13:32 DEBUG { 20:13:32 DEBUG "metric": { 20:13:32 DEBUG "pod": "ds-idrepo-0" 20:13:32 DEBUG }, 20:13:32 DEBUG "value": [ 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG [...] 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG "pod": "ds-idrepo-2" 20:13:32 DEBUG }, 20:13:32 DEBUG "value": [ 20:13:32 DEBUG 1713554012, 20:13:32 DEBUG "0" 20:13:32 DEBUG ] 20:13:32 DEBUG } 20:13:32 DEBUG ] 20:13:32 DEBUG } 20:13:32 DEBUG } 20:13:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554012" 20:13:32 ERROR [http_cmd]: ERROR 20:13:32 INFO [http_cmd]: http status code OK 20:13:32 INFO [http_cmd]: http status code OK 20:13:32 ERROR --- status code --- 20:13:32 DEBUG --- status code --- 20:13:32 DEBUG --- status code --- 20:13:32 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:13:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:32 ERROR --- http response --- 20:13:32 DEBUG --- http response --- 20:13:32 DEBUG --- http response --- 20:13:32 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:13:32 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:13:32 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:13:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:13:32 DEBUG ----- output ----- 20:13:32 DEBUG { 20:13:32 DEBUG "status": "success", 20:13:32 DEBUG "data": { 20:13:32 DEBUG "resultType": "vector", 20:13:32 DEBUG "result": [ 20:13:32 DEBUG { 20:13:32 DEBUG "metric": { 20:13:32 DEBUG "instance": "10.142.0.107:9100" 20:13:32 DEBUG }, 20:13:32 DEBUG "value": [ 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG [...] 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG "instance": "10.142.0.43:9100" 20:13:32 DEBUG }, 20:13:32 DEBUG "value": [ 20:13:32 DEBUG 1713554012, 20:13:32 DEBUG "0.06666666666667424" 20:13:32 DEBUG ] 20:13:32 DEBUG } 20:13:32 DEBUG ] 20:13:32 DEBUG } 20:13:32 DEBUG } 20:13:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554012" 20:13:32 INFO [http_cmd]: http status code OK 20:13:32 DEBUG --- status code --- 20:13:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:32 DEBUG --- http response --- 20:13:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:13:32 DEBUG ----- output ----- 20:13:32 DEBUG { 20:13:32 DEBUG "status": "success", 20:13:32 DEBUG "data": { 20:13:32 DEBUG "resultType": "vector", 20:13:32 DEBUG "result": [ 20:13:32 DEBUG { 20:13:32 DEBUG "metric": { 20:13:32 DEBUG "instance": "10.142.0.107:9100" 20:13:32 DEBUG }, 20:13:32 DEBUG "value": [ 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG [...] 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG "instance": "10.142.0.43:9100" 20:13:32 DEBUG }, 20:13:32 DEBUG "value": [ 20:13:32 DEBUG 1713554012, 20:13:32 DEBUG "0.1289666666667671" 20:13:32 DEBUG ] 20:13:32 DEBUG } 20:13:32 DEBUG ] 20:13:32 DEBUG } 20:13:32 DEBUG } 20:13:32 INFO [http_cmd]: http status code OK 20:13:32 DEBUG --- status code --- 20:13:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:32 DEBUG --- http response --- 20:13:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:13:32 DEBUG ----- output ----- 20:13:32 DEBUG { 20:13:32 DEBUG "status": "success", 20:13:32 DEBUG "data": { 20:13:32 DEBUG "resultType": "vector", 20:13:32 DEBUG "result": [ 20:13:32 DEBUG { 20:13:32 DEBUG "metric": { 20:13:32 DEBUG "pod": "ds-idrepo-0" 20:13:32 DEBUG }, 20:13:32 DEBUG "value": [ 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG [...] 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG "pod": "ds-idrepo-2" 20:13:32 DEBUG }, 20:13:32 DEBUG "value": [ 20:13:32 DEBUG 1713554012, 20:13:32 DEBUG "0" 20:13:32 DEBUG ] 20:13:32 DEBUG } 20:13:32 DEBUG ] 20:13:32 DEBUG } 20:13:32 DEBUG } 20:13:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713554012" 20:13:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554012" 20:13:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713554012" 20:13:32 INFO [http_cmd]: http status code OK 20:13:32 DEBUG --- status code --- 20:13:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:32 DEBUG --- http response --- 20:13:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:13:32 DEBUG ----- output ----- 20:13:32 DEBUG { 20:13:32 DEBUG "status": "success", 20:13:32 DEBUG "data": { 20:13:32 DEBUG "resultType": "vector", 20:13:32 DEBUG "result": [ 20:13:32 DEBUG { 20:13:32 DEBUG "metric": { 20:13:32 DEBUG "pod": "ds-cts-0" 20:13:32 DEBUG }, 20:13:32 DEBUG "value": [ 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG [...] 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG "pod": "ds-cts-2" 20:13:32 DEBUG }, 20:13:32 DEBUG "value": [ 20:13:32 DEBUG 1713554012, 20:13:32 DEBUG "0.001574411111111111" 20:13:32 DEBUG ] 20:13:32 DEBUG } 20:13:32 DEBUG ] 20:13:32 DEBUG } 20:13:32 DEBUG } 20:13:32 INFO [http_cmd]: http status code OK 20:13:32 DEBUG --- status code --- 20:13:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:32 DEBUG --- http response --- 20:13:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:13:32 DEBUG ----- output ----- 20:13:32 DEBUG { 20:13:32 DEBUG "status": "success", 20:13:32 DEBUG "data": { 20:13:32 DEBUG "resultType": "vector", 20:13:32 DEBUG "result": [ 20:13:32 DEBUG { 20:13:32 DEBUG "metric": { 20:13:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:13:32 DEBUG }, 20:13:32 DEBUG "value": [ 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG [...] 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:13:32 DEBUG }, 20:13:32 DEBUG "value": [ 20:13:32 DEBUG 1713554012, 20:13:32 DEBUG "0.015" 20:13:32 DEBUG ] 20:13:32 DEBUG } 20:13:32 DEBUG ] 20:13:32 DEBUG } 20:13:32 DEBUG } 20:13:32 INFO [http_cmd]: http status code OK 20:13:32 DEBUG --- status code --- 20:13:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:32 DEBUG --- http response --- 20:13:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:13:32 DEBUG ----- output ----- 20:13:32 DEBUG { 20:13:32 DEBUG "status": "success", 20:13:32 DEBUG "data": { 20:13:32 DEBUG "resultType": "vector", 20:13:32 DEBUG "result": [ 20:13:32 DEBUG { 20:13:32 DEBUG "metric": { 20:13:32 DEBUG "pod": "ds-idrepo-0" 20:13:32 DEBUG }, 20:13:32 DEBUG "value": [ 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG [...] 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG "pod": "ds-cts-2" 20:13:32 DEBUG }, 20:13:32 DEBUG "value": [ 20:13:32 DEBUG 1713554012, 20:13:32 DEBUG "0" 20:13:32 DEBUG ] 20:13:32 DEBUG } 20:13:32 DEBUG ] 20:13:32 DEBUG } 20:13:32 DEBUG } 20:13:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554012" 20:13:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554012" 20:13:32 INFO [http_cmd]: http status code OK 20:13:32 DEBUG --- status code --- 20:13:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:32 DEBUG --- http response --- 20:13:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:13:32 DEBUG ----- output ----- 20:13:32 DEBUG { 20:13:32 DEBUG "status": "success", 20:13:32 DEBUG "data": { 20:13:32 DEBUG "resultType": "vector", 20:13:32 DEBUG "result": [ 20:13:32 DEBUG { 20:13:32 DEBUG "metric": { 20:13:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:13:32 DEBUG }, 20:13:32 DEBUG "value": [ 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG [...] 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:13:32 DEBUG }, 20:13:32 DEBUG "value": [ 20:13:32 DEBUG 1713554012, 20:13:32 DEBUG "0" 20:13:32 DEBUG ] 20:13:32 DEBUG } 20:13:32 DEBUG ] 20:13:32 DEBUG } 20:13:32 DEBUG } 20:13:32 INFO [http_cmd]: http status code OK 20:13:32 DEBUG --- status code --- 20:13:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:32 DEBUG --- http response --- 20:13:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:13:32 DEBUG ----- output ----- 20:13:32 DEBUG { 20:13:32 DEBUG "status": "success", 20:13:32 DEBUG "data": { 20:13:32 DEBUG "resultType": "vector", 20:13:32 DEBUG "result": [ 20:13:32 DEBUG { 20:13:32 DEBUG "metric": { 20:13:32 DEBUG "pod": "ds-idrepo-1" 20:13:32 DEBUG }, 20:13:32 DEBUG "value": [ 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG [...] 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:13:32 DEBUG }, 20:13:32 DEBUG "value": [ 20:13:32 DEBUG 1713554012, 20:13:32 DEBUG "0" 20:13:32 DEBUG ] 20:13:32 DEBUG } 20:13:32 DEBUG ] 20:13:32 DEBUG } 20:13:32 DEBUG } 20:13:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554012" 20:13:32 INFO [http_cmd]: http status code OK 20:13:32 DEBUG --- status code --- 20:13:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:32 DEBUG --- http response --- 20:13:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:13:32 DEBUG ----- output ----- 20:13:32 DEBUG { 20:13:32 DEBUG "status": "success", 20:13:32 DEBUG "data": { 20:13:32 DEBUG "resultType": "vector", 20:13:32 DEBUG "result": [ 20:13:32 DEBUG { 20:13:32 DEBUG "metric": { 20:13:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:13:32 DEBUG }, 20:13:32 DEBUG "value": [ 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG [...] 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:13:32 DEBUG }, 20:13:32 DEBUG "value": [ 20:13:32 DEBUG 1713554012, 20:13:32 DEBUG "2502.0333333333333" 20:13:32 DEBUG ] 20:13:32 DEBUG } 20:13:32 DEBUG ] 20:13:32 DEBUG } 20:13:32 DEBUG } 20:13:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713554012" 20:13:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554012" 20:13:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554012" 20:13:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554012" 20:13:32 INFO [http_cmd]: http status code OK 20:13:32 DEBUG --- status code --- 20:13:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:32 DEBUG --- http response --- 20:13:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:13:32 DEBUG ----- output ----- 20:13:32 DEBUG { 20:13:32 DEBUG "status": "success", 20:13:32 DEBUG "data": { 20:13:32 DEBUG "resultType": "vector", 20:13:32 DEBUG "result": [ 20:13:32 DEBUG { 20:13:32 DEBUG "metric": { 20:13:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:13:32 DEBUG }, 20:13:32 DEBUG "value": [ 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG [...] 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:13:32 DEBUG }, 20:13:32 DEBUG "value": [ 20:13:32 DEBUG 1713554012, 20:13:32 DEBUG "2499.2" 20:13:32 DEBUG ] 20:13:32 DEBUG } 20:13:32 DEBUG ] 20:13:32 DEBUG } 20:13:32 DEBUG } 20:13:32 INFO [http_cmd]: http status code OK 20:13:32 INFO [http_cmd]: http status code OK 20:13:32 DEBUG --- status code --- 20:13:32 DEBUG --- status code --- 20:13:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:32 DEBUG --- http response --- 20:13:32 DEBUG --- http response --- 20:13:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:13:32 DEBUG ----- output ----- 20:13:32 DEBUG { 20:13:32 DEBUG "status": "success", 20:13:32 DEBUG "data": { 20:13:32 INFO [http_cmd]: http status code OK 20:13:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:13:32 DEBUG "resultType": "vector", 20:13:32 DEBUG --- status code --- 20:13:32 DEBUG ----- output ----- 20:13:32 DEBUG "result": [ 20:13:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:32 DEBUG { 20:13:32 DEBUG { 20:13:32 DEBUG --- http response --- 20:13:32 DEBUG "status": "success", 20:13:32 DEBUG "metric": { 20:13:32 DEBUG "data": { 20:13:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:13:32 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:13:32 DEBUG "resultType": "vector", 20:13:32 DEBUG ----- output ----- 20:13:32 DEBUG }, 20:13:32 DEBUG "result": [ 20:13:32 DEBUG { 20:13:32 DEBUG "value": [ 20:13:32 DEBUG { 20:13:32 DEBUG "status": "success", 20:13:32 DEBUG 20:13:32 DEBUG "metric": { 20:13:32 DEBUG "data": { 20:13:32 DEBUG 20:13:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:13:32 DEBUG "resultType": "vector", 20:13:32 DEBUG 20:13:32 DEBUG }, 20:13:32 DEBUG "result": [ 20:13:32 DEBUG [...] 20:13:32 DEBUG "value": [ 20:13:32 DEBUG { 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG "metric": { 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG "pod": "ds-idrepo-0" 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG }, 20:13:32 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:13:32 DEBUG [...] 20:13:32 DEBUG "value": [ 20:13:32 DEBUG }, 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG "value": [ 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 1713554012, 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG "8.045020583452809" 20:13:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:13:32 DEBUG [...] 20:13:32 DEBUG ] 20:13:32 DEBUG }, 20:13:32 DEBUG 20:13:32 DEBUG } 20:13:32 DEBUG "value": [ 20:13:32 DEBUG 20:13:32 DEBUG ] 20:13:32 DEBUG 1713554012, 20:13:32 DEBUG 20:13:32 DEBUG } 20:13:32 DEBUG "2502.4333333333334" 20:13:32 DEBUG "pod": "ds-idrepo-2" 20:13:32 DEBUG } 20:13:32 DEBUG ] 20:13:32 DEBUG }, 20:13:32 DEBUG } 20:13:32 DEBUG "value": [ 20:13:32 DEBUG ] 20:13:32 DEBUG 1713554012, 20:13:32 DEBUG } 20:13:32 DEBUG "0" 20:13:32 DEBUG } 20:13:32 DEBUG ] 20:13:32 DEBUG } 20:13:32 DEBUG ] 20:13:32 DEBUG } 20:13:32 DEBUG } 20:13:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554012" 20:13:32 INFO [http_cmd]: http status code OK 20:13:32 DEBUG --- status code --- 20:13:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:32 DEBUG --- http response --- 20:13:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 20:13:32 DEBUG ----- output ----- 20:13:32 DEBUG { 20:13:32 DEBUG "status": "success", 20:13:32 DEBUG "data": { 20:13:32 DEBUG "resultType": "vector", 20:13:32 DEBUG "result": [ 20:13:32 DEBUG { 20:13:32 DEBUG "metric": { 20:13:32 DEBUG "pod": "ds-idrepo-1" 20:13:32 DEBUG }, 20:13:32 DEBUG "value": [ 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG [...] 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG "pod": "idm-b85f9d978-txpwv" 20:13:32 DEBUG }, 20:13:32 DEBUG "value": [ 20:13:32 DEBUG 1713554012, 20:13:32 DEBUG "531.89624387235" 20:13:32 DEBUG ] 20:13:32 DEBUG } 20:13:32 DEBUG ] 20:13:32 DEBUG } 20:13:32 DEBUG } 20:13:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554012" 20:13:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554012" 20:13:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713554012" 20:13:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554012" 20:13:32 INFO [http_cmd]: http status code OK 20:13:32 DEBUG --- status code --- 20:13:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:32 DEBUG --- http response --- 20:13:32 INFO [http_cmd]: http status code OK 20:13:32 INFO [http_cmd]: http status code OK 20:13:32 INFO [http_cmd]: http status code OK 20:13:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:13:32 DEBUG --- status code --- 20:13:32 DEBUG --- status code --- 20:13:32 DEBUG --- status code --- 20:13:32 DEBUG ----- output ----- 20:13:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:32 DEBUG { 20:13:32 DEBUG --- http response --- 20:13:32 DEBUG --- http response --- 20:13:32 DEBUG --- http response --- 20:13:32 DEBUG "status": "success", 20:13:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 20:13:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:13:32 DEBUG "data": { 20:13:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:13:32 DEBUG ----- output ----- 20:13:32 DEBUG ----- output ----- 20:13:32 DEBUG "resultType": "vector", 20:13:32 DEBUG ----- output ----- 20:13:32 DEBUG { 20:13:32 DEBUG { 20:13:32 DEBUG "result": [ 20:13:32 DEBUG { 20:13:32 DEBUG "status": "success", 20:13:32 DEBUG "status": "success", 20:13:32 DEBUG { 20:13:32 DEBUG "status": "success", 20:13:32 DEBUG "data": { 20:13:32 DEBUG "data": { 20:13:32 DEBUG "metric": { 20:13:32 DEBUG "data": { 20:13:32 DEBUG "resultType": "vector", 20:13:32 DEBUG "resultType": "vector", 20:13:32 DEBUG "pod": "ds-cts-0" 20:13:32 DEBUG "resultType": "vector", 20:13:32 DEBUG "result": [ 20:13:32 DEBUG "result": [ 20:13:32 DEBUG }, 20:13:32 DEBUG "result": [ 20:13:32 DEBUG { 20:13:32 DEBUG { 20:13:32 DEBUG "value": [ 20:13:32 DEBUG { 20:13:32 DEBUG "metric": { 20:13:32 DEBUG "metric": { 20:13:32 DEBUG 20:13:32 DEBUG "metric": { 20:13:32 DEBUG "pod": "ds-idrepo-1" 20:13:32 DEBUG "instance": "10.142.0.107:9100" 20:13:32 DEBUG 20:13:32 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:13:32 DEBUG }, 20:13:32 DEBUG }, 20:13:32 DEBUG 20:13:32 DEBUG }, 20:13:32 DEBUG "value": [ 20:13:32 DEBUG "value": [ 20:13:32 DEBUG [...] 20:13:32 DEBUG "value": [ 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG [...] 20:13:32 DEBUG [...] 20:13:32 DEBUG "pod": "ds-cts-2" 20:13:32 DEBUG [...] 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG }, 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG "value": [ 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 1713554012, 20:13:32 DEBUG 20:13:32 DEBUG "pod": "idm-b85f9d978-txpwv" 20:13:32 DEBUG "instance": "10.142.0.43:9100" 20:13:32 DEBUG "0" 20:13:32 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:13:32 DEBUG }, 20:13:32 DEBUG }, 20:13:32 DEBUG ] 20:13:32 DEBUG }, 20:13:32 DEBUG "value": [ 20:13:32 DEBUG "value": [ 20:13:32 DEBUG } 20:13:32 DEBUG "value": [ 20:13:32 DEBUG 1713554012, 20:13:32 DEBUG 1713554012, 20:13:32 DEBUG ] 20:13:32 DEBUG 1713554012, 20:13:32 DEBUG "0" 20:13:32 DEBUG "75560239.46666667" 20:13:32 DEBUG } 20:13:32 DEBUG "3679297536" 20:13:32 DEBUG ] 20:13:32 DEBUG ] 20:13:32 DEBUG } 20:13:32 DEBUG ] 20:13:32 DEBUG } 20:13:32 DEBUG } 20:13:32 DEBUG } 20:13:32 DEBUG ] 20:13:32 DEBUG ] 20:13:32 DEBUG ] 20:13:32 DEBUG } 20:13:32 DEBUG } 20:13:32 DEBUG } 20:13:32 DEBUG } 20:13:32 DEBUG } 20:13:32 DEBUG } 20:13:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554012" 20:13:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554012" 20:13:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554012" 20:13:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554012" 20:13:32 INFO [http_cmd]: http status code OK 20:13:32 DEBUG --- status code --- 20:13:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:32 DEBUG --- http response --- 20:13:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:13:32 DEBUG ----- output ----- 20:13:32 DEBUG { 20:13:32 DEBUG "status": "success", 20:13:32 DEBUG "data": { 20:13:32 DEBUG "resultType": "vector", 20:13:32 DEBUG "result": [ 20:13:32 DEBUG { 20:13:32 DEBUG "metric": { 20:13:32 DEBUG "pod": "ds-cts-0" 20:13:32 DEBUG }, 20:13:32 DEBUG "value": [ 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG [...] 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG "pod": "ds-cts-2" 20:13:32 DEBUG }, 20:13:32 DEBUG "value": [ 20:13:32 DEBUG 1713554012, 20:13:32 DEBUG "29.966666666666665" 20:13:32 DEBUG ] 20:13:32 DEBUG } 20:13:32 DEBUG ] 20:13:32 DEBUG } 20:13:32 DEBUG } 20:13:32 INFO [http_cmd]: http status code OK 20:13:32 DEBUG --- status code --- 20:13:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:32 DEBUG --- http response --- 20:13:32 INFO [http_cmd]: http status code OK 20:13:32 INFO [http_cmd]: http status code OK 20:13:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:13:32 DEBUG --- status code --- 20:13:32 DEBUG --- status code --- 20:13:32 DEBUG ----- output ----- 20:13:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:32 DEBUG { 20:13:32 DEBUG --- http response --- 20:13:32 DEBUG --- http response --- 20:13:32 DEBUG "status": "success", 20:13:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:13:32 DEBUG "data": { 20:13:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:13:32 DEBUG ----- output ----- 20:13:32 DEBUG "resultType": "vector", 20:13:32 DEBUG ----- output ----- 20:13:32 DEBUG { 20:13:32 DEBUG "result": [ 20:13:32 DEBUG { 20:13:32 DEBUG "status": "success", 20:13:32 DEBUG { 20:13:32 DEBUG "status": "success", 20:13:32 DEBUG "data": { 20:13:32 DEBUG "metric": { 20:13:32 DEBUG "data": { 20:13:32 DEBUG "resultType": "vector", 20:13:32 DEBUG "instance": "10.142.0.107:9100" 20:13:32 DEBUG "resultType": "vector", 20:13:32 DEBUG "result": [ 20:13:32 DEBUG }, 20:13:32 DEBUG "result": [ 20:13:32 DEBUG { 20:13:32 DEBUG "value": [ 20:13:32 DEBUG { 20:13:32 DEBUG "metric": { 20:13:32 DEBUG 20:13:32 DEBUG "metric": { 20:13:32 DEBUG "instance": "10.142.0.107:9100" 20:13:32 DEBUG 20:13:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:13:32 DEBUG }, 20:13:32 DEBUG 20:13:32 DEBUG }, 20:13:32 DEBUG "value": [ 20:13:32 DEBUG [...] 20:13:32 DEBUG "value": [ 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG [...] 20:13:32 DEBUG "instance": "10.142.0.43:9100" 20:13:32 DEBUG [...] 20:13:32 DEBUG 20:13:32 DEBUG }, 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG "value": [ 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 1713554012, 20:13:32 DEBUG 20:13:32 DEBUG "instance": "10.142.0.43:9100" 20:13:32 DEBUG "2048915592192" 20:13:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:13:32 DEBUG }, 20:13:32 DEBUG ] 20:13:32 DEBUG }, 20:13:32 DEBUG "value": [ 20:13:32 DEBUG } 20:13:32 DEBUG "value": [ 20:13:32 DEBUG 1713554012, 20:13:32 DEBUG ] 20:13:32 DEBUG 1713554012, 20:13:32 DEBUG "75557023.43333334" 20:13:32 DEBUG } 20:13:32 DEBUG "2502.3333333333335" 20:13:32 DEBUG ] 20:13:32 DEBUG } 20:13:32 DEBUG ] 20:13:32 DEBUG } 20:13:32 DEBUG } 20:13:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554012" 20:13:32 DEBUG ] 20:13:32 DEBUG ] 20:13:32 DEBUG } 20:13:32 DEBUG } 20:13:32 DEBUG } 20:13:32 DEBUG } 20:13:32 INFO [http_cmd]: http status code OK 20:13:32 DEBUG --- status code --- 20:13:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:32 DEBUG --- http response --- 20:13:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:13:32 DEBUG ----- output ----- 20:13:32 DEBUG { 20:13:32 DEBUG "status": "success", 20:13:32 DEBUG "data": { 20:13:32 DEBUG "resultType": "vector", 20:13:32 DEBUG "result": [ 20:13:32 DEBUG { 20:13:32 DEBUG "metric": { 20:13:32 DEBUG "pod": "ds-idrepo-1" 20:13:32 DEBUG }, 20:13:32 DEBUG "value": [ 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG [...] 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG 20:13:32 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:13:32 DEBUG }, 20:13:32 DEBUG "value": [ 20:13:32 DEBUG 1713554012, 20:13:32 DEBUG "0" 20:13:32 DEBUG ] 20:13:32 DEBUG } 20:13:32 DEBUG ] 20:13:32 DEBUG } 20:13:32 DEBUG } 20:13:34 INFO 20:13:34 INFO [loop_until]: kubectl --namespace=xlou top pods 20:13:34 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:13:34 INFO [loop_until]: OK (rc = 0) 20:13:34 DEBUG --- stdout --- 20:13:34 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 16149m 9007Mi am-767f6b67b5-qcqz2 16053m 8812Mi am-767f6b67b5-vbltz 17929m 8999Mi ds-cts-0 16938m 10979Mi ds-cts-1 15710m 11225Mi ds-cts-2 17179m 10983Mi ds-idrepo-0 6999m 3692Mi ds-idrepo-1 7176m 3537Mi ds-idrepo-2 7424m 3493Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8210m 3420Mi 20:13:34 DEBUG --- stderr --- 20:13:34 DEBUG 20:13:35 INFO 20:13:35 INFO [loop_until]: kubectl --namespace=xlou top node 20:13:35 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:13:35 INFO [loop_until]: OK (rc = 0) 20:13:35 DEBUG --- stdout --- 20:13:35 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 67m 0% 6240Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7415m 24% 7288Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16086m 53% 11703Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 85m 0% 3342Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 17316m 58% 13937Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 145m 0% 5169Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 16650m 55% 11695Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15789m 52% 2977Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 18252m 61% 11216Mi 10% gke-xlou-cdm-ds-0f601751-j488 7663m 25% 6756Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 15683m 52% 15991Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 7491m 25% 7421Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 17830m 59% 13426Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 8196m 51% 7132Mi 12% 20:13:35 DEBUG --- stderr --- 20:13:35 DEBUG 20:14:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554072" 20:14:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554072" 20:14:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713554072" 20:14:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554072" 20:14:32 INFO [http_cmd]: http status code OK 20:14:32 DEBUG --- status code --- 20:14:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:32 DEBUG --- http response --- 20:14:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:14:32 DEBUG ----- output ----- 20:14:32 DEBUG { 20:14:32 DEBUG "status": "success", 20:14:32 DEBUG "data": { 20:14:32 DEBUG "resultType": "vector", 20:14:32 DEBUG "result": [ 20:14:32 DEBUG { 20:14:32 DEBUG "metric": { 20:14:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:14:32 DEBUG }, 20:14:32 DEBUG "value": [ 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG [...] 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:14:32 DEBUG }, 20:14:32 DEBUG "value": [ 20:14:32 DEBUG 1713554072, 20:14:32 DEBUG "450099.5" 20:14:32 DEBUG ] 20:14:32 DEBUG } 20:14:32 DEBUG ] 20:14:32 DEBUG } 20:14:32 DEBUG } 20:14:32 INFO [http_cmd]: http status code OK 20:14:32 DEBUG --- status code --- 20:14:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:32 DEBUG --- http response --- 20:14:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:14:32 DEBUG ----- output ----- 20:14:32 DEBUG { 20:14:32 DEBUG "status": "success", 20:14:32 DEBUG "data": { 20:14:32 DEBUG "resultType": "vector", 20:14:32 DEBUG "result": [ 20:14:32 DEBUG { 20:14:32 DEBUG "metric": { 20:14:32 DEBUG "instance": "10.142.0.107:9100" 20:14:32 DEBUG }, 20:14:32 DEBUG "value": [ 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG [...] 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG "instance": "10.142.0.43:9100" 20:14:32 DEBUG }, 20:14:32 DEBUG "value": [ 20:14:32 DEBUG 1713554072, 20:14:32 DEBUG "0.07466666666666885" 20:14:32 DEBUG ] 20:14:32 DEBUG } 20:14:32 DEBUG ] 20:14:32 DEBUG } 20:14:32 DEBUG } 20:14:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554072" 20:14:32 INFO [http_cmd]: http status code OK 20:14:32 ERROR [http_cmd]: ERROR 20:14:32 DEBUG --- status code --- 20:14:32 ERROR --- status code --- 20:14:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:32 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:14:32 DEBUG --- http response --- 20:14:32 ERROR --- http response --- 20:14:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:14:32 DEBUG ----- output ----- 20:14:32 DEBUG { 20:14:32 DEBUG "status": "success", 20:14:32 DEBUG "data": { 20:14:32 DEBUG "resultType": "vector", 20:14:32 DEBUG "result": [ 20:14:32 DEBUG { 20:14:32 DEBUG "metric": { 20:14:32 DEBUG "pod": "ds-idrepo-0" 20:14:32 DEBUG }, 20:14:32 DEBUG "value": [ 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG [...] 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG "pod": "ds-idrepo-2" 20:14:32 DEBUG }, 20:14:32 DEBUG "value": [ 20:14:32 DEBUG 1713554072, 20:14:32 DEBUG "0" 20:14:32 DEBUG ] 20:14:32 DEBUG } 20:14:32 DEBUG ] 20:14:32 DEBUG } 20:14:32 DEBUG } 20:14:32 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:14:32 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:14:32 INFO [http_cmd]: http status code OK 20:14:32 DEBUG --- status code --- 20:14:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:32 DEBUG --- http response --- 20:14:32 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:14:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554072" 20:14:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713554072" 20:14:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554072" 20:14:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554072" 20:14:32 INFO [http_cmd]: http status code OK 20:14:32 DEBUG --- status code --- 20:14:32 INFO [http_cmd]: http status code OK 20:14:32 INFO [http_cmd]: http status code OK 20:14:32 INFO [http_cmd]: http status code OK 20:14:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:32 DEBUG --- status code --- 20:14:32 DEBUG --- status code --- 20:14:32 DEBUG --- status code --- 20:14:32 DEBUG --- http response --- 20:14:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:14:32 DEBUG --- http response --- 20:14:32 DEBUG --- http response --- 20:14:32 DEBUG --- http response --- 20:14:32 DEBUG ----- output ----- 20:14:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:14:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:14:32 DEBUG { 20:14:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:14:32 DEBUG ----- output ----- 20:14:32 DEBUG ----- output ----- 20:14:32 DEBUG "status": "success", 20:14:32 DEBUG ----- output ----- 20:14:32 DEBUG { 20:14:32 DEBUG { 20:14:32 DEBUG "data": { 20:14:32 DEBUG { 20:14:32 DEBUG "status": "success", 20:14:32 DEBUG "status": "success", 20:14:32 DEBUG "resultType": "vector", 20:14:32 DEBUG "status": "success", 20:14:32 DEBUG "data": { 20:14:32 DEBUG "data": { 20:14:32 DEBUG "result": [ 20:14:32 DEBUG "data": { 20:14:32 DEBUG "resultType": "vector", 20:14:32 DEBUG "resultType": "vector", 20:14:32 DEBUG { 20:14:32 DEBUG "resultType": "vector", 20:14:32 DEBUG "result": [ 20:14:32 DEBUG "result": [ 20:14:32 DEBUG "metric": { 20:14:32 DEBUG "result": [ 20:14:32 DEBUG { 20:14:32 DEBUG { 20:14:32 DEBUG "pod": "ds-idrepo-0" 20:14:32 DEBUG { 20:14:32 DEBUG "metric": { 20:14:32 DEBUG "metric": { 20:14:32 DEBUG }, 20:14:32 DEBUG "metric": { 20:14:32 DEBUG "pod": "ds-idrepo-0" 20:14:32 DEBUG "instance": "10.142.0.107:9100" 20:14:32 DEBUG "value": [ 20:14:32 DEBUG "pod": "ds-cts-0" 20:14:32 DEBUG }, 20:14:32 DEBUG }, 20:14:32 DEBUG 20:14:32 DEBUG }, 20:14:32 DEBUG "value": [ 20:14:32 DEBUG "value": [ 20:14:32 DEBUG 20:14:32 DEBUG "value": [ 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG [...] 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG [...] 20:14:32 DEBUG [...] 20:14:32 DEBUG 20:14:32 DEBUG [...] 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG "pod": "ds-idrepo-2" 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG }, 20:14:32 DEBUG 20:14:32 DEBUG "pod": "ds-cts-2" 20:14:32 DEBUG "instance": "10.142.0.43:9100" 20:14:32 DEBUG "value": [ 20:14:32 DEBUG "pod": "ds-cts-2" 20:14:32 DEBUG }, 20:14:32 DEBUG }, 20:14:32 DEBUG 1713554072, 20:14:32 DEBUG }, 20:14:32 DEBUG "value": [ 20:14:32 DEBUG "value": [ 20:14:32 DEBUG "0" 20:14:32 DEBUG "value": [ 20:14:32 DEBUG 1713554072, 20:14:32 DEBUG 1713554072, 20:14:32 DEBUG ] 20:14:32 DEBUG 1713554072, 20:14:32 DEBUG "0" 20:14:32 DEBUG "0.1684666666667302" 20:14:32 DEBUG } 20:14:32 DEBUG "0.0005313133333333333" 20:14:32 DEBUG ] 20:14:32 DEBUG ] 20:14:32 DEBUG ] 20:14:32 DEBUG ] 20:14:32 DEBUG } 20:14:32 DEBUG } 20:14:32 DEBUG } 20:14:32 DEBUG } 20:14:32 DEBUG ] 20:14:32 DEBUG ] 20:14:32 DEBUG } 20:14:32 DEBUG ] 20:14:32 DEBUG } 20:14:32 DEBUG } 20:14:32 DEBUG } 20:14:32 DEBUG } 20:14:32 DEBUG } 20:14:32 DEBUG } 20:14:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713554072" 20:14:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554072" 20:14:32 INFO [http_cmd]: http status code OK 20:14:32 DEBUG --- status code --- 20:14:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:32 DEBUG --- http response --- 20:14:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:14:32 DEBUG ----- output ----- 20:14:32 DEBUG { 20:14:32 DEBUG "status": "success", 20:14:32 DEBUG "data": { 20:14:32 DEBUG "resultType": "vector", 20:14:32 DEBUG "result": [ 20:14:32 DEBUG { 20:14:32 DEBUG "metric": { 20:14:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:14:32 DEBUG }, 20:14:32 DEBUG "value": [ 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG [...] 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:14:32 DEBUG }, 20:14:32 DEBUG "value": [ 20:14:32 DEBUG 1713554072, 20:14:32 DEBUG "0.015" 20:14:32 DEBUG ] 20:14:32 DEBUG } 20:14:32 DEBUG ] 20:14:32 DEBUG } 20:14:32 DEBUG } 20:14:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554072" 20:14:32 INFO [http_cmd]: http status code OK 20:14:32 DEBUG --- status code --- 20:14:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:32 DEBUG --- http response --- 20:14:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:14:32 DEBUG ----- output ----- 20:14:32 DEBUG { 20:14:32 DEBUG "status": "success", 20:14:32 DEBUG "data": { 20:14:32 DEBUG "resultType": "vector", 20:14:32 DEBUG "result": [ 20:14:32 DEBUG { 20:14:32 DEBUG "metric": { 20:14:32 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:14:32 DEBUG }, 20:14:32 DEBUG "value": [ 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG [...] 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 20:14:32 DEBUG }, 20:14:32 DEBUG "value": [ 20:14:32 DEBUG 1713554072, 20:14:32 DEBUG "0" 20:14:32 DEBUG ] 20:14:32 DEBUG } 20:14:32 DEBUG ] 20:14:32 DEBUG } 20:14:32 DEBUG } 20:14:32 INFO [http_cmd]: http status code OK 20:14:32 DEBUG --- status code --- 20:14:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:32 DEBUG --- http response --- 20:14:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:14:32 DEBUG ----- output ----- 20:14:32 DEBUG { 20:14:32 DEBUG "status": "success", 20:14:32 DEBUG "data": { 20:14:32 DEBUG "resultType": "vector", 20:14:32 DEBUG "result": [ 20:14:32 DEBUG { 20:14:32 DEBUG "metric": { 20:14:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:14:32 DEBUG }, 20:14:32 DEBUG "value": [ 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG [...] 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:14:32 DEBUG }, 20:14:32 DEBUG "value": [ 20:14:32 DEBUG 1713554072, 20:14:32 DEBUG "0" 20:14:32 DEBUG ] 20:14:32 DEBUG } 20:14:32 DEBUG ] 20:14:32 DEBUG } 20:14:32 DEBUG } 20:14:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554072" 20:14:32 INFO [http_cmd]: http status code OK 20:14:32 DEBUG --- status code --- 20:14:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:32 DEBUG --- http response --- 20:14:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:14:32 DEBUG ----- output ----- 20:14:32 DEBUG { 20:14:32 DEBUG "status": "success", 20:14:32 DEBUG "data": { 20:14:32 DEBUG "resultType": "vector", 20:14:32 DEBUG "result": [ 20:14:32 DEBUG { 20:14:32 DEBUG "metric": { 20:14:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:14:32 DEBUG }, 20:14:32 DEBUG "value": [ 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG [...] 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:14:32 DEBUG }, 20:14:32 DEBUG "value": [ 20:14:32 DEBUG 1713554072, 20:14:32 DEBUG "2485.5" 20:14:32 DEBUG ] 20:14:32 DEBUG } 20:14:32 DEBUG ] 20:14:32 DEBUG } 20:14:32 DEBUG } 20:14:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554072" 20:14:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713554072" 20:14:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554072" 20:14:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554072" 20:14:32 INFO [http_cmd]: http status code OK 20:14:32 DEBUG --- status code --- 20:14:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:32 DEBUG --- http response --- 20:14:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:14:32 DEBUG ----- output ----- 20:14:32 DEBUG { 20:14:32 DEBUG "status": "success", 20:14:32 DEBUG "data": { 20:14:32 DEBUG "resultType": "vector", 20:14:32 DEBUG "result": [ 20:14:32 DEBUG { 20:14:32 DEBUG "metric": { 20:14:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:14:32 DEBUG }, 20:14:32 DEBUG "value": [ 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG [...] 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:14:32 DEBUG }, 20:14:32 DEBUG "value": [ 20:14:32 DEBUG 1713554072, 20:14:32 DEBUG "2506.6666666666665" 20:14:32 DEBUG ] 20:14:32 DEBUG } 20:14:32 DEBUG ] 20:14:32 DEBUG } 20:14:32 DEBUG } 20:14:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554072" 20:14:32 INFO [http_cmd]: http status code OK 20:14:32 INFO [http_cmd]: http status code OK 20:14:32 DEBUG --- status code --- 20:14:32 DEBUG --- status code --- 20:14:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:32 DEBUG --- http response --- 20:14:32 DEBUG --- http response --- 20:14:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:14:32 INFO [http_cmd]: http status code OK 20:14:32 DEBUG ----- output ----- 20:14:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:14:32 DEBUG --- status code --- 20:14:32 DEBUG { 20:14:32 DEBUG ----- output ----- 20:14:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:32 DEBUG "status": "success", 20:14:32 DEBUG { 20:14:32 DEBUG --- http response --- 20:14:32 DEBUG "data": { 20:14:32 DEBUG "status": "success", 20:14:32 DEBUG "resultType": "vector", 20:14:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:14:32 DEBUG "data": { 20:14:32 DEBUG "result": [ 20:14:32 DEBUG ----- output ----- 20:14:32 DEBUG "resultType": "vector", 20:14:32 DEBUG { 20:14:32 DEBUG { 20:14:32 DEBUG "result": [ 20:14:32 DEBUG "metric": { 20:14:32 DEBUG "status": "success", 20:14:32 DEBUG { 20:14:32 DEBUG "pod": "ds-idrepo-0" 20:14:32 DEBUG "data": { 20:14:32 DEBUG "metric": { 20:14:32 DEBUG }, 20:14:32 DEBUG "resultType": "vector", 20:14:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:14:32 DEBUG "value": [ 20:14:32 DEBUG "result": [ 20:14:32 DEBUG }, 20:14:32 DEBUG 20:14:32 DEBUG { 20:14:32 DEBUG "value": [ 20:14:32 DEBUG 20:14:32 DEBUG "metric": { 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:14:32 DEBUG 20:14:32 DEBUG [...] 20:14:32 DEBUG }, 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG "value": [ 20:14:32 DEBUG [...] 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG "pod": "ds-idrepo-2" 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG }, 20:14:32 DEBUG [...] 20:14:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:14:32 DEBUG "value": [ 20:14:32 DEBUG 20:14:32 DEBUG }, 20:14:32 DEBUG 1713554072, 20:14:32 DEBUG 20:14:32 DEBUG "value": [ 20:14:32 DEBUG "0" 20:14:32 DEBUG 20:14:32 DEBUG 1713554072, 20:14:32 DEBUG ] 20:14:32 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:14:32 DEBUG "2483.7" 20:14:32 DEBUG } 20:14:32 DEBUG }, 20:14:32 DEBUG ] 20:14:32 DEBUG ] 20:14:32 DEBUG "value": [ 20:14:32 DEBUG } 20:14:32 DEBUG } 20:14:32 DEBUG 1713554072, 20:14:32 DEBUG ] 20:14:32 DEBUG } 20:14:32 DEBUG "8.043251465331938" 20:14:32 DEBUG } 20:14:32 DEBUG ] 20:14:32 DEBUG } 20:14:32 DEBUG } 20:14:32 DEBUG ] 20:14:32 DEBUG } 20:14:32 DEBUG } 20:14:32 INFO [http_cmd]: http status code OK 20:14:32 DEBUG --- status code --- 20:14:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:32 DEBUG --- http response --- 20:14:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:14:32 DEBUG ----- output ----- 20:14:32 DEBUG { 20:14:32 DEBUG "status": "success", 20:14:32 DEBUG "data": { 20:14:32 DEBUG "resultType": "vector", 20:14:32 DEBUG "result": [ 20:14:32 DEBUG { 20:14:32 DEBUG "metric": { 20:14:32 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:14:32 DEBUG }, 20:14:32 DEBUG "value": [ 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG [...] 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 20:14:32 DEBUG }, 20:14:32 DEBUG "value": [ 20:14:32 DEBUG 1713554072, 20:14:32 DEBUG "0" 20:14:32 DEBUG ] 20:14:32 DEBUG } 20:14:32 DEBUG ] 20:14:32 DEBUG } 20:14:32 DEBUG } 20:14:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554072" 20:14:32 INFO [http_cmd]: http status code OK 20:14:32 DEBUG --- status code --- 20:14:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:32 DEBUG --- http response --- 20:14:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:14:32 DEBUG ----- output ----- 20:14:32 DEBUG { 20:14:32 DEBUG "status": "success", 20:14:32 DEBUG "data": { 20:14:32 DEBUG "resultType": "vector", 20:14:32 DEBUG "result": [ 20:14:32 DEBUG { 20:14:32 DEBUG "metric": { 20:14:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:14:32 DEBUG }, 20:14:32 DEBUG "value": [ 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG [...] 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:14:32 DEBUG }, 20:14:32 DEBUG "value": [ 20:14:32 DEBUG 1713554072, 20:14:32 DEBUG "2485.6" 20:14:32 DEBUG ] 20:14:32 DEBUG } 20:14:32 DEBUG ] 20:14:32 DEBUG } 20:14:32 DEBUG } 20:14:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554072" 20:14:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554072" 20:14:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713554072" 20:14:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554072" 20:14:32 INFO [http_cmd]: http status code OK 20:14:32 DEBUG --- status code --- 20:14:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:32 DEBUG --- http response --- 20:14:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:14:32 DEBUG ----- output ----- 20:14:32 DEBUG { 20:14:32 DEBUG "status": "success", 20:14:32 DEBUG "data": { 20:14:32 DEBUG "resultType": "vector", 20:14:32 DEBUG "result": [ 20:14:32 DEBUG { 20:14:32 DEBUG "metric": { 20:14:32 DEBUG "instance": "10.142.0.107:9100" 20:14:32 DEBUG }, 20:14:32 DEBUG "value": [ 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG [...] 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG "instance": "10.142.0.43:9100" 20:14:32 DEBUG }, 20:14:32 DEBUG "value": [ 20:14:32 DEBUG 1713554072, 20:14:32 DEBUG "75169257.9" 20:14:32 DEBUG ] 20:14:32 DEBUG } 20:14:32 DEBUG ] 20:14:32 DEBUG } 20:14:32 DEBUG } 20:14:32 INFO [http_cmd]: http status code OK 20:14:32 DEBUG --- status code --- 20:14:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:32 DEBUG --- http response --- 20:14:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:14:32 DEBUG ----- output ----- 20:14:32 DEBUG { 20:14:32 DEBUG "status": "success", 20:14:32 DEBUG "data": { 20:14:32 DEBUG "resultType": "vector", 20:14:32 DEBUG "result": [ 20:14:32 DEBUG { 20:14:32 DEBUG "metric": { 20:14:32 DEBUG "pod": "ds-cts-0" 20:14:32 DEBUG }, 20:14:32 DEBUG "value": [ 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG [...] 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG "pod": "ds-cts-2" 20:14:32 DEBUG }, 20:14:32 DEBUG "value": [ 20:14:32 DEBUG 1713554072, 20:14:32 DEBUG "0" 20:14:32 DEBUG ] 20:14:32 DEBUG } 20:14:32 DEBUG ] 20:14:32 DEBUG } 20:14:32 DEBUG } 20:14:32 INFO [http_cmd]: http status code OK 20:14:32 DEBUG --- status code --- 20:14:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:32 DEBUG --- http response --- 20:14:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:14:32 DEBUG ----- output ----- 20:14:32 DEBUG { 20:14:32 DEBUG "status": "success", 20:14:32 DEBUG "data": { 20:14:32 DEBUG "resultType": "vector", 20:14:32 DEBUG "result": [ 20:14:32 DEBUG { 20:14:32 DEBUG "metric": { 20:14:32 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:14:32 DEBUG }, 20:14:32 DEBUG "value": [ 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG [...] 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 20:14:32 DEBUG }, 20:14:32 DEBUG "value": [ 20:14:32 DEBUG 1713554072, 20:14:32 DEBUG "0" 20:14:32 DEBUG ] 20:14:32 DEBUG } 20:14:32 DEBUG ] 20:14:32 DEBUG } 20:14:32 DEBUG } 20:14:32 INFO [http_cmd]: http status code OK 20:14:32 DEBUG --- status code --- 20:14:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:32 DEBUG --- http response --- 20:14:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554072" 20:14:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:14:32 DEBUG ----- output ----- 20:14:32 DEBUG { 20:14:32 DEBUG "status": "success", 20:14:32 DEBUG "data": { 20:14:32 DEBUG "resultType": "vector", 20:14:32 DEBUG "result": [ 20:14:32 DEBUG { 20:14:32 DEBUG "metric": { 20:14:32 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:14:32 DEBUG }, 20:14:32 DEBUG "value": [ 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG [...] 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:14:32 DEBUG }, 20:14:32 DEBUG "value": [ 20:14:32 DEBUG 1713554072, 20:14:32 DEBUG "3683741696" 20:14:32 DEBUG ] 20:14:32 DEBUG } 20:14:32 DEBUG ] 20:14:32 DEBUG } 20:14:32 DEBUG } 20:14:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554072" 20:14:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554072" 20:14:32 INFO [http_cmd]: http status code OK 20:14:32 DEBUG --- status code --- 20:14:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:32 DEBUG --- http response --- 20:14:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:14:32 DEBUG ----- output ----- 20:14:32 DEBUG { 20:14:32 DEBUG "status": "success", 20:14:32 DEBUG "data": { 20:14:32 DEBUG "resultType": "vector", 20:14:32 DEBUG "result": [ 20:14:32 DEBUG { 20:14:32 DEBUG "metric": { 20:14:32 DEBUG "pod": "ds-cts-0" 20:14:32 DEBUG }, 20:14:32 DEBUG "value": [ 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG [...] 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG "pod": "ds-cts-2" 20:14:32 DEBUG }, 20:14:32 DEBUG "value": [ 20:14:32 DEBUG 1713554072, 20:14:32 DEBUG "30" 20:14:32 DEBUG ] 20:14:32 DEBUG } 20:14:32 DEBUG ] 20:14:32 DEBUG } 20:14:32 DEBUG } 20:14:32 INFO [http_cmd]: http status code OK 20:14:32 DEBUG --- status code --- 20:14:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:32 DEBUG --- http response --- 20:14:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:14:32 DEBUG ----- output ----- 20:14:32 DEBUG { 20:14:32 DEBUG "status": "success", 20:14:32 DEBUG "data": { 20:14:32 DEBUG "resultType": "vector", 20:14:32 DEBUG "result": [ 20:14:32 DEBUG { 20:14:32 DEBUG "metric": { 20:14:32 DEBUG "instance": "10.142.0.107:9100" 20:14:32 DEBUG }, 20:14:32 DEBUG "value": [ 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG [...] 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG "instance": "10.142.0.43:9100" 20:14:32 DEBUG }, 20:14:32 DEBUG "value": [ 20:14:32 DEBUG 1713554072, 20:14:32 DEBUG "2052537764864" 20:14:32 DEBUG ] 20:14:32 DEBUG } 20:14:32 DEBUG ] 20:14:32 DEBUG } 20:14:32 DEBUG } 20:14:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554072" 20:14:32 INFO [http_cmd]: http status code OK 20:14:32 DEBUG --- status code --- 20:14:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:32 DEBUG --- http response --- 20:14:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:14:32 DEBUG ----- output ----- 20:14:32 DEBUG { 20:14:32 DEBUG "status": "success", 20:14:32 DEBUG "data": { 20:14:32 DEBUG "resultType": "vector", 20:14:32 DEBUG "result": [ 20:14:32 DEBUG { 20:14:32 DEBUG "metric": { 20:14:32 DEBUG "instance": "10.142.0.107:9100" 20:14:32 DEBUG }, 20:14:32 DEBUG "value": [ 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG [...] 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG "instance": "10.142.0.43:9100" 20:14:32 DEBUG }, 20:14:32 DEBUG "value": [ 20:14:32 DEBUG 1713554072, 20:14:32 DEBUG "75167175.6" 20:14:32 DEBUG ] 20:14:32 DEBUG } 20:14:32 DEBUG ] 20:14:32 DEBUG } 20:14:32 DEBUG } 20:14:32 INFO [http_cmd]: http status code OK 20:14:32 DEBUG --- status code --- 20:14:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:32 DEBUG --- http response --- 20:14:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:14:32 DEBUG ----- output ----- 20:14:32 DEBUG { 20:14:32 DEBUG "status": "success", 20:14:32 DEBUG "data": { 20:14:32 DEBUG "resultType": "vector", 20:14:32 DEBUG "result": [ 20:14:32 DEBUG { 20:14:32 DEBUG "metric": { 20:14:32 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:14:32 DEBUG }, 20:14:32 DEBUG "value": [ 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG [...] 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG 20:14:32 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 20:14:32 DEBUG }, 20:14:32 DEBUG "value": [ 20:14:32 DEBUG 1713554072, 20:14:32 DEBUG "0.00002392366030882197" 20:14:32 DEBUG ] 20:14:32 DEBUG } 20:14:32 DEBUG ] 20:14:32 DEBUG } 20:14:32 DEBUG } 20:14:34 INFO 20:14:34 INFO [loop_until]: kubectl --namespace=xlou top pods 20:14:34 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:14:34 INFO [loop_until]: OK (rc = 0) 20:14:34 DEBUG --- stdout --- 20:14:34 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 15734m 9007Mi am-767f6b67b5-qcqz2 16650m 8812Mi am-767f6b67b5-vbltz 17358m 9001Mi ds-cts-0 17383m 10965Mi ds-cts-1 15352m 11240Mi ds-cts-2 17245m 10998Mi ds-idrepo-0 7085m 3691Mi ds-idrepo-1 7263m 3537Mi ds-idrepo-2 7322m 3492Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8069m 3421Mi 20:14:34 DEBUG --- stderr --- 20:14:34 DEBUG 20:14:35 INFO 20:14:35 INFO [loop_until]: kubectl --namespace=xlou top node 20:14:35 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:14:35 INFO [loop_until]: OK (rc = 0) 20:14:35 DEBUG --- stdout --- 20:14:35 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 69m 0% 6252Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7326m 24% 7296Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 15994m 53% 11722Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 82m 0% 3343Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 17204m 57% 13939Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 162m 0% 5174Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 16317m 54% 11693Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15233m 51% 2973Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 18030m 60% 11206Mi 10% gke-xlou-cdm-ds-0f601751-j488 7647m 25% 6749Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 15692m 52% 15987Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 7454m 24% 7436Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 17304m 57% 13434Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 8200m 51% 7133Mi 12% 20:14:35 DEBUG --- stderr --- 20:14:35 DEBUG 20:15:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554132" 20:15:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554132" 20:15:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554132" 20:15:32 INFO [http_cmd]: http status code OK 20:15:32 DEBUG --- status code --- 20:15:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:32 DEBUG --- http response --- 20:15:32 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:15:32 INFO [http_cmd]: http status code OK 20:15:32 DEBUG --- status code --- 20:15:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:32 DEBUG --- http response --- 20:15:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:15:32 DEBUG ----- output ----- 20:15:32 DEBUG { 20:15:32 DEBUG "status": "success", 20:15:32 DEBUG "data": { 20:15:32 DEBUG "resultType": "vector", 20:15:32 DEBUG "result": [ 20:15:32 DEBUG { 20:15:32 DEBUG "metric": { 20:15:32 DEBUG "instance": "10.142.0.107:9100" 20:15:32 DEBUG }, 20:15:32 DEBUG "value": [ 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG [...] 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG "instance": "10.142.0.43:9100" 20:15:32 DEBUG }, 20:15:32 DEBUG "value": [ 20:15:32 DEBUG 1713554132, 20:15:32 DEBUG "0.11200000000000046" 20:15:32 DEBUG ] 20:15:32 DEBUG } 20:15:32 DEBUG ] 20:15:32 DEBUG } 20:15:32 DEBUG } 20:15:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713554132" 20:15:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554132" 20:15:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554132" 20:15:32 INFO [http_cmd]: http status code OK 20:15:32 DEBUG --- status code --- 20:15:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:32 DEBUG --- http response --- 20:15:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:15:32 DEBUG ----- output ----- 20:15:32 DEBUG { 20:15:32 DEBUG "status": "success", 20:15:32 DEBUG "data": { 20:15:32 DEBUG "resultType": "vector", 20:15:32 DEBUG "result": [ 20:15:32 DEBUG { 20:15:32 DEBUG "metric": { 20:15:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:15:32 DEBUG }, 20:15:32 DEBUG "value": [ 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG [...] 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:15:32 DEBUG }, 20:15:32 DEBUG "value": [ 20:15:32 DEBUG 1713554132, 20:15:32 DEBUG "449781" 20:15:32 DEBUG ] 20:15:32 DEBUG } 20:15:32 DEBUG ] 20:15:32 DEBUG } 20:15:32 DEBUG } 20:15:32 ERROR [http_cmd]: ERROR 20:15:32 ERROR --- status code --- 20:15:32 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:15:32 ERROR --- http response --- 20:15:32 INFO [http_cmd]: http status code OK 20:15:32 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:15:32 DEBUG --- status code --- 20:15:32 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:15:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:32 DEBUG --- http response --- 20:15:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:15:32 DEBUG ----- output ----- 20:15:32 DEBUG { 20:15:32 DEBUG "status": "success", 20:15:32 DEBUG "data": { 20:15:32 DEBUG "resultType": "vector", 20:15:32 INFO [http_cmd]: http status code OK 20:15:32 DEBUG "result": [ 20:15:32 DEBUG --- status code --- 20:15:32 DEBUG { 20:15:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:32 DEBUG "metric": { 20:15:32 DEBUG --- http response --- 20:15:32 DEBUG "pod": "ds-idrepo-0" 20:15:32 DEBUG }, 20:15:32 DEBUG "value": [ 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG [...] 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG "pod": "ds-idrepo-2" 20:15:32 DEBUG }, 20:15:32 DEBUG "value": [ 20:15:32 DEBUG 1713554132, 20:15:32 DEBUG "0" 20:15:32 DEBUG ] 20:15:32 DEBUG } 20:15:32 DEBUG ] 20:15:32 DEBUG } 20:15:32 DEBUG } 20:15:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:15:32 DEBUG ----- output ----- 20:15:32 DEBUG { 20:15:32 DEBUG "status": "success", 20:15:32 DEBUG "data": { 20:15:32 DEBUG "resultType": "vector", 20:15:32 DEBUG "result": [ 20:15:32 DEBUG { 20:15:32 DEBUG "metric": { 20:15:32 DEBUG "pod": "ds-idrepo-0" 20:15:32 DEBUG }, 20:15:32 DEBUG "value": [ 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG [...] 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG "pod": "ds-idrepo-2" 20:15:32 DEBUG }, 20:15:32 DEBUG "value": [ 20:15:32 DEBUG 1713554132, 20:15:32 DEBUG "0" 20:15:32 DEBUG ] 20:15:32 DEBUG } 20:15:32 DEBUG ] 20:15:32 DEBUG } 20:15:32 DEBUG } 20:15:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554132" 20:15:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554132" 20:15:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713554132" 20:15:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554132" 20:15:32 INFO [http_cmd]: http status code OK 20:15:32 INFO [http_cmd]: http status code OK 20:15:32 INFO [http_cmd]: http status code OK 20:15:32 DEBUG --- status code --- 20:15:32 DEBUG --- status code --- 20:15:32 DEBUG --- status code --- 20:15:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:32 DEBUG --- http response --- 20:15:32 DEBUG --- http response --- 20:15:32 DEBUG --- http response --- 20:15:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:15:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:15:32 DEBUG ----- output ----- 20:15:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:15:32 DEBUG ----- output ----- 20:15:32 DEBUG { 20:15:32 DEBUG ----- output ----- 20:15:32 DEBUG { 20:15:32 DEBUG "status": "success", 20:15:32 DEBUG { 20:15:32 DEBUG "status": "success", 20:15:32 DEBUG "data": { 20:15:32 DEBUG "status": "success", 20:15:32 DEBUG "data": { 20:15:32 DEBUG "resultType": "vector", 20:15:32 DEBUG "data": { 20:15:32 DEBUG "resultType": "vector", 20:15:32 DEBUG "result": [ 20:15:32 DEBUG "resultType": "vector", 20:15:32 DEBUG "result": [ 20:15:32 DEBUG { 20:15:32 DEBUG "result": [ 20:15:32 DEBUG { 20:15:32 DEBUG "metric": { 20:15:32 DEBUG { 20:15:32 DEBUG "metric": { 20:15:32 DEBUG "pod": "ds-cts-0" 20:15:32 DEBUG "metric": { 20:15:32 DEBUG "pod": "ds-idrepo-0" 20:15:32 DEBUG }, 20:15:32 DEBUG "instance": "10.142.0.107:9100" 20:15:32 DEBUG }, 20:15:32 DEBUG "value": [ 20:15:32 DEBUG }, 20:15:32 DEBUG "value": [ 20:15:32 DEBUG 20:15:32 DEBUG "value": [ 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG [...] 20:15:32 DEBUG 20:15:32 DEBUG [...] 20:15:32 DEBUG 20:15:32 DEBUG [...] 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG "pod": "ds-cts-2" 20:15:32 DEBUG 20:15:32 DEBUG "pod": "ds-cts-2" 20:15:32 DEBUG }, 20:15:32 DEBUG "instance": "10.142.0.43:9100" 20:15:32 DEBUG }, 20:15:32 DEBUG "value": [ 20:15:32 DEBUG }, 20:15:32 INFO [http_cmd]: http status code OK 20:15:32 DEBUG "value": [ 20:15:32 DEBUG 1713554132, 20:15:32 DEBUG "value": [ 20:15:32 DEBUG --- status code --- 20:15:32 DEBUG 1713554132, 20:15:32 DEBUG "0.00038013555555555553" 20:15:32 DEBUG 1713554132, 20:15:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:32 DEBUG "0" 20:15:32 DEBUG ] 20:15:32 DEBUG "0.2641333333333146" 20:15:32 DEBUG --- http response --- 20:15:32 DEBUG ] 20:15:32 DEBUG } 20:15:32 DEBUG ] 20:15:32 DEBUG } 20:15:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:15:32 DEBUG ] 20:15:32 DEBUG } 20:15:32 DEBUG ] 20:15:32 DEBUG ----- output ----- 20:15:32 DEBUG } 20:15:32 DEBUG ] 20:15:32 DEBUG } 20:15:32 DEBUG { 20:15:32 DEBUG } 20:15:32 DEBUG } 20:15:32 DEBUG } 20:15:32 DEBUG "status": "success", 20:15:32 DEBUG } 20:15:32 DEBUG "data": { 20:15:32 DEBUG "resultType": "vector", 20:15:32 DEBUG "result": [ 20:15:32 DEBUG { 20:15:32 DEBUG "metric": { 20:15:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:15:32 DEBUG }, 20:15:32 DEBUG "value": [ 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG [...] 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:15:32 DEBUG }, 20:15:32 DEBUG "value": [ 20:15:32 DEBUG 1713554132, 20:15:32 DEBUG "0" 20:15:32 DEBUG ] 20:15:32 DEBUG } 20:15:32 DEBUG ] 20:15:32 DEBUG } 20:15:32 DEBUG } 20:15:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713554132" 20:15:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554132" 20:15:32 INFO [http_cmd]: http status code OK 20:15:32 DEBUG --- status code --- 20:15:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:32 DEBUG --- http response --- 20:15:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:15:32 DEBUG ----- output ----- 20:15:32 DEBUG { 20:15:32 DEBUG "status": "success", 20:15:32 DEBUG "data": { 20:15:32 DEBUG "resultType": "vector", 20:15:32 DEBUG "result": [ 20:15:32 DEBUG { 20:15:32 DEBUG "metric": { 20:15:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:15:32 DEBUG }, 20:15:32 DEBUG "value": [ 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG [...] 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:15:32 DEBUG }, 20:15:32 DEBUG "value": [ 20:15:32 DEBUG 1713554132, 20:15:32 DEBUG "0.015" 20:15:32 DEBUG ] 20:15:32 DEBUG } 20:15:32 DEBUG ] 20:15:32 DEBUG } 20:15:32 DEBUG } 20:15:32 INFO [http_cmd]: http status code OK 20:15:32 DEBUG --- status code --- 20:15:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:32 DEBUG --- http response --- 20:15:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:15:32 DEBUG ----- output ----- 20:15:32 DEBUG { 20:15:32 DEBUG "status": "success", 20:15:32 DEBUG "data": { 20:15:32 DEBUG "resultType": "vector", 20:15:32 DEBUG "result": [ 20:15:32 DEBUG { 20:15:32 DEBUG "metric": { 20:15:32 DEBUG "pod": "ds-idrepo-1" 20:15:32 DEBUG }, 20:15:32 DEBUG "value": [ 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG [...] 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:15:32 DEBUG }, 20:15:32 DEBUG "value": [ 20:15:32 DEBUG 1713554132, 20:15:32 DEBUG "0" 20:15:32 DEBUG ] 20:15:32 DEBUG } 20:15:32 DEBUG ] 20:15:32 DEBUG } 20:15:32 DEBUG } 20:15:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554132" 20:15:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554132" 20:15:32 INFO [http_cmd]: http status code OK 20:15:32 DEBUG --- status code --- 20:15:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:32 DEBUG --- http response --- 20:15:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:15:32 DEBUG ----- output ----- 20:15:32 DEBUG { 20:15:32 DEBUG "status": "success", 20:15:32 DEBUG "data": { 20:15:32 DEBUG "resultType": "vector", 20:15:32 DEBUG "result": [ 20:15:32 DEBUG { 20:15:32 DEBUG "metric": { 20:15:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:15:32 DEBUG }, 20:15:32 DEBUG "value": [ 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG [...] 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:15:32 DEBUG }, 20:15:32 DEBUG "value": [ 20:15:32 DEBUG 1713554132, 20:15:32 DEBUG "2509.9" 20:15:32 DEBUG ] 20:15:32 DEBUG } 20:15:32 DEBUG ] 20:15:32 DEBUG } 20:15:32 DEBUG } 20:15:32 INFO [http_cmd]: http status code OK 20:15:32 DEBUG --- status code --- 20:15:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:32 DEBUG --- http response --- 20:15:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:15:32 DEBUG ----- output ----- 20:15:32 DEBUG { 20:15:32 DEBUG "status": "success", 20:15:32 DEBUG "data": { 20:15:32 DEBUG "resultType": "vector", 20:15:32 DEBUG "result": [ 20:15:32 DEBUG { 20:15:32 DEBUG "metric": { 20:15:32 DEBUG "pod": "ds-idrepo-1" 20:15:32 DEBUG }, 20:15:32 DEBUG "value": [ 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG [...] 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:15:32 DEBUG }, 20:15:32 DEBUG "value": [ 20:15:32 DEBUG 1713554132, 20:15:32 DEBUG "0" 20:15:32 DEBUG ] 20:15:32 DEBUG } 20:15:32 DEBUG ] 20:15:32 DEBUG } 20:15:32 DEBUG } 20:15:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554132" 20:15:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713554132" 20:15:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554132" 20:15:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554132" 20:15:32 INFO [http_cmd]: http status code OK 20:15:32 INFO [http_cmd]: http status code OK 20:15:32 DEBUG --- status code --- 20:15:32 DEBUG --- status code --- 20:15:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:32 DEBUG --- http response --- 20:15:32 DEBUG --- http response --- 20:15:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:15:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:15:32 DEBUG ----- output ----- 20:15:32 DEBUG ----- output ----- 20:15:32 DEBUG { 20:15:32 DEBUG { 20:15:32 DEBUG "status": "success", 20:15:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554132" 20:15:32 DEBUG "status": "success", 20:15:32 DEBUG "data": { 20:15:32 DEBUG "data": { 20:15:32 DEBUG "resultType": "vector", 20:15:32 DEBUG "resultType": "vector", 20:15:32 DEBUG "result": [ 20:15:32 DEBUG "result": [ 20:15:32 DEBUG { 20:15:32 DEBUG { 20:15:32 DEBUG "metric": { 20:15:32 DEBUG "metric": { 20:15:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:15:32 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:15:32 DEBUG }, 20:15:32 DEBUG }, 20:15:32 DEBUG "value": [ 20:15:32 DEBUG "value": [ 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG [...] 20:15:32 DEBUG [...] 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:15:32 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:15:32 DEBUG }, 20:15:32 DEBUG }, 20:15:32 DEBUG "value": [ 20:15:32 DEBUG "value": [ 20:15:32 DEBUG 1713554132, 20:15:32 DEBUG 1713554132, 20:15:32 DEBUG "2505.2666666666664" 20:15:32 DEBUG "8.039233385807584" 20:15:32 DEBUG ] 20:15:32 DEBUG ] 20:15:32 DEBUG } 20:15:32 DEBUG } 20:15:32 DEBUG ] 20:15:32 DEBUG ] 20:15:32 DEBUG } 20:15:32 DEBUG } 20:15:32 DEBUG } 20:15:32 DEBUG } 20:15:32 INFO [http_cmd]: http status code OK 20:15:32 INFO [http_cmd]: http status code OK 20:15:32 DEBUG --- status code --- 20:15:32 DEBUG --- status code --- 20:15:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:32 DEBUG --- http response --- 20:15:32 DEBUG --- http response --- 20:15:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:15:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:15:32 DEBUG ----- output ----- 20:15:32 DEBUG ----- output ----- 20:15:32 DEBUG { 20:15:32 DEBUG { 20:15:32 DEBUG "status": "success", 20:15:32 DEBUG "status": "success", 20:15:32 DEBUG "data": { 20:15:32 DEBUG "data": { 20:15:32 DEBUG "resultType": "vector", 20:15:32 DEBUG "resultType": "vector", 20:15:32 DEBUG "result": [ 20:15:32 DEBUG "result": [ 20:15:32 DEBUG { 20:15:32 DEBUG { 20:15:32 DEBUG "metric": { 20:15:32 DEBUG "metric": { 20:15:32 DEBUG "pod": "ds-idrepo-0" 20:15:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:15:32 DEBUG }, 20:15:32 DEBUG }, 20:15:32 DEBUG "value": [ 20:15:32 DEBUG "value": [ 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG [...] 20:15:32 DEBUG [...] 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG "pod": "ds-idrepo-2" 20:15:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:15:32 DEBUG }, 20:15:32 DEBUG }, 20:15:32 DEBUG "value": [ 20:15:32 DEBUG "value": [ 20:15:32 DEBUG 1713554132, 20:15:32 DEBUG 1713554132, 20:15:32 DEBUG "0" 20:15:32 DEBUG "2508.766666666667" 20:15:32 DEBUG ] 20:15:32 DEBUG ] 20:15:32 DEBUG } 20:15:32 DEBUG } 20:15:32 DEBUG ] 20:15:32 DEBUG ] 20:15:32 DEBUG } 20:15:32 DEBUG } 20:15:32 DEBUG } 20:15:32 DEBUG } 20:15:32 INFO [http_cmd]: http status code OK 20:15:32 DEBUG --- status code --- 20:15:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:32 DEBUG --- http response --- 20:15:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:15:32 DEBUG ----- output ----- 20:15:32 DEBUG { 20:15:32 DEBUG "status": "success", 20:15:32 DEBUG "data": { 20:15:32 DEBUG "resultType": "vector", 20:15:32 DEBUG "result": [ 20:15:32 DEBUG { 20:15:32 DEBUG "metric": { 20:15:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:15:32 DEBUG }, 20:15:32 DEBUG "value": [ 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG [...] 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:15:32 DEBUG }, 20:15:32 DEBUG "value": [ 20:15:32 DEBUG 1713554132, 20:15:32 DEBUG "2509.133333333333" 20:15:32 DEBUG ] 20:15:32 DEBUG } 20:15:32 DEBUG ] 20:15:32 DEBUG } 20:15:32 DEBUG } 20:15:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554132" 20:15:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554132" 20:15:32 INFO [http_cmd]: http status code OK 20:15:32 DEBUG --- status code --- 20:15:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:32 DEBUG --- http response --- 20:15:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:15:32 DEBUG ----- output ----- 20:15:32 DEBUG { 20:15:32 DEBUG "status": "success", 20:15:32 DEBUG "data": { 20:15:32 DEBUG "resultType": "vector", 20:15:32 DEBUG "result": [ 20:15:32 DEBUG { 20:15:32 DEBUG "metric": { 20:15:32 DEBUG "instance": "10.142.0.107:9100" 20:15:32 DEBUG }, 20:15:32 DEBUG "value": [ 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG [...] 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG "instance": "10.142.0.43:9100" 20:15:32 DEBUG }, 20:15:32 DEBUG "value": [ 20:15:32 DEBUG 1713554132, 20:15:32 DEBUG "75456539.96666667" 20:15:32 DEBUG ] 20:15:32 DEBUG } 20:15:32 DEBUG ] 20:15:32 DEBUG } 20:15:32 DEBUG } 20:15:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554132" 20:15:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713554132" 20:15:32 INFO [http_cmd]: http status code OK 20:15:32 DEBUG --- status code --- 20:15:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:32 DEBUG --- http response --- 20:15:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:15:32 DEBUG ----- output ----- 20:15:32 DEBUG { 20:15:32 DEBUG "status": "success", 20:15:32 DEBUG "data": { 20:15:32 DEBUG "resultType": "vector", 20:15:32 DEBUG "result": [ 20:15:32 DEBUG { 20:15:32 DEBUG "metric": { 20:15:32 DEBUG "pod": "ds-cts-0" 20:15:32 DEBUG }, 20:15:32 DEBUG "value": [ 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG [...] 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG "pod": "ds-cts-2" 20:15:32 DEBUG }, 20:15:32 DEBUG "value": [ 20:15:32 DEBUG 1713554132, 20:15:32 DEBUG "0" 20:15:32 DEBUG ] 20:15:32 DEBUG } 20:15:32 DEBUG ] 20:15:32 DEBUG } 20:15:32 DEBUG } 20:15:32 INFO [http_cmd]: http status code OK 20:15:32 DEBUG --- status code --- 20:15:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:32 DEBUG --- http response --- 20:15:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:15:32 DEBUG ----- output ----- 20:15:32 DEBUG { 20:15:32 DEBUG "status": "success", 20:15:32 DEBUG "data": { 20:15:32 DEBUG "resultType": "vector", 20:15:32 DEBUG "result": [ 20:15:32 DEBUG { 20:15:32 DEBUG "metric": { 20:15:32 DEBUG "pod": "ds-idrepo-1" 20:15:32 DEBUG }, 20:15:32 DEBUG "value": [ 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG [...] 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:15:32 DEBUG }, 20:15:32 DEBUG "value": [ 20:15:32 DEBUG 1713554132, 20:15:32 DEBUG "0" 20:15:32 DEBUG ] 20:15:32 DEBUG } 20:15:32 DEBUG ] 20:15:32 DEBUG } 20:15:32 DEBUG } 20:15:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554132" 20:15:32 INFO [http_cmd]: http status code OK 20:15:32 DEBUG --- status code --- 20:15:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:32 DEBUG --- http response --- 20:15:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:15:32 DEBUG ----- output ----- 20:15:32 DEBUG { 20:15:32 DEBUG "status": "success", 20:15:32 DEBUG "data": { 20:15:32 DEBUG "resultType": "vector", 20:15:32 DEBUG "result": [ 20:15:32 DEBUG { 20:15:32 DEBUG "metric": { 20:15:32 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:15:32 DEBUG }, 20:15:32 DEBUG "value": [ 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG [...] 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:15:32 DEBUG }, 20:15:32 DEBUG "value": [ 20:15:32 DEBUG 1713554132, 20:15:32 DEBUG "3687804928" 20:15:32 DEBUG ] 20:15:32 DEBUG } 20:15:32 DEBUG ] 20:15:32 DEBUG } 20:15:32 DEBUG } 20:15:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554132" 20:15:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554132" 20:15:32 INFO [http_cmd]: http status code OK 20:15:32 DEBUG --- status code --- 20:15:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:32 DEBUG --- http response --- 20:15:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:15:32 DEBUG ----- output ----- 20:15:32 DEBUG { 20:15:32 DEBUG "status": "success", 20:15:32 DEBUG "data": { 20:15:32 DEBUG "resultType": "vector", 20:15:32 DEBUG "result": [ 20:15:32 DEBUG { 20:15:32 DEBUG "metric": { 20:15:32 DEBUG "pod": "ds-cts-0" 20:15:32 DEBUG }, 20:15:32 DEBUG "value": [ 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG [...] 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG "pod": "ds-cts-2" 20:15:32 DEBUG }, 20:15:32 DEBUG "value": [ 20:15:32 DEBUG 1713554132, 20:15:32 DEBUG "29.966666666666665" 20:15:32 DEBUG ] 20:15:32 DEBUG } 20:15:32 DEBUG ] 20:15:32 DEBUG } 20:15:32 DEBUG } 20:15:32 INFO [http_cmd]: http status code OK 20:15:32 DEBUG --- status code --- 20:15:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:32 DEBUG --- http response --- 20:15:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:15:32 DEBUG ----- output ----- 20:15:32 DEBUG { 20:15:32 DEBUG "status": "success", 20:15:32 DEBUG "data": { 20:15:32 DEBUG "resultType": "vector", 20:15:32 DEBUG "result": [ 20:15:32 DEBUG { 20:15:32 DEBUG "metric": { 20:15:32 DEBUG "instance": "10.142.0.107:9100" 20:15:32 DEBUG }, 20:15:32 DEBUG "value": [ 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG [...] 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG "instance": "10.142.0.43:9100" 20:15:32 DEBUG }, 20:15:32 DEBUG "value": [ 20:15:32 DEBUG 1713554132, 20:15:32 DEBUG "2056573957120" 20:15:32 DEBUG ] 20:15:32 DEBUG } 20:15:32 DEBUG ] 20:15:32 DEBUG } 20:15:32 DEBUG } 20:15:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554132" 20:15:32 INFO [http_cmd]: http status code OK 20:15:32 DEBUG --- status code --- 20:15:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:32 DEBUG --- http response --- 20:15:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:15:32 DEBUG ----- output ----- 20:15:32 DEBUG { 20:15:32 DEBUG "status": "success", 20:15:32 DEBUG "data": { 20:15:32 DEBUG "resultType": "vector", 20:15:32 DEBUG "result": [ 20:15:32 DEBUG { 20:15:32 DEBUG "metric": { 20:15:32 DEBUG "instance": "10.142.0.107:9100" 20:15:32 DEBUG }, 20:15:32 DEBUG "value": [ 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG [...] 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG "instance": "10.142.0.43:9100" 20:15:32 DEBUG }, 20:15:32 DEBUG "value": [ 20:15:32 DEBUG 1713554132, 20:15:32 DEBUG "75454691.83333333" 20:15:32 DEBUG ] 20:15:32 DEBUG } 20:15:32 DEBUG ] 20:15:32 DEBUG } 20:15:32 DEBUG } 20:15:32 INFO [http_cmd]: http status code OK 20:15:32 DEBUG --- status code --- 20:15:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:32 DEBUG --- http response --- 20:15:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 20:15:32 DEBUG ----- output ----- 20:15:32 DEBUG { 20:15:32 DEBUG "status": "success", 20:15:32 DEBUG "data": { 20:15:32 DEBUG "resultType": "vector", 20:15:32 DEBUG "result": [ 20:15:32 DEBUG { 20:15:32 DEBUG "metric": { 20:15:32 DEBUG "pod": "ds-idrepo-1" 20:15:32 DEBUG }, 20:15:32 DEBUG "value": [ 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG [...] 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG 20:15:32 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:15:32 DEBUG }, 20:15:32 DEBUG "value": [ 20:15:32 DEBUG 1713554132, 20:15:32 DEBUG "0.000020593822825524368" 20:15:32 DEBUG ] 20:15:32 DEBUG } 20:15:32 DEBUG ] 20:15:32 DEBUG } 20:15:32 DEBUG } 20:15:34 INFO 20:15:34 INFO [loop_until]: kubectl --namespace=xlou top pods 20:15:34 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:15:34 INFO [loop_until]: OK (rc = 0) 20:15:34 DEBUG --- stdout --- 20:15:34 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 16334m 9007Mi am-767f6b67b5-qcqz2 16386m 8810Mi am-767f6b67b5-vbltz 18152m 8998Mi ds-cts-0 17471m 10981Mi ds-cts-1 16321m 11222Mi ds-cts-2 17508m 10991Mi ds-idrepo-0 7317m 3692Mi ds-idrepo-1 7037m 3536Mi ds-idrepo-2 7278m 3493Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 1m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8079m 3424Mi 20:15:34 DEBUG --- stderr --- 20:15:34 DEBUG 20:15:35 INFO 20:15:35 INFO [loop_until]: kubectl --namespace=xlou top node 20:15:35 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:15:35 INFO [loop_until]: OK (rc = 0) 20:15:35 DEBUG --- stdout --- 20:15:35 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 73m 0% 6255Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7470m 25% 7289Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16249m 54% 11715Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 87m 0% 3353Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 17226m 57% 13950Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 155m 0% 5205Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 16566m 55% 11690Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15897m 53% 2987Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 18224m 61% 11214Mi 10% gke-xlou-cdm-ds-0f601751-j488 7718m 25% 6741Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 16229m 54% 15988Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 7239m 24% 7427Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 16834m 56% 13431Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 7934m 49% 7136Mi 12% 20:15:35 DEBUG --- stderr --- 20:15:35 DEBUG 20:15:52 WARNING File /tmp/lodemon_data-20240419_191228/node_disk_read_bytes_total.json does not exist. Skipping average value computation... 127.0.0.1 - - [19/Apr/2024 20:15:52] "GET /monitoring/average?start_time=20240419_184643&stop_time=20240419_191351 HTTP/1.1" 200 - 20:16:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554192" 20:16:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554192" 20:16:32 INFO [http_cmd]: http status code OK 20:16:32 DEBUG --- status code --- 20:16:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:32 DEBUG --- http response --- 20:16:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:16:32 DEBUG ----- output ----- 20:16:32 DEBUG { 20:16:32 DEBUG "status": "success", 20:16:32 DEBUG "data": { 20:16:32 DEBUG "resultType": "vector", 20:16:32 DEBUG "result": [ 20:16:32 DEBUG { 20:16:32 DEBUG "metric": { 20:16:32 DEBUG "instance": "10.142.0.107:9100" 20:16:32 DEBUG }, 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG [...] 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG "instance": "10.142.0.43:9100" 20:16:32 DEBUG }, 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 1713554192, 20:16:32 DEBUG "0.0630000000000109" 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG } 20:16:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713554192" 20:16:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554192" 20:16:32 INFO [http_cmd]: http status code OK 20:16:32 DEBUG --- status code --- 20:16:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:32 DEBUG --- http response --- 20:16:32 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:16:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554192" 20:16:32 ERROR [http_cmd]: ERROR 20:16:32 ERROR --- status code --- 20:16:32 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:16:32 ERROR --- http response --- 20:16:32 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:16:32 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:16:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554192" 20:16:32 INFO [http_cmd]: http status code OK 20:16:32 DEBUG --- status code --- 20:16:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:32 DEBUG --- http response --- 20:16:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:16:32 DEBUG ----- output ----- 20:16:32 DEBUG { 20:16:32 DEBUG "status": "success", 20:16:32 DEBUG "data": { 20:16:32 DEBUG "resultType": "vector", 20:16:32 DEBUG "result": [ 20:16:32 DEBUG { 20:16:32 DEBUG "metric": { 20:16:32 DEBUG "pod": "ds-idrepo-0" 20:16:32 DEBUG }, 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG [...] 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG "pod": "ds-idrepo-2" 20:16:32 DEBUG }, 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 1713554192, 20:16:32 DEBUG "0" 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG } 20:16:32 INFO [http_cmd]: http status code OK 20:16:32 INFO [http_cmd]: http status code OK 20:16:32 DEBUG --- status code --- 20:16:32 DEBUG --- status code --- 20:16:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:32 DEBUG --- http response --- 20:16:32 DEBUG --- http response --- 20:16:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:16:32 DEBUG ----- output ----- 20:16:32 DEBUG { 20:16:32 DEBUG "status": "success", 20:16:32 DEBUG "data": { 20:16:32 DEBUG "resultType": "vector", 20:16:32 DEBUG "result": [ 20:16:32 DEBUG { 20:16:32 DEBUG "metric": { 20:16:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:16:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:16:32 DEBUG ----- output ----- 20:16:32 DEBUG }, 20:16:32 DEBUG { 20:16:32 DEBUG "value": [ 20:16:32 DEBUG "status": "success", 20:16:32 DEBUG 20:16:32 DEBUG "data": { 20:16:32 DEBUG 20:16:32 DEBUG "resultType": "vector", 20:16:32 DEBUG 20:16:32 DEBUG "result": [ 20:16:32 DEBUG [...] 20:16:32 DEBUG { 20:16:32 DEBUG 20:16:32 DEBUG "metric": { 20:16:32 DEBUG 20:16:32 DEBUG "pod": "ds-idrepo-0" 20:16:32 DEBUG 20:16:32 DEBUG }, 20:16:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:16:32 DEBUG "value": [ 20:16:32 DEBUG }, 20:16:32 DEBUG 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 20:16:32 DEBUG 1713554192, 20:16:32 DEBUG 20:16:32 DEBUG "367113.5" 20:16:32 DEBUG [...] 20:16:32 DEBUG ] 20:16:32 DEBUG 20:16:32 DEBUG } 20:16:32 DEBUG 20:16:32 DEBUG ] 20:16:32 DEBUG 20:16:32 DEBUG } 20:16:32 DEBUG "pod": "ds-idrepo-2" 20:16:32 DEBUG } 20:16:32 DEBUG }, 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 1713554192, 20:16:32 DEBUG "0" 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG } 20:16:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554192" 20:16:32 INFO [http_cmd]: http status code OK 20:16:32 DEBUG --- status code --- 20:16:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:32 DEBUG --- http response --- 20:16:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:16:32 DEBUG ----- output ----- 20:16:32 DEBUG { 20:16:32 DEBUG "status": "success", 20:16:32 DEBUG "data": { 20:16:32 DEBUG "resultType": "vector", 20:16:32 DEBUG "result": [ 20:16:32 DEBUG { 20:16:32 DEBUG "metric": { 20:16:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:16:32 DEBUG }, 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG [...] 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:16:32 DEBUG }, 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 1713554192, 20:16:32 DEBUG "0" 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG } 20:16:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554192" 20:16:32 INFO [http_cmd]: http status code OK 20:16:32 DEBUG --- status code --- 20:16:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:32 DEBUG --- http response --- 20:16:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:16:32 DEBUG ----- output ----- 20:16:32 DEBUG { 20:16:32 DEBUG "status": "success", 20:16:32 DEBUG "data": { 20:16:32 DEBUG "resultType": "vector", 20:16:32 DEBUG "result": [ 20:16:32 DEBUG { 20:16:32 DEBUG "metric": { 20:16:32 DEBUG "pod": "ds-cts-0" 20:16:32 DEBUG }, 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG [...] 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG "pod": "ds-cts-2" 20:16:32 DEBUG }, 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 1713554192, 20:16:32 DEBUG "0" 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG } 20:16:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713554192" 20:16:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554192" 20:16:32 INFO [http_cmd]: http status code OK 20:16:32 DEBUG --- status code --- 20:16:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:32 DEBUG --- http response --- 20:16:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:16:32 DEBUG ----- output ----- 20:16:32 DEBUG { 20:16:32 DEBUG "status": "success", 20:16:32 DEBUG "data": { 20:16:32 DEBUG "resultType": "vector", 20:16:32 DEBUG "result": [ 20:16:32 DEBUG { 20:16:32 DEBUG "metric": { 20:16:32 DEBUG "pod": "ds-idrepo-0" 20:16:32 DEBUG }, 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG [...] 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG "pod": "ds-cts-2" 20:16:32 DEBUG }, 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 1713554192, 20:16:32 DEBUG "0" 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG } 20:16:32 INFO [http_cmd]: http status code OK 20:16:32 DEBUG --- status code --- 20:16:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:32 DEBUG --- http response --- 20:16:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:16:32 DEBUG ----- output ----- 20:16:32 DEBUG { 20:16:32 DEBUG "status": "success", 20:16:32 DEBUG "data": { 20:16:32 DEBUG "resultType": "vector", 20:16:32 DEBUG "result": [ 20:16:32 DEBUG { 20:16:32 DEBUG "metric": { 20:16:32 DEBUG "instance": "10.142.0.107:9100" 20:16:32 DEBUG }, 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG [...] 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG "instance": "10.142.0.43:9100" 20:16:32 DEBUG }, 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 1713554192, 20:16:32 DEBUG "0.11876666666666248" 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG } 20:16:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554192" 20:16:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713554192" 20:16:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554192" 20:16:32 INFO [http_cmd]: http status code OK 20:16:32 DEBUG --- status code --- 20:16:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:32 DEBUG --- http response --- 20:16:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 20:16:32 DEBUG ----- output ----- 20:16:32 DEBUG { 20:16:32 DEBUG "status": "success", 20:16:32 DEBUG "data": { 20:16:32 DEBUG "resultType": "vector", 20:16:32 DEBUG "result": [ 20:16:32 DEBUG { 20:16:32 DEBUG "metric": { 20:16:32 DEBUG "pod": "ds-idrepo-0" 20:16:32 DEBUG }, 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG [...] 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:16:32 DEBUG }, 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 1713554192, 20:16:32 DEBUG "0" 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG } 20:16:32 INFO [http_cmd]: http status code OK 20:16:32 DEBUG --- status code --- 20:16:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:32 DEBUG --- http response --- 20:16:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:16:32 DEBUG ----- output ----- 20:16:32 DEBUG { 20:16:32 DEBUG "status": "success", 20:16:32 DEBUG "data": { 20:16:32 DEBUG "resultType": "vector", 20:16:32 DEBUG "result": [ 20:16:32 DEBUG { 20:16:32 DEBUG "metric": { 20:16:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:16:32 DEBUG }, 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG [...] 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:16:32 DEBUG }, 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 1713554192, 20:16:32 DEBUG "0.015" 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG } 20:16:32 INFO [http_cmd]: http status code OK 20:16:32 DEBUG --- status code --- 20:16:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:32 DEBUG --- http response --- 20:16:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:16:32 DEBUG ----- output ----- 20:16:32 DEBUG { 20:16:32 DEBUG "status": "success", 20:16:32 DEBUG "data": { 20:16:32 DEBUG "resultType": "vector", 20:16:32 DEBUG "result": [ 20:16:32 DEBUG { 20:16:32 DEBUG "metric": { 20:16:32 DEBUG "pod": "ds-idrepo-0" 20:16:32 DEBUG }, 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG [...] 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:16:32 DEBUG }, 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 1713554192, 20:16:32 DEBUG "103.43520796019901" 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG } 20:16:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554192" 20:16:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713554192" 20:16:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554192" 20:16:32 INFO [http_cmd]: http status code OK 20:16:32 DEBUG --- status code --- 20:16:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:32 DEBUG --- http response --- 20:16:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:16:32 DEBUG ----- output ----- 20:16:32 DEBUG { 20:16:32 DEBUG "status": "success", 20:16:32 DEBUG "data": { 20:16:32 DEBUG "resultType": "vector", 20:16:32 DEBUG "result": [ 20:16:32 DEBUG { 20:16:32 DEBUG "metric": { 20:16:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:16:32 DEBUG }, 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG [...] 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:16:32 DEBUG }, 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 1713554192, 20:16:32 DEBUG "2502.4666666666667" 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG } 20:16:32 INFO [http_cmd]: http status code OK 20:16:32 DEBUG --- status code --- 20:16:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:32 DEBUG --- http response --- 20:16:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:16:32 DEBUG ----- output ----- 20:16:32 DEBUG { 20:16:32 DEBUG "status": "success", 20:16:32 DEBUG "data": { 20:16:32 DEBUG "resultType": "vector", 20:16:32 DEBUG "result": [ 20:16:32 DEBUG { 20:16:32 DEBUG "metric": { 20:16:32 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:16:32 DEBUG }, 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG [...] 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:16:32 DEBUG }, 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 1713554192, 20:16:32 DEBUG "2.157334010887854" 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG } 20:16:32 INFO [http_cmd]: http status code OK 20:16:32 DEBUG --- status code --- 20:16:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:32 DEBUG --- http response --- 20:16:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:16:32 DEBUG ----- output ----- 20:16:32 DEBUG { 20:16:32 DEBUG "status": "success", 20:16:32 DEBUG "data": { 20:16:32 DEBUG "resultType": "vector", 20:16:32 DEBUG "result": [ 20:16:32 DEBUG { 20:16:32 DEBUG "metric": { 20:16:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:16:32 DEBUG }, 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG [...] 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:16:32 DEBUG }, 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 1713554192, 20:16:32 DEBUG "0" 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG } 20:16:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554192" 20:16:32 INFO [http_cmd]: http status code OK 20:16:32 DEBUG --- status code --- 20:16:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:32 DEBUG --- http response --- 20:16:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:16:32 DEBUG ----- output ----- 20:16:32 DEBUG { 20:16:32 DEBUG "status": "success", 20:16:32 DEBUG "data": { 20:16:32 DEBUG "resultType": "vector", 20:16:32 DEBUG "result": [ 20:16:32 DEBUG { 20:16:32 DEBUG "metric": { 20:16:32 DEBUG "pod": "ds-idrepo-0" 20:16:32 DEBUG }, 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG [...] 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG "pod": "ds-idrepo-2" 20:16:32 DEBUG }, 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 1713554192, 20:16:32 DEBUG "0" 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG } 20:16:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554192" 20:16:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554192" 20:16:32 INFO [http_cmd]: http status code OK 20:16:32 DEBUG --- status code --- 20:16:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:32 DEBUG --- http response --- 20:16:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:16:32 DEBUG ----- output ----- 20:16:32 DEBUG { 20:16:32 DEBUG "status": "success", 20:16:32 DEBUG "data": { 20:16:32 DEBUG "resultType": "vector", 20:16:32 DEBUG "result": [ 20:16:32 DEBUG { 20:16:32 DEBUG "metric": { 20:16:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:16:32 DEBUG }, 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG [...] 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:16:32 DEBUG }, 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 1713554192, 20:16:32 DEBUG "0" 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG } 20:16:32 INFO [http_cmd]: http status code OK 20:16:32 DEBUG --- status code --- 20:16:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:32 DEBUG --- http response --- 20:16:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:16:32 DEBUG ----- output ----- 20:16:32 DEBUG { 20:16:32 DEBUG "status": "success", 20:16:32 DEBUG "data": { 20:16:32 DEBUG "resultType": "vector", 20:16:32 DEBUG "result": [ 20:16:32 DEBUG { 20:16:32 DEBUG "metric": { 20:16:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:16:32 DEBUG }, 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG [...] 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:16:32 DEBUG }, 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 1713554192, 20:16:32 DEBUG "0" 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG } 20:16:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554192" 20:16:32 INFO [http_cmd]: http status code OK 20:16:32 DEBUG --- status code --- 20:16:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:32 DEBUG --- http response --- 20:16:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:16:32 DEBUG ----- output ----- 20:16:32 DEBUG { 20:16:32 DEBUG "status": "success", 20:16:32 DEBUG "data": { 20:16:32 DEBUG "resultType": "vector", 20:16:32 DEBUG "result": [ 20:16:32 DEBUG { 20:16:32 DEBUG "metric": { 20:16:32 DEBUG "instance": "10.142.0.107:9100" 20:16:32 DEBUG }, 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG [...] 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG "instance": "10.142.0.43:9100" 20:16:32 DEBUG }, 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 1713554192, 20:16:32 DEBUG "2059504069632" 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG } 20:16:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554192" 20:16:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554192" 20:16:32 INFO [http_cmd]: http status code OK 20:16:32 DEBUG --- status code --- 20:16:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:32 DEBUG --- http response --- 20:16:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:16:32 DEBUG ----- output ----- 20:16:32 DEBUG { 20:16:32 DEBUG "status": "success", 20:16:32 DEBUG "data": { 20:16:32 DEBUG "resultType": "vector", 20:16:32 DEBUG "result": [ 20:16:32 DEBUG { 20:16:32 DEBUG "metric": { 20:16:32 DEBUG "instance": "10.142.0.107:9100" 20:16:32 DEBUG }, 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG [...] 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG "instance": "10.142.0.43:9100" 20:16:32 DEBUG }, 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 1713554192, 20:16:32 DEBUG "16399199.100000001" 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG } 20:16:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554192" 20:16:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713554192" 20:16:32 INFO [http_cmd]: http status code OK 20:16:32 DEBUG --- status code --- 20:16:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:32 DEBUG --- http response --- 20:16:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:16:32 DEBUG ----- output ----- 20:16:32 DEBUG { 20:16:32 DEBUG "status": "success", 20:16:32 DEBUG "data": { 20:16:32 DEBUG "resultType": "vector", 20:16:32 DEBUG "result": [ 20:16:32 DEBUG { 20:16:32 DEBUG "metric": { 20:16:32 DEBUG "pod": "ds-cts-0" 20:16:32 DEBUG }, 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG [...] 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG "pod": "ds-cts-2" 20:16:32 DEBUG }, 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 1713554192, 20:16:32 DEBUG "0" 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG } 20:16:32 INFO [http_cmd]: http status code OK 20:16:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554192" 20:16:32 DEBUG --- status code --- 20:16:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:32 DEBUG --- http response --- 20:16:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:16:32 DEBUG ----- output ----- 20:16:32 DEBUG { 20:16:32 DEBUG "status": "success", 20:16:32 DEBUG "data": { 20:16:32 DEBUG "resultType": "vector", 20:16:32 DEBUG "result": [ 20:16:32 DEBUG { 20:16:32 DEBUG "metric": { 20:16:32 DEBUG "pod": "ds-idrepo-0" 20:16:32 DEBUG }, 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG [...] 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:16:32 DEBUG }, 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 1713554192, 20:16:32 DEBUG "0" 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG } 20:16:32 INFO [http_cmd]: http status code OK 20:16:32 DEBUG --- status code --- 20:16:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:32 DEBUG --- http response --- 20:16:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:16:32 DEBUG ----- output ----- 20:16:32 DEBUG { 20:16:32 DEBUG "status": "success", 20:16:32 DEBUG "data": { 20:16:32 DEBUG "resultType": "vector", 20:16:32 DEBUG "result": [ 20:16:32 DEBUG { 20:16:32 DEBUG "metric": { 20:16:32 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:16:32 DEBUG }, 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG [...] 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:16:32 DEBUG }, 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 1713554192, 20:16:32 DEBUG "2961866752" 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG } 20:16:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554192" 20:16:32 INFO [http_cmd]: http status code OK 20:16:32 DEBUG --- status code --- 20:16:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:32 DEBUG --- http response --- 20:16:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:16:32 DEBUG ----- output ----- 20:16:32 DEBUG { 20:16:32 DEBUG "status": "success", 20:16:32 DEBUG "data": { 20:16:32 DEBUG "resultType": "vector", 20:16:32 DEBUG "result": [ 20:16:32 DEBUG { 20:16:32 DEBUG "metric": { 20:16:32 DEBUG "pod": "ds-cts-0" 20:16:32 DEBUG }, 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG [...] 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG "pod": "ds-cts-2" 20:16:32 DEBUG }, 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 1713554192, 20:16:32 DEBUG "29.966666666666665" 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG } 20:16:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554192" 20:16:32 INFO [http_cmd]: http status code OK 20:16:32 DEBUG --- status code --- 20:16:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:32 DEBUG --- http response --- 20:16:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:16:32 DEBUG ----- output ----- 20:16:32 DEBUG { 20:16:32 DEBUG "status": "success", 20:16:32 DEBUG "data": { 20:16:32 DEBUG "resultType": "vector", 20:16:32 DEBUG "result": [ 20:16:32 DEBUG { 20:16:32 DEBUG "metric": { 20:16:32 DEBUG "instance": "10.142.0.107:9100" 20:16:32 DEBUG }, 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG [...] 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG "instance": "10.142.0.43:9100" 20:16:32 DEBUG }, 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 1713554192, 20:16:32 DEBUG "16625386.666666668" 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG } 20:16:32 INFO [http_cmd]: http status code OK 20:16:32 DEBUG --- status code --- 20:16:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:32 DEBUG --- http response --- 20:16:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:16:32 DEBUG ----- output ----- 20:16:32 DEBUG { 20:16:32 DEBUG "status": "success", 20:16:32 DEBUG "data": { 20:16:32 DEBUG "resultType": "vector", 20:16:32 DEBUG "result": [ 20:16:32 DEBUG { 20:16:32 DEBUG "metric": { 20:16:32 DEBUG "pod": "ds-idrepo-0" 20:16:32 DEBUG }, 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG [...] 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG 20:16:32 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:16:32 DEBUG }, 20:16:32 DEBUG "value": [ 20:16:32 DEBUG 1713554192, 20:16:32 DEBUG "0.000017651662686567294" 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG ] 20:16:32 DEBUG } 20:16:32 DEBUG } 20:16:34 INFO 20:16:34 INFO [loop_until]: kubectl --namespace=xlou top pods 20:16:34 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:16:34 INFO [loop_until]: OK (rc = 0) 20:16:34 DEBUG --- stdout --- 20:16:34 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 2420m 9007Mi am-767f6b67b5-qcqz2 2504m 8810Mi am-767f6b67b5-vbltz 2769m 8998Mi ds-cts-0 7194m 10965Mi ds-cts-1 6622m 11212Mi ds-cts-2 7646m 10959Mi ds-idrepo-0 739m 3682Mi ds-idrepo-1 710m 3537Mi ds-idrepo-2 712m 3493Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 1m 2733Mi 20:16:34 DEBUG --- stderr --- 20:16:34 DEBUG 20:16:35 INFO 20:16:35 INFO [loop_until]: kubectl --namespace=xlou top node 20:16:35 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:16:35 INFO [loop_until]: OK (rc = 0) 20:16:35 DEBUG --- stdout --- 20:16:35 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 79m 0% 6252Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 764m 2% 7279Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 2396m 8% 11753Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 98m 0% 3395Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 7265m 24% 13959Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 170m 0% 5216Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 2519m 8% 11726Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 77m 0% 2974Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 2935m 9% 11250Mi 10% gke-xlou-cdm-ds-0f601751-j488 793m 2% 6732Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 6669m 22% 15979Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 814m 2% 7433Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 7709m 25% 13430Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 70m 0% 6456Mi 11% 20:16:35 DEBUG --- stderr --- 20:16:35 DEBUG 20:17:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554252" 20:17:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554252" 20:17:32 INFO [http_cmd]: http status code OK 20:17:32 DEBUG --- status code --- 20:17:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:32 DEBUG --- http response --- 20:17:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:17:32 DEBUG ----- output ----- 20:17:32 DEBUG { 20:17:32 DEBUG "status": "success", 20:17:32 DEBUG "data": { 20:17:32 DEBUG "resultType": "vector", 20:17:32 DEBUG "result": [ 20:17:32 DEBUG { 20:17:32 DEBUG "metric": { 20:17:32 DEBUG "instance": "10.142.0.107:9100" 20:17:32 DEBUG }, 20:17:32 DEBUG "value": [ 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG [...] 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG "instance": "10.142.0.43:9100" 20:17:32 DEBUG }, 20:17:32 DEBUG "value": [ 20:17:32 DEBUG 1713554252, 20:17:32 DEBUG "0.14066666666666003" 20:17:32 DEBUG ] 20:17:32 DEBUG } 20:17:32 DEBUG ] 20:17:32 DEBUG } 20:17:32 DEBUG } 20:17:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554252" 20:17:32 INFO [http_cmd]: http status code OK 20:17:32 DEBUG --- status code --- 20:17:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:32 DEBUG --- http response --- 20:17:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:17:32 DEBUG ----- output ----- 20:17:32 DEBUG { 20:17:32 DEBUG "status": "success", 20:17:32 DEBUG "data": { 20:17:32 DEBUG "resultType": "vector", 20:17:32 DEBUG "result": [ 20:17:32 DEBUG { 20:17:32 DEBUG "metric": { 20:17:32 DEBUG "pod": "ds-idrepo-0" 20:17:32 DEBUG }, 20:17:32 DEBUG "value": [ 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG [...] 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG "pod": "ds-idrepo-2" 20:17:32 DEBUG }, 20:17:32 DEBUG "value": [ 20:17:32 DEBUG 1713554252, 20:17:32 DEBUG "6.766666666666667" 20:17:32 DEBUG ] 20:17:32 DEBUG } 20:17:32 DEBUG ] 20:17:32 DEBUG } 20:17:32 DEBUG } 20:17:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713554252" 20:17:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554252" 20:17:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554252" 20:17:32 ERROR [http_cmd]: ERROR 20:17:32 INFO [http_cmd]: http status code OK 20:17:32 ERROR --- status code --- 20:17:32 DEBUG --- status code --- 20:17:32 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:17:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:32 ERROR --- http response --- 20:17:32 DEBUG --- http response --- 20:17:32 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:17:32 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:17:32 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:17:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554252" 20:17:32 INFO [http_cmd]: http status code OK 20:17:32 DEBUG --- status code --- 20:17:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:32 DEBUG --- http response --- 20:17:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:17:32 DEBUG ----- output ----- 20:17:32 DEBUG { 20:17:32 DEBUG "status": "success", 20:17:32 DEBUG "data": { 20:17:32 DEBUG "resultType": "vector", 20:17:32 DEBUG "result": [ 20:17:32 DEBUG { 20:17:32 DEBUG "metric": { 20:17:32 DEBUG "pod": "ds-idrepo-0" 20:17:32 DEBUG }, 20:17:32 DEBUG "value": [ 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG [...] 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG "pod": "ds-idrepo-2" 20:17:32 DEBUG }, 20:17:32 DEBUG "value": [ 20:17:32 DEBUG 1713554252, 20:17:32 DEBUG "62.96666666666667" 20:17:32 DEBUG ] 20:17:32 DEBUG } 20:17:32 DEBUG ] 20:17:32 DEBUG } 20:17:32 DEBUG } 20:17:32 INFO [http_cmd]: http status code OK 20:17:32 INFO [http_cmd]: http status code OK 20:17:32 DEBUG --- status code --- 20:17:32 DEBUG --- status code --- 20:17:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:32 DEBUG --- http response --- 20:17:32 DEBUG --- http response --- 20:17:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:17:32 DEBUG ----- output ----- 20:17:32 DEBUG { 20:17:32 DEBUG "status": "success", 20:17:32 DEBUG "data": { 20:17:32 DEBUG "resultType": "vector", 20:17:32 DEBUG "result": [ 20:17:32 DEBUG { 20:17:32 DEBUG "metric": { 20:17:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:17:32 DEBUG }, 20:17:32 DEBUG "value": [ 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG [...] 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:17:32 DEBUG }, 20:17:32 DEBUG "value": [ 20:17:32 DEBUG 1713554252, 20:17:32 DEBUG "217628" 20:17:32 DEBUG ] 20:17:32 DEBUG } 20:17:32 DEBUG ] 20:17:32 DEBUG } 20:17:32 DEBUG } 20:17:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:17:32 DEBUG ----- output ----- 20:17:32 DEBUG { 20:17:32 DEBUG "status": "success", 20:17:32 DEBUG "data": { 20:17:32 DEBUG "resultType": "vector", 20:17:32 DEBUG "result": [ 20:17:32 DEBUG { 20:17:32 DEBUG "metric": { 20:17:32 DEBUG "pod": "ds-cts-0" 20:17:32 DEBUG }, 20:17:32 DEBUG "value": [ 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG [...] 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG "pod": "ds-cts-2" 20:17:32 DEBUG }, 20:17:32 DEBUG "value": [ 20:17:32 DEBUG 1713554252, 20:17:32 DEBUG "0" 20:17:32 DEBUG ] 20:17:32 DEBUG } 20:17:32 DEBUG ] 20:17:32 DEBUG } 20:17:32 DEBUG } 20:17:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554252" 20:17:32 INFO [http_cmd]: http status code OK 20:17:32 DEBUG --- status code --- 20:17:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:32 DEBUG --- http response --- 20:17:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:17:32 DEBUG ----- output ----- 20:17:32 DEBUG { 20:17:32 DEBUG "status": "success", 20:17:32 DEBUG "data": { 20:17:32 DEBUG "resultType": "vector", 20:17:32 DEBUG "result": [ 20:17:32 DEBUG { 20:17:32 DEBUG "metric": { 20:17:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:17:32 DEBUG }, 20:17:32 DEBUG "value": [ 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG [...] 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:17:32 DEBUG }, 20:17:32 DEBUG "value": [ 20:17:32 DEBUG 1713554252, 20:17:32 DEBUG "0" 20:17:32 DEBUG ] 20:17:32 DEBUG } 20:17:32 DEBUG ] 20:17:32 DEBUG } 20:17:32 DEBUG } 20:17:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713554252" 20:17:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554252" 20:17:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554252" 20:17:32 INFO [http_cmd]: http status code OK 20:17:32 DEBUG --- status code --- 20:17:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:32 DEBUG --- http response --- 20:17:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:17:32 DEBUG ----- output ----- 20:17:32 DEBUG { 20:17:32 DEBUG "status": "success", 20:17:32 DEBUG "data": { 20:17:32 DEBUG "resultType": "vector", 20:17:32 DEBUG "result": [ 20:17:32 DEBUG { 20:17:32 DEBUG "metric": { 20:17:32 DEBUG "pod": "ds-idrepo-0" 20:17:32 DEBUG }, 20:17:32 DEBUG "value": [ 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG [...] 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG "pod": "ds-cts-2" 20:17:32 DEBUG }, 20:17:32 DEBUG "value": [ 20:17:32 DEBUG 1713554252, 20:17:32 DEBUG "0" 20:17:32 DEBUG ] 20:17:32 DEBUG } 20:17:32 DEBUG ] 20:17:32 DEBUG } 20:17:32 DEBUG } 20:17:32 INFO [http_cmd]: http status code OK 20:17:32 INFO [http_cmd]: http status code OK 20:17:32 DEBUG --- status code --- 20:17:32 DEBUG --- status code --- 20:17:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:32 DEBUG --- http response --- 20:17:32 DEBUG --- http response --- 20:17:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:17:32 DEBUG ----- output ----- 20:17:32 DEBUG { 20:17:32 DEBUG "status": "success", 20:17:32 DEBUG "data": { 20:17:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 20:17:32 DEBUG "resultType": "vector", 20:17:32 DEBUG ----- output ----- 20:17:32 DEBUG "result": [ 20:17:32 DEBUG { 20:17:32 DEBUG { 20:17:32 DEBUG "status": "success", 20:17:32 DEBUG "metric": { 20:17:32 DEBUG "data": { 20:17:32 DEBUG "instance": "10.142.0.107:9100" 20:17:32 DEBUG "resultType": "vector", 20:17:32 DEBUG }, 20:17:32 DEBUG "result": [ 20:17:32 DEBUG "value": [ 20:17:32 DEBUG { 20:17:32 DEBUG 20:17:32 DEBUG "metric": { 20:17:32 DEBUG 20:17:32 DEBUG "pod": "ds-idrepo-1" 20:17:32 DEBUG 20:17:32 DEBUG }, 20:17:32 DEBUG [...] 20:17:32 DEBUG "value": [ 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG "instance": "10.142.0.43:9100" 20:17:32 DEBUG [...] 20:17:32 DEBUG }, 20:17:32 DEBUG 20:17:32 DEBUG "value": [ 20:17:32 DEBUG 20:17:32 DEBUG 1713554252, 20:17:32 DEBUG 20:17:32 DEBUG "0.23519999999997576" 20:17:32 DEBUG "pod": "am-767f6b67b5-qcqz2" 20:17:32 DEBUG ] 20:17:32 DEBUG }, 20:17:32 DEBUG } 20:17:32 DEBUG "value": [ 20:17:32 DEBUG ] 20:17:32 DEBUG 1713554252, 20:17:32 DEBUG } 20:17:32 DEBUG "311947825.8038258" 20:17:32 DEBUG } 20:17:32 DEBUG ] 20:17:32 DEBUG } 20:17:32 DEBUG ] 20:17:32 DEBUG } 20:17:32 DEBUG } 20:17:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554252" 20:17:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713554252" 20:17:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713554252" 20:17:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554252" 20:17:32 INFO [http_cmd]: http status code OK 20:17:32 DEBUG --- status code --- 20:17:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:32 DEBUG --- http response --- 20:17:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 20:17:32 DEBUG ----- output ----- 20:17:32 DEBUG { 20:17:32 DEBUG "status": "success", 20:17:32 DEBUG "data": { 20:17:32 DEBUG "resultType": "vector", 20:17:32 DEBUG "result": [ 20:17:32 DEBUG { 20:17:32 DEBUG "metric": { 20:17:32 DEBUG "pod": "ds-idrepo-1" 20:17:32 DEBUG }, 20:17:32 DEBUG "value": [ 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG [...] 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG "pod": "am-767f6b67b5-qcqz2" 20:17:32 DEBUG }, 20:17:32 DEBUG "value": [ 20:17:32 DEBUG 1713554252, 20:17:32 DEBUG "555.691222357889" 20:17:32 DEBUG ] 20:17:32 DEBUG } 20:17:32 DEBUG ] 20:17:32 DEBUG } 20:17:32 DEBUG } 20:17:32 INFO [http_cmd]: http status code OK 20:17:32 DEBUG --- status code --- 20:17:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:32 DEBUG --- http response --- 20:17:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:17:32 DEBUG ----- output ----- 20:17:32 DEBUG { 20:17:32 DEBUG "status": "success", 20:17:32 DEBUG "data": { 20:17:32 DEBUG "resultType": "vector", 20:17:32 DEBUG "result": [ 20:17:32 DEBUG { 20:17:32 DEBUG "metric": { 20:17:32 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:17:32 DEBUG }, 20:17:32 DEBUG "value": [ 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG [...] 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:17:32 DEBUG }, 20:17:32 DEBUG "value": [ 20:17:32 DEBUG 1713554252, 20:17:32 DEBUG "1.1257603130314138" 20:17:32 DEBUG ] 20:17:32 DEBUG } 20:17:32 DEBUG ] 20:17:32 DEBUG } 20:17:32 DEBUG } 20:17:32 INFO [http_cmd]: http status code OK 20:17:32 DEBUG --- status code --- 20:17:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:32 DEBUG --- http response --- 20:17:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:17:32 DEBUG ----- output ----- 20:17:32 DEBUG { 20:17:32 DEBUG "status": "success", 20:17:32 DEBUG "data": { 20:17:32 DEBUG "resultType": "vector", 20:17:32 DEBUG "result": [ 20:17:32 DEBUG { 20:17:32 DEBUG "metric": { 20:17:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:17:32 DEBUG }, 20:17:32 DEBUG "value": [ 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG [...] 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:17:32 DEBUG }, 20:17:32 DEBUG "value": [ 20:17:32 DEBUG 1713554252, 20:17:32 DEBUG "0.015" 20:17:32 DEBUG ] 20:17:32 DEBUG } 20:17:32 DEBUG ] 20:17:32 DEBUG } 20:17:32 DEBUG } 20:17:32 INFO [http_cmd]: http status code OK 20:17:32 DEBUG --- status code --- 20:17:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:32 DEBUG --- http response --- 20:17:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:17:32 DEBUG ----- output ----- 20:17:32 DEBUG { 20:17:32 DEBUG "status": "success", 20:17:32 DEBUG "data": { 20:17:32 DEBUG "resultType": "vector", 20:17:32 DEBUG "result": [ 20:17:32 DEBUG { 20:17:32 DEBUG "metric": { 20:17:32 DEBUG "pod": "ds-idrepo-0" 20:17:32 DEBUG }, 20:17:32 DEBUG "value": [ 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG [...] 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG "pod": "ds-idrepo-2" 20:17:32 DEBUG }, 20:17:32 DEBUG "value": [ 20:17:32 DEBUG 1713554252, 20:17:32 DEBUG "8.966666666666667" 20:17:32 DEBUG ] 20:17:32 DEBUG } 20:17:32 DEBUG ] 20:17:32 DEBUG } 20:17:32 DEBUG } 20:17:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554252" 20:17:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554252" 20:17:32 INFO [http_cmd]: http status code OK 20:17:32 DEBUG --- status code --- 20:17:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:32 DEBUG --- http response --- 20:17:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:17:32 DEBUG ----- output ----- 20:17:32 DEBUG { 20:17:32 DEBUG "status": "success", 20:17:32 DEBUG "data": { 20:17:32 DEBUG "resultType": "vector", 20:17:32 DEBUG "result": [ 20:17:32 DEBUG { 20:17:32 DEBUG "metric": { 20:17:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:17:32 DEBUG }, 20:17:32 DEBUG "value": [ 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG [...] 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:17:32 DEBUG }, 20:17:32 DEBUG "value": [ 20:17:32 DEBUG 1713554252, 20:17:32 DEBUG "2486.233333333333" 20:17:32 DEBUG ] 20:17:32 DEBUG } 20:17:32 DEBUG ] 20:17:32 DEBUG } 20:17:32 DEBUG } 20:17:32 INFO [http_cmd]: http status code OK 20:17:32 DEBUG --- status code --- 20:17:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:32 DEBUG --- http response --- 20:17:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:17:32 DEBUG ----- output ----- 20:17:32 DEBUG { 20:17:32 DEBUG "status": "success", 20:17:32 DEBUG "data": { 20:17:32 DEBUG "resultType": "vector", 20:17:32 DEBUG "result": [ 20:17:32 DEBUG { 20:17:32 DEBUG "metric": { 20:17:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:17:32 DEBUG }, 20:17:32 DEBUG "value": [ 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG [...] 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:17:32 DEBUG }, 20:17:32 DEBUG "value": [ 20:17:32 DEBUG 1713554252, 20:17:32 DEBUG "0" 20:17:32 DEBUG ] 20:17:32 DEBUG } 20:17:32 DEBUG ] 20:17:32 DEBUG } 20:17:32 DEBUG } 20:17:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554252" 20:17:32 INFO [http_cmd]: http status code OK 20:17:32 DEBUG --- status code --- 20:17:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:32 DEBUG --- http response --- 20:17:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:17:32 DEBUG ----- output ----- 20:17:32 DEBUG { 20:17:32 DEBUG "status": "success", 20:17:32 DEBUG "data": { 20:17:32 DEBUG "resultType": "vector", 20:17:32 DEBUG "result": [ 20:17:32 DEBUG { 20:17:32 DEBUG "metric": { 20:17:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:17:32 DEBUG }, 20:17:32 DEBUG "value": [ 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG [...] 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:17:32 DEBUG }, 20:17:32 DEBUG "value": [ 20:17:32 DEBUG 1713554252, 20:17:32 DEBUG "0" 20:17:32 DEBUG ] 20:17:32 DEBUG } 20:17:32 DEBUG ] 20:17:32 DEBUG } 20:17:32 DEBUG } 20:17:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554252" 20:17:32 INFO [http_cmd]: http status code OK 20:17:32 DEBUG --- status code --- 20:17:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:32 DEBUG --- http response --- 20:17:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:17:32 DEBUG ----- output ----- 20:17:32 DEBUG { 20:17:32 DEBUG "status": "success", 20:17:32 DEBUG "data": { 20:17:32 DEBUG "resultType": "vector", 20:17:32 DEBUG "result": [ 20:17:32 DEBUG { 20:17:32 DEBUG "metric": { 20:17:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:17:32 DEBUG }, 20:17:32 DEBUG "value": [ 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG [...] 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:17:32 DEBUG }, 20:17:32 DEBUG "value": [ 20:17:32 DEBUG 1713554252, 20:17:32 DEBUG "0" 20:17:32 DEBUG ] 20:17:32 DEBUG } 20:17:32 DEBUG ] 20:17:32 DEBUG } 20:17:32 DEBUG } 20:17:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554252" 20:17:32 INFO [http_cmd]: http status code OK 20:17:32 DEBUG --- status code --- 20:17:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:32 DEBUG --- http response --- 20:17:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:17:32 DEBUG ----- output ----- 20:17:32 DEBUG { 20:17:32 DEBUG "status": "success", 20:17:32 DEBUG "data": { 20:17:32 DEBUG "resultType": "vector", 20:17:32 DEBUG "result": [ 20:17:32 DEBUG { 20:17:32 DEBUG "metric": { 20:17:32 DEBUG "instance": "10.142.0.107:9100" 20:17:32 DEBUG }, 20:17:32 DEBUG "value": [ 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG [...] 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG "instance": "10.142.0.43:9100" 20:17:32 DEBUG }, 20:17:32 DEBUG "value": [ 20:17:32 DEBUG 1713554252, 20:17:32 DEBUG "2061000078336" 20:17:32 DEBUG ] 20:17:32 DEBUG } 20:17:32 DEBUG ] 20:17:32 DEBUG } 20:17:32 DEBUG } 20:17:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713554252" 20:17:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554252" 20:17:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554252" 20:17:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554252" 20:17:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554252" 20:17:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554252" 20:17:32 INFO [http_cmd]: http status code OK 20:17:32 DEBUG --- status code --- 20:17:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:32 DEBUG --- http response --- 20:17:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:17:32 DEBUG ----- output ----- 20:17:32 DEBUG { 20:17:32 DEBUG "status": "success", 20:17:32 DEBUG "data": { 20:17:32 DEBUG "resultType": "vector", 20:17:32 DEBUG "result": [ 20:17:32 DEBUG { 20:17:32 DEBUG "metric": { 20:17:32 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:17:32 DEBUG }, 20:17:32 DEBUG "value": [ 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG [...] 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:17:32 DEBUG }, 20:17:32 DEBUG "value": [ 20:17:32 DEBUG 1713554252, 20:17:32 DEBUG "3283578880" 20:17:32 DEBUG ] 20:17:32 DEBUG } 20:17:32 DEBUG ] 20:17:32 DEBUG } 20:17:32 DEBUG } 20:17:32 INFO [http_cmd]: http status code OK 20:17:32 DEBUG --- status code --- 20:17:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:32 DEBUG --- http response --- 20:17:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:17:32 DEBUG ----- output ----- 20:17:32 DEBUG { 20:17:32 DEBUG "status": "success", 20:17:32 DEBUG "data": { 20:17:32 DEBUG "resultType": "vector", 20:17:32 DEBUG "result": [ 20:17:32 DEBUG { 20:17:32 DEBUG "metric": { 20:17:32 DEBUG "instance": "10.142.0.107:9100" 20:17:32 DEBUG }, 20:17:32 DEBUG "value": [ 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG [...] 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG "instance": "10.142.0.43:9100" 20:17:32 DEBUG }, 20:17:32 DEBUG "value": [ 20:17:32 DEBUG 1713554252, 20:17:32 DEBUG "16354727.600000001" 20:17:32 DEBUG ] 20:17:32 DEBUG } 20:17:32 DEBUG ] 20:17:32 DEBUG } 20:17:32 DEBUG } 20:17:32 INFO [http_cmd]: http status code OK 20:17:32 INFO [http_cmd]: http status code OK 20:17:32 DEBUG --- status code --- 20:17:32 DEBUG --- status code --- 20:17:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:32 DEBUG --- http response --- 20:17:32 DEBUG --- http response --- 20:17:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 20:17:32 DEBUG ----- output ----- 20:17:32 DEBUG { 20:17:32 DEBUG "status": "success", 20:17:32 DEBUG "data": { 20:17:32 DEBUG "resultType": "vector", 20:17:32 DEBUG "result": [ 20:17:32 DEBUG { 20:17:32 DEBUG "metric": { 20:17:32 DEBUG "pod": "ds-idrepo-1" 20:17:32 DEBUG }, 20:17:32 DEBUG "value": [ 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG [...] 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG "pod": "am-767f6b67b5-qcqz2" 20:17:32 DEBUG }, 20:17:32 DEBUG "value": [ 20:17:32 DEBUG 1713554252, 20:17:32 DEBUG "0" 20:17:32 DEBUG ] 20:17:32 DEBUG } 20:17:32 DEBUG ] 20:17:32 DEBUG } 20:17:32 DEBUG } 20:17:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:17:32 INFO [http_cmd]: http status code OK 20:17:32 DEBUG ----- output ----- 20:17:32 DEBUG --- status code --- 20:17:32 DEBUG { 20:17:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:32 DEBUG "status": "success", 20:17:32 DEBUG --- http response --- 20:17:32 DEBUG "data": { 20:17:32 DEBUG "resultType": "vector", 20:17:32 DEBUG "result": [ 20:17:32 DEBUG { 20:17:32 DEBUG "metric": { 20:17:32 DEBUG "pod": "ds-cts-0" 20:17:32 DEBUG }, 20:17:32 DEBUG "value": [ 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG [...] 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG "pod": "ds-cts-2" 20:17:32 DEBUG }, 20:17:32 DEBUG "value": [ 20:17:32 DEBUG 1713554252, 20:17:32 DEBUG "0" 20:17:32 DEBUG ] 20:17:32 DEBUG } 20:17:32 DEBUG ] 20:17:32 DEBUG } 20:17:32 DEBUG } 20:17:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:17:32 DEBUG ----- output ----- 20:17:32 DEBUG { 20:17:32 DEBUG "status": "success", 20:17:32 DEBUG "data": { 20:17:32 INFO [http_cmd]: http status code OK 20:17:32 DEBUG "resultType": "vector", 20:17:32 DEBUG --- status code --- 20:17:32 DEBUG "result": [ 20:17:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:32 DEBUG { 20:17:32 DEBUG --- http response --- 20:17:32 DEBUG "metric": { 20:17:32 DEBUG "pod": "ds-cts-0" 20:17:32 DEBUG }, 20:17:32 DEBUG "value": [ 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG [...] 20:17:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:17:32 DEBUG 20:17:32 DEBUG ----- output ----- 20:17:32 DEBUG 20:17:32 DEBUG { 20:17:32 DEBUG 20:17:32 DEBUG "status": "success", 20:17:32 DEBUG "pod": "ds-cts-2" 20:17:32 DEBUG "data": { 20:17:32 DEBUG }, 20:17:32 DEBUG "resultType": "vector", 20:17:32 DEBUG "value": [ 20:17:32 DEBUG "result": [ 20:17:32 DEBUG 1713554252, 20:17:32 DEBUG { 20:17:32 DEBUG "29.966666666666665" 20:17:32 DEBUG "metric": { 20:17:32 DEBUG ] 20:17:32 DEBUG "instance": "10.142.0.107:9100" 20:17:32 DEBUG } 20:17:32 DEBUG }, 20:17:32 DEBUG ] 20:17:32 DEBUG "value": [ 20:17:32 DEBUG } 20:17:32 DEBUG 20:17:32 DEBUG } 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG [...] 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG "instance": "10.142.0.43:9100" 20:17:32 DEBUG }, 20:17:32 DEBUG "value": [ 20:17:32 DEBUG 1713554252, 20:17:32 DEBUG "16354348.766666666" 20:17:32 DEBUG ] 20:17:32 DEBUG } 20:17:32 DEBUG ] 20:17:32 DEBUG } 20:17:32 DEBUG } 20:17:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554252" 20:17:32 INFO [http_cmd]: http status code OK 20:17:32 DEBUG --- status code --- 20:17:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:32 DEBUG --- http response --- 20:17:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 20:17:32 DEBUG ----- output ----- 20:17:32 DEBUG { 20:17:32 DEBUG "status": "success", 20:17:32 DEBUG "data": { 20:17:32 DEBUG "resultType": "vector", 20:17:32 DEBUG "result": [ 20:17:32 DEBUG { 20:17:32 DEBUG "metric": { 20:17:32 DEBUG "pod": "ds-idrepo-1" 20:17:32 DEBUG }, 20:17:32 DEBUG "value": [ 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG [...] 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG 20:17:32 DEBUG "pod": "am-767f6b67b5-qcqz2" 20:17:32 DEBUG }, 20:17:32 DEBUG "value": [ 20:17:32 DEBUG 1713554252, 20:17:32 DEBUG "2.460889431556041" 20:17:32 DEBUG ] 20:17:32 DEBUG } 20:17:32 DEBUG ] 20:17:32 DEBUG } 20:17:32 DEBUG } 20:17:34 INFO 20:17:34 INFO [loop_until]: kubectl --namespace=xlou top pods 20:17:34 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:17:34 INFO [loop_until]: OK (rc = 0) 20:17:34 DEBUG --- stdout --- 20:17:34 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 2332m 8974Mi am-767f6b67b5-qcqz2 2460m 8770Mi am-767f6b67b5-vbltz 2776m 8963Mi ds-cts-0 7285m 10942Mi ds-cts-1 6382m 11201Mi ds-cts-2 7644m 10960Mi ds-idrepo-0 1466m 3714Mi ds-idrepo-1 1077m 3561Mi ds-idrepo-2 937m 3492Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1175Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 1025m 3230Mi 20:17:34 DEBUG --- stderr --- 20:17:34 DEBUG 20:17:36 INFO 20:17:36 INFO [loop_until]: kubectl --namespace=xlou top node 20:17:36 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:17:36 INFO [loop_until]: OK (rc = 0) 20:17:36 DEBUG --- stdout --- 20:17:36 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 72m 0% 6237Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 1278m 4% 7384Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 2395m 8% 11711Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 86m 0% 3364Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 7373m 24% 13954Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 162m 0% 5195Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 2489m 8% 11681Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 76m 0% 2986Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 2891m 9% 11208Mi 10% gke-xlou-cdm-ds-0f601751-j488 1083m 3% 6718Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 6575m 22% 15957Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 1143m 3% 7444Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 7518m 25% 13431Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 1075m 6% 6964Mi 11% 20:17:36 DEBUG --- stderr --- 20:17:36 DEBUG 20:18:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554312" 20:18:32 INFO [http_cmd]: http status code OK 20:18:32 DEBUG --- status code --- 20:18:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:32 DEBUG --- http response --- 20:18:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:18:32 DEBUG ----- output ----- 20:18:32 DEBUG { 20:18:32 DEBUG "status": "success", 20:18:32 DEBUG "data": { 20:18:32 DEBUG "resultType": "vector", 20:18:32 DEBUG "result": [ 20:18:32 DEBUG { 20:18:32 DEBUG "metric": { 20:18:32 DEBUG "pod": "ds-idrepo-0" 20:18:32 DEBUG }, 20:18:32 DEBUG "value": [ 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG [...] 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG "pod": "ds-idrepo-2" 20:18:32 DEBUG }, 20:18:32 DEBUG "value": [ 20:18:32 DEBUG 1713554312, 20:18:32 DEBUG "62.9" 20:18:32 DEBUG ] 20:18:32 DEBUG } 20:18:32 DEBUG ] 20:18:32 DEBUG } 20:18:32 DEBUG } 20:18:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554312" 20:18:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554312" 20:18:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554312" 20:18:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554312" 20:18:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713554312" 20:18:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554312" 20:18:32 INFO [http_cmd]: http status code OK 20:18:32 DEBUG --- status code --- 20:18:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:32 DEBUG --- http response --- 20:18:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:18:32 DEBUG ----- output ----- 20:18:32 DEBUG { 20:18:32 DEBUG "status": "success", 20:18:32 DEBUG "data": { 20:18:32 DEBUG "resultType": "vector", 20:18:32 DEBUG "result": [ 20:18:32 DEBUG { 20:18:32 DEBUG "metric": { 20:18:32 DEBUG "instance": "10.142.0.107:9100" 20:18:32 DEBUG }, 20:18:32 DEBUG "value": [ 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG [...] 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG "instance": "10.142.0.43:9100" 20:18:32 DEBUG }, 20:18:32 DEBUG "value": [ 20:18:32 DEBUG 1713554312, 20:18:32 DEBUG "0.014666666666659008" 20:18:32 DEBUG ] 20:18:32 DEBUG } 20:18:32 DEBUG ] 20:18:32 DEBUG } 20:18:32 DEBUG } 20:18:32 INFO [http_cmd]: http status code OK 20:18:32 DEBUG --- status code --- 20:18:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:32 DEBUG --- http response --- 20:18:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:18:32 DEBUG ----- output ----- 20:18:32 DEBUG { 20:18:32 DEBUG "status": "success", 20:18:32 DEBUG "data": { 20:18:32 DEBUG "resultType": "vector", 20:18:32 DEBUG "result": [ 20:18:32 DEBUG { 20:18:32 DEBUG "metric": { 20:18:32 DEBUG "pod": "ds-idrepo-0" 20:18:32 DEBUG }, 20:18:32 DEBUG "value": [ 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG [...] 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG "pod": "ds-idrepo-2" 20:18:32 DEBUG }, 20:18:32 DEBUG "value": [ 20:18:32 DEBUG 1713554312, 20:18:32 DEBUG "6.7" 20:18:32 DEBUG ] 20:18:32 DEBUG } 20:18:32 DEBUG ] 20:18:32 DEBUG } 20:18:32 DEBUG } 20:18:32 INFO [http_cmd]: http status code OK 20:18:32 ERROR [http_cmd]: ERROR 20:18:32 INFO [http_cmd]: http status code OK 20:18:32 INFO [http_cmd]: http status code OK 20:18:32 DEBUG --- status code --- 20:18:32 ERROR --- status code --- 20:18:32 DEBUG --- status code --- 20:18:32 DEBUG --- status code --- 20:18:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:32 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:18:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:32 DEBUG --- http response --- 20:18:32 ERROR --- http response --- 20:18:32 DEBUG --- http response --- 20:18:32 DEBUG --- http response --- 20:18:32 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:18:32 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:18:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:18:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:18:32 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:18:32 DEBUG ----- output ----- 20:18:32 DEBUG ----- output ----- 20:18:32 DEBUG { 20:18:32 DEBUG { 20:18:32 DEBUG "status": "success", 20:18:32 DEBUG "status": "success", 20:18:32 DEBUG "data": { 20:18:32 DEBUG "data": { 20:18:32 DEBUG "resultType": "vector", 20:18:32 DEBUG "resultType": "vector", 20:18:32 DEBUG "result": [ 20:18:32 DEBUG "result": [ 20:18:32 DEBUG { 20:18:32 DEBUG { 20:18:32 DEBUG "metric": { 20:18:32 DEBUG "metric": { 20:18:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:18:32 DEBUG "pod": "ds-cts-0" 20:18:32 DEBUG }, 20:18:32 DEBUG }, 20:18:32 DEBUG "value": [ 20:18:32 DEBUG "value": [ 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG [...] 20:18:32 DEBUG [...] 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554312" 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:18:32 DEBUG "pod": "ds-cts-2" 20:18:32 DEBUG }, 20:18:32 DEBUG }, 20:18:32 DEBUG "value": [ 20:18:32 DEBUG "value": [ 20:18:32 DEBUG 1713554312, 20:18:32 DEBUG 1713554312, 20:18:32 DEBUG "0" 20:18:32 DEBUG "0.000021548888888888887" 20:18:32 DEBUG ] 20:18:32 DEBUG ] 20:18:32 DEBUG } 20:18:32 DEBUG } 20:18:32 DEBUG ] 20:18:32 DEBUG ] 20:18:32 DEBUG } 20:18:32 DEBUG } 20:18:32 DEBUG } 20:18:32 DEBUG } 20:18:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713554312" 20:18:32 INFO [http_cmd]: http status code OK 20:18:32 DEBUG --- status code --- 20:18:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:32 DEBUG --- http response --- 20:18:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:18:32 DEBUG ----- output ----- 20:18:32 DEBUG { 20:18:32 DEBUG "status": "success", 20:18:32 DEBUG "data": { 20:18:32 DEBUG "resultType": "vector", 20:18:32 DEBUG "result": [ 20:18:32 DEBUG { 20:18:32 DEBUG "metric": { 20:18:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:18:32 DEBUG }, 20:18:32 DEBUG "value": [ 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG [...] 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:18:32 DEBUG }, 20:18:32 DEBUG "value": [ 20:18:32 DEBUG 1713554312, 20:18:32 DEBUG "67745.5" 20:18:32 DEBUG ] 20:18:32 DEBUG } 20:18:32 DEBUG ] 20:18:32 DEBUG } 20:18:32 DEBUG } 20:18:32 INFO [http_cmd]: http status code OK 20:18:32 DEBUG --- status code --- 20:18:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:32 DEBUG --- http response --- 20:18:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:18:32 DEBUG ----- output ----- 20:18:32 DEBUG { 20:18:32 DEBUG "status": "success", 20:18:32 DEBUG "data": { 20:18:32 DEBUG "resultType": "vector", 20:18:32 DEBUG "result": [ 20:18:32 DEBUG { 20:18:32 DEBUG "metric": { 20:18:32 DEBUG "pod": "ds-idrepo-0" 20:18:32 DEBUG }, 20:18:32 DEBUG "value": [ 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG [...] 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG "pod": "ds-cts-2" 20:18:32 DEBUG }, 20:18:32 DEBUG "value": [ 20:18:32 DEBUG 1713554312, 20:18:32 DEBUG "0" 20:18:32 DEBUG ] 20:18:32 DEBUG } 20:18:32 DEBUG ] 20:18:32 DEBUG } 20:18:32 DEBUG } 20:18:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554312" 20:18:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554312" 20:18:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713554312" 20:18:32 INFO [http_cmd]: http status code OK 20:18:32 INFO [http_cmd]: http status code OK 20:18:32 DEBUG --- status code --- 20:18:32 DEBUG --- status code --- 20:18:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:32 DEBUG --- http response --- 20:18:32 DEBUG --- http response --- 20:18:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:18:32 DEBUG ----- output ----- 20:18:32 DEBUG { 20:18:32 DEBUG "status": "success", 20:18:32 DEBUG "data": { 20:18:32 DEBUG "resultType": "vector", 20:18:32 DEBUG "result": [ 20:18:32 DEBUG { 20:18:32 DEBUG "metric": { 20:18:32 DEBUG "instance": "10.142.0.107:9100" 20:18:32 DEBUG }, 20:18:32 DEBUG "value": [ 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG [...] 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG "instance": "10.142.0.43:9100" 20:18:32 DEBUG }, 20:18:32 DEBUG "value": [ 20:18:32 DEBUG 1713554312, 20:18:32 DEBUG "0.019266666666623185" 20:18:32 DEBUG ] 20:18:32 DEBUG } 20:18:32 DEBUG ] 20:18:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 20:18:32 DEBUG } 20:18:32 DEBUG ----- output ----- 20:18:32 DEBUG } 20:18:32 DEBUG { 20:18:32 DEBUG "status": "success", 20:18:32 DEBUG "data": { 20:18:32 DEBUG "resultType": "vector", 20:18:32 DEBUG "result": [ 20:18:32 DEBUG { 20:18:32 DEBUG "metric": { 20:18:32 DEBUG "pod": "ds-idrepo-1" 20:18:32 DEBUG }, 20:18:32 DEBUG "value": [ 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG [...] 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:18:32 DEBUG }, 20:18:32 DEBUG "value": [ 20:18:32 DEBUG 1713554312, 20:18:32 DEBUG "118.80576044959606" 20:18:32 DEBUG ] 20:18:32 DEBUG } 20:18:32 DEBUG ] 20:18:32 DEBUG } 20:18:32 DEBUG } 20:18:32 INFO [http_cmd]: http status code OK 20:18:32 DEBUG --- status code --- 20:18:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:32 DEBUG --- http response --- 20:18:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:18:32 DEBUG ----- output ----- 20:18:32 DEBUG { 20:18:32 DEBUG "status": "success", 20:18:32 DEBUG "data": { 20:18:32 DEBUG "resultType": "vector", 20:18:32 DEBUG "result": [ 20:18:32 DEBUG { 20:18:32 DEBUG "metric": { 20:18:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:18:32 DEBUG }, 20:18:32 DEBUG "value": [ 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG [...] 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:18:32 DEBUG }, 20:18:32 DEBUG "value": [ 20:18:32 DEBUG 1713554312, 20:18:32 DEBUG "0.015" 20:18:32 DEBUG ] 20:18:32 DEBUG } 20:18:32 DEBUG ] 20:18:32 DEBUG } 20:18:32 DEBUG } 20:18:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554312" 20:18:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713554312" 20:18:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554312" 20:18:32 INFO [http_cmd]: http status code OK 20:18:32 DEBUG --- status code --- 20:18:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:32 DEBUG --- http response --- 20:18:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 20:18:32 DEBUG ----- output ----- 20:18:32 DEBUG { 20:18:32 DEBUG "status": "success", 20:18:32 DEBUG "data": { 20:18:32 DEBUG "resultType": "vector", 20:18:32 DEBUG "result": [ 20:18:32 DEBUG { 20:18:32 DEBUG "metric": { 20:18:32 DEBUG "pod": "ds-idrepo-1" 20:18:32 DEBUG }, 20:18:32 DEBUG "value": [ 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG [...] 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:18:32 DEBUG }, 20:18:32 DEBUG "value": [ 20:18:32 DEBUG 1713554312, 20:18:32 DEBUG "0" 20:18:32 DEBUG ] 20:18:32 DEBUG } 20:18:32 DEBUG ] 20:18:32 DEBUG } 20:18:32 DEBUG } 20:18:32 INFO [http_cmd]: http status code OK 20:18:32 DEBUG --- status code --- 20:18:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:32 DEBUG --- http response --- 20:18:32 INFO [http_cmd]: http status code OK 20:18:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:18:32 DEBUG --- status code --- 20:18:32 DEBUG ----- output ----- 20:18:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:32 DEBUG { 20:18:32 DEBUG --- http response --- 20:18:32 DEBUG "status": "success", 20:18:32 DEBUG "data": { 20:18:32 DEBUG "resultType": "vector", 20:18:32 DEBUG "result": [ 20:18:32 DEBUG { 20:18:32 DEBUG "metric": { 20:18:32 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:18:32 DEBUG }, 20:18:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:18:32 DEBUG "value": [ 20:18:32 DEBUG ----- output ----- 20:18:32 DEBUG 20:18:32 DEBUG { 20:18:32 DEBUG 20:18:32 DEBUG "status": "success", 20:18:32 DEBUG 20:18:32 DEBUG "data": { 20:18:32 DEBUG [...] 20:18:32 DEBUG "resultType": "vector", 20:18:32 DEBUG 20:18:32 DEBUG "result": [ 20:18:32 DEBUG 20:18:32 DEBUG { 20:18:32 DEBUG 20:18:32 DEBUG "metric": { 20:18:32 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:18:32 DEBUG "pod": "ds-idrepo-0" 20:18:32 DEBUG }, 20:18:32 DEBUG }, 20:18:32 DEBUG "value": [ 20:18:32 DEBUG "value": [ 20:18:32 DEBUG 1713554312, 20:18:32 DEBUG 20:18:32 DEBUG "1.0277291242391613" 20:18:32 DEBUG 20:18:32 DEBUG ] 20:18:32 DEBUG 20:18:32 DEBUG } 20:18:32 DEBUG [...] 20:18:32 DEBUG ] 20:18:32 DEBUG 20:18:32 DEBUG } 20:18:32 DEBUG 20:18:32 DEBUG } 20:18:32 DEBUG 20:18:32 DEBUG "pod": "ds-idrepo-2" 20:18:32 DEBUG }, 20:18:32 DEBUG "value": [ 20:18:32 DEBUG 1713554312, 20:18:32 DEBUG "8.9" 20:18:32 DEBUG ] 20:18:32 DEBUG } 20:18:32 DEBUG ] 20:18:32 DEBUG } 20:18:32 DEBUG } 20:18:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554312" 20:18:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554312" 20:18:32 INFO [http_cmd]: http status code OK 20:18:32 DEBUG --- status code --- 20:18:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:32 DEBUG --- http response --- 20:18:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:18:32 DEBUG ----- output ----- 20:18:32 DEBUG { 20:18:32 DEBUG "status": "success", 20:18:32 DEBUG "data": { 20:18:32 DEBUG "resultType": "vector", 20:18:32 DEBUG "result": [ 20:18:32 DEBUG { 20:18:32 DEBUG "metric": { 20:18:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:18:32 DEBUG }, 20:18:32 DEBUG "value": [ 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG [...] 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:18:32 DEBUG }, 20:18:32 DEBUG "value": [ 20:18:32 DEBUG 1713554312, 20:18:32 DEBUG "2510.233333333333" 20:18:32 DEBUG ] 20:18:32 DEBUG } 20:18:32 DEBUG ] 20:18:32 DEBUG } 20:18:32 DEBUG } 20:18:32 INFO [http_cmd]: http status code OK 20:18:32 DEBUG --- status code --- 20:18:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:32 DEBUG --- http response --- 20:18:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:18:32 DEBUG ----- output ----- 20:18:32 DEBUG { 20:18:32 DEBUG "status": "success", 20:18:32 DEBUG "data": { 20:18:32 DEBUG "resultType": "vector", 20:18:32 DEBUG "result": [ 20:18:32 DEBUG { 20:18:32 DEBUG "metric": { 20:18:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:18:32 DEBUG }, 20:18:32 DEBUG "value": [ 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG [...] 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:18:32 DEBUG }, 20:18:32 DEBUG "value": [ 20:18:32 DEBUG 1713554312, 20:18:32 DEBUG "0" 20:18:32 DEBUG ] 20:18:32 DEBUG } 20:18:32 DEBUG ] 20:18:32 DEBUG } 20:18:32 DEBUG } 20:18:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554312" 20:18:32 INFO [http_cmd]: http status code OK 20:18:32 DEBUG --- status code --- 20:18:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:32 DEBUG --- http response --- 20:18:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:18:32 DEBUG ----- output ----- 20:18:32 DEBUG { 20:18:32 DEBUG "status": "success", 20:18:32 DEBUG "data": { 20:18:32 DEBUG "resultType": "vector", 20:18:32 DEBUG "result": [ 20:18:32 DEBUG { 20:18:32 DEBUG "metric": { 20:18:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:18:32 DEBUG }, 20:18:32 DEBUG "value": [ 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG [...] 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:18:32 DEBUG }, 20:18:32 DEBUG "value": [ 20:18:32 DEBUG 1713554312, 20:18:32 DEBUG "0" 20:18:32 DEBUG ] 20:18:32 DEBUG } 20:18:32 DEBUG ] 20:18:32 DEBUG } 20:18:32 DEBUG } 20:18:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554312" 20:18:32 INFO [http_cmd]: http status code OK 20:18:32 DEBUG --- status code --- 20:18:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:32 DEBUG --- http response --- 20:18:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:18:32 DEBUG ----- output ----- 20:18:32 DEBUG { 20:18:32 DEBUG "status": "success", 20:18:32 DEBUG "data": { 20:18:32 DEBUG "resultType": "vector", 20:18:32 DEBUG "result": [ 20:18:32 DEBUG { 20:18:32 DEBUG "metric": { 20:18:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:18:32 DEBUG }, 20:18:32 DEBUG "value": [ 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG [...] 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:18:32 DEBUG }, 20:18:32 DEBUG "value": [ 20:18:32 DEBUG 1713554312, 20:18:32 DEBUG "0" 20:18:32 DEBUG ] 20:18:32 DEBUG } 20:18:32 DEBUG ] 20:18:32 DEBUG } 20:18:32 DEBUG } 20:18:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554312" 20:18:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554312" 20:18:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554312" 20:18:32 INFO [http_cmd]: http status code OK 20:18:32 DEBUG --- status code --- 20:18:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:32 DEBUG --- http response --- 20:18:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:18:32 DEBUG ----- output ----- 20:18:32 DEBUG { 20:18:32 DEBUG "status": "success", 20:18:32 DEBUG "data": { 20:18:32 DEBUG "resultType": "vector", 20:18:32 DEBUG "result": [ 20:18:32 DEBUG { 20:18:32 DEBUG "metric": { 20:18:32 DEBUG "instance": "10.142.0.107:9100" 20:18:32 DEBUG }, 20:18:32 DEBUG "value": [ 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG [...] 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG "instance": "10.142.0.43:9100" 20:18:32 DEBUG }, 20:18:32 DEBUG "value": [ 20:18:32 DEBUG 1713554312, 20:18:32 DEBUG "2063018492928" 20:18:32 DEBUG ] 20:18:32 DEBUG } 20:18:32 DEBUG ] 20:18:32 DEBUG } 20:18:32 DEBUG } 20:18:32 INFO [http_cmd]: http status code OK 20:18:32 DEBUG --- status code --- 20:18:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:32 DEBUG --- http response --- 20:18:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 20:18:32 DEBUG ----- output ----- 20:18:32 DEBUG { 20:18:32 DEBUG "status": "success", 20:18:32 DEBUG "data": { 20:18:32 DEBUG "resultType": "vector", 20:18:32 DEBUG "result": [ 20:18:32 DEBUG { 20:18:32 DEBUG "metric": { 20:18:32 DEBUG "pod": "ds-idrepo-1" 20:18:32 DEBUG }, 20:18:32 DEBUG "value": [ 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG [...] 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:18:32 DEBUG }, 20:18:32 DEBUG "value": [ 20:18:32 DEBUG 1713554312, 20:18:32 DEBUG "0.000011805162232175559" 20:18:32 DEBUG ] 20:18:32 DEBUG } 20:18:32 DEBUG ] 20:18:32 DEBUG } 20:18:32 DEBUG } 20:18:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554312" 20:18:32 INFO [http_cmd]: http status code OK 20:18:32 DEBUG --- status code --- 20:18:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:32 DEBUG --- http response --- 20:18:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:18:32 DEBUG ----- output ----- 20:18:32 DEBUG { 20:18:32 DEBUG "status": "success", 20:18:32 DEBUG "data": { 20:18:32 DEBUG "resultType": "vector", 20:18:32 DEBUG "result": [ 20:18:32 DEBUG { 20:18:32 DEBUG "metric": { 20:18:32 DEBUG "instance": "10.142.0.107:9100" 20:18:32 DEBUG }, 20:18:32 DEBUG "value": [ 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG [...] 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG "instance": "10.142.0.43:9100" 20:18:32 DEBUG }, 20:18:32 DEBUG "value": [ 20:18:32 DEBUG 1713554312, 20:18:32 DEBUG "16489022.266666666" 20:18:32 DEBUG ] 20:18:32 DEBUG } 20:18:32 DEBUG ] 20:18:32 DEBUG } 20:18:32 DEBUG } 20:18:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713554312" 20:18:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554312" 20:18:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554312" 20:18:32 INFO [http_cmd]: http status code OK 20:18:32 DEBUG --- status code --- 20:18:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:32 DEBUG --- http response --- 20:18:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:18:32 DEBUG ----- output ----- 20:18:32 DEBUG { 20:18:32 DEBUG "status": "success", 20:18:32 DEBUG "data": { 20:18:32 DEBUG "resultType": "vector", 20:18:32 DEBUG "result": [ 20:18:32 DEBUG { 20:18:32 DEBUG "metric": { 20:18:32 DEBUG "pod": "ds-cts-0" 20:18:32 DEBUG }, 20:18:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554312" 20:18:32 DEBUG "value": [ 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG [...] 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG "pod": "ds-cts-2" 20:18:32 INFO [http_cmd]: http status code OK 20:18:32 DEBUG }, 20:18:32 DEBUG --- status code --- 20:18:32 DEBUG "value": [ 20:18:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:32 DEBUG 1713554312, 20:18:32 DEBUG --- http response --- 20:18:32 DEBUG "29.966666666666665" 20:18:32 DEBUG ] 20:18:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:18:32 DEBUG } 20:18:32 DEBUG ----- output ----- 20:18:32 DEBUG ] 20:18:32 DEBUG { 20:18:32 DEBUG } 20:18:32 DEBUG "status": "success", 20:18:32 DEBUG } 20:18:32 DEBUG "data": { 20:18:32 DEBUG "resultType": "vector", 20:18:32 DEBUG "result": [ 20:18:32 DEBUG { 20:18:32 DEBUG "metric": { 20:18:32 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:18:32 DEBUG }, 20:18:32 DEBUG "value": [ 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG [...] 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:18:32 DEBUG }, 20:18:32 DEBUG "value": [ 20:18:32 DEBUG 1713554312, 20:18:32 DEBUG "3737296896" 20:18:32 DEBUG ] 20:18:32 DEBUG } 20:18:32 DEBUG ] 20:18:32 DEBUG } 20:18:32 DEBUG } 20:18:32 INFO [http_cmd]: http status code OK 20:18:32 DEBUG --- status code --- 20:18:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:32 DEBUG --- http response --- 20:18:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:18:32 DEBUG ----- output ----- 20:18:32 DEBUG { 20:18:32 DEBUG "status": "success", 20:18:32 DEBUG "data": { 20:18:32 DEBUG "resultType": "vector", 20:18:32 DEBUG "result": [ 20:18:32 DEBUG { 20:18:32 DEBUG "metric": { 20:18:32 DEBUG "pod": "ds-cts-0" 20:18:32 DEBUG }, 20:18:32 DEBUG "value": [ 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG [...] 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG "pod": "ds-cts-2" 20:18:32 DEBUG }, 20:18:32 DEBUG "value": [ 20:18:32 DEBUG 1713554312, 20:18:32 DEBUG "0" 20:18:32 DEBUG ] 20:18:32 DEBUG } 20:18:32 DEBUG ] 20:18:32 DEBUG } 20:18:32 DEBUG } 20:18:32 INFO [http_cmd]: http status code OK 20:18:32 DEBUG --- status code --- 20:18:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:32 DEBUG --- http response --- 20:18:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 20:18:32 DEBUG ----- output ----- 20:18:32 DEBUG { 20:18:32 DEBUG "status": "success", 20:18:32 DEBUG "data": { 20:18:32 DEBUG "resultType": "vector", 20:18:32 DEBUG "result": [ 20:18:32 DEBUG { 20:18:32 DEBUG "metric": { 20:18:32 DEBUG "pod": "ds-idrepo-1" 20:18:32 DEBUG }, 20:18:32 DEBUG "value": [ 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG [...] 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:18:32 DEBUG }, 20:18:32 DEBUG "value": [ 20:18:32 DEBUG 1713554312, 20:18:32 DEBUG "0" 20:18:32 DEBUG ] 20:18:32 DEBUG } 20:18:32 DEBUG ] 20:18:32 DEBUG } 20:18:32 DEBUG } 20:18:32 INFO [http_cmd]: http status code OK 20:18:32 DEBUG --- status code --- 20:18:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:32 DEBUG --- http response --- 20:18:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:18:32 DEBUG ----- output ----- 20:18:32 DEBUG { 20:18:32 DEBUG "status": "success", 20:18:32 DEBUG "data": { 20:18:32 DEBUG "resultType": "vector", 20:18:32 DEBUG "result": [ 20:18:32 DEBUG { 20:18:32 DEBUG "metric": { 20:18:32 DEBUG "instance": "10.142.0.107:9100" 20:18:32 DEBUG }, 20:18:32 DEBUG "value": [ 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG [...] 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG 20:18:32 DEBUG "instance": "10.142.0.43:9100" 20:18:32 DEBUG }, 20:18:32 DEBUG "value": [ 20:18:32 DEBUG 1713554312, 20:18:32 DEBUG "16489020" 20:18:32 DEBUG ] 20:18:32 DEBUG } 20:18:32 DEBUG ] 20:18:32 DEBUG } 20:18:32 DEBUG } 20:18:34 INFO 20:18:34 INFO [loop_until]: kubectl --namespace=xlou top pods 20:18:34 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:18:34 INFO [loop_until]: OK (rc = 0) 20:18:34 DEBUG --- stdout --- 20:18:34 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 2450m 8973Mi am-767f6b67b5-qcqz2 2469m 8769Mi am-767f6b67b5-vbltz 2791m 8960Mi ds-cts-0 7123m 11008Mi ds-cts-1 6471m 11273Mi ds-cts-2 7273m 11041Mi ds-idrepo-0 738m 3716Mi ds-idrepo-1 746m 3562Mi ds-idrepo-2 717m 3493Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1175Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 1023m 3605Mi 20:18:34 DEBUG --- stderr --- 20:18:34 DEBUG 20:18:36 INFO 20:18:36 INFO [loop_until]: kubectl --namespace=xlou top node 20:18:36 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:18:36 INFO [loop_until]: OK (rc = 0) 20:18:36 DEBUG --- stdout --- 20:18:36 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 73m 0% 6244Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 804m 2% 7316Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 2404m 8% 11705Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 3362Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 7102m 23% 14022Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 162m 0% 5174Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 2517m 8% 11670Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 78m 0% 2966Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 2923m 9% 11206Mi 10% gke-xlou-cdm-ds-0f601751-j488 783m 2% 6742Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 6426m 21% 16029Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 819m 2% 7457Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 7346m 24% 13500Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 1073m 6% 7414Mi 12% 20:18:36 DEBUG --- stderr --- 20:18:36 DEBUG 20:19:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554372" 20:19:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554372" 20:19:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554372" 20:19:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554372" 20:19:32 INFO [http_cmd]: http status code OK 20:19:32 DEBUG --- status code --- 20:19:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:32 DEBUG --- http response --- 20:19:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:19:32 DEBUG ----- output ----- 20:19:32 DEBUG { 20:19:32 DEBUG "status": "success", 20:19:32 DEBUG "data": { 20:19:32 DEBUG "resultType": "vector", 20:19:32 DEBUG "result": [ 20:19:32 DEBUG { 20:19:32 DEBUG "metric": { 20:19:32 DEBUG "pod": "ds-idrepo-0" 20:19:32 DEBUG }, 20:19:32 DEBUG "value": [ 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG [...] 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG "pod": "ds-idrepo-2" 20:19:32 DEBUG }, 20:19:32 DEBUG "value": [ 20:19:32 DEBUG 1713554372, 20:19:32 DEBUG "0" 20:19:32 DEBUG ] 20:19:32 DEBUG } 20:19:32 DEBUG ] 20:19:32 DEBUG } 20:19:32 DEBUG } 20:19:32 INFO [http_cmd]: http status code OK 20:19:32 DEBUG --- status code --- 20:19:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:32 DEBUG --- http response --- 20:19:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:19:32 DEBUG ----- output ----- 20:19:32 DEBUG { 20:19:32 DEBUG "status": "success", 20:19:32 DEBUG "data": { 20:19:32 DEBUG "resultType": "vector", 20:19:32 DEBUG "result": [ 20:19:32 DEBUG { 20:19:32 DEBUG "metric": { 20:19:32 DEBUG "instance": "10.142.0.107:9100" 20:19:32 DEBUG }, 20:19:32 DEBUG "value": [ 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG [...] 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG "instance": "10.142.0.43:9100" 20:19:32 DEBUG }, 20:19:32 DEBUG "value": [ 20:19:32 DEBUG 1713554372, 20:19:32 DEBUG "0.002666666666668031" 20:19:32 DEBUG ] 20:19:32 DEBUG } 20:19:32 DEBUG ] 20:19:32 DEBUG } 20:19:32 DEBUG } 20:19:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554372" 20:19:32 INFO [http_cmd]: http status code OK 20:19:32 INFO [http_cmd]: http status code OK 20:19:32 DEBUG --- status code --- 20:19:32 DEBUG --- status code --- 20:19:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:32 DEBUG --- http response --- 20:19:32 DEBUG --- http response --- 20:19:32 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:19:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:19:32 DEBUG ----- output ----- 20:19:32 DEBUG { 20:19:32 DEBUG "status": "success", 20:19:32 DEBUG "data": { 20:19:32 DEBUG "resultType": "vector", 20:19:32 DEBUG "result": [ 20:19:32 DEBUG { 20:19:32 DEBUG "metric": { 20:19:32 DEBUG "pod": "ds-cts-0" 20:19:32 DEBUG }, 20:19:32 DEBUG "value": [ 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG [...] 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG "pod": "ds-cts-2" 20:19:32 DEBUG }, 20:19:32 DEBUG "value": [ 20:19:32 DEBUG 1713554372, 20:19:32 DEBUG "0" 20:19:32 DEBUG ] 20:19:32 DEBUG } 20:19:32 DEBUG ] 20:19:32 DEBUG } 20:19:32 DEBUG } 20:19:32 INFO [http_cmd]: http status code OK 20:19:32 DEBUG --- status code --- 20:19:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:32 DEBUG --- http response --- 20:19:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:19:32 DEBUG ----- output ----- 20:19:32 DEBUG { 20:19:32 DEBUG "status": "success", 20:19:32 DEBUG "data": { 20:19:32 DEBUG "resultType": "vector", 20:19:32 DEBUG "result": [ 20:19:32 DEBUG { 20:19:32 DEBUG "metric": { 20:19:32 DEBUG "pod": "ds-idrepo-0" 20:19:32 DEBUG }, 20:19:32 DEBUG "value": [ 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG [...] 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG "pod": "ds-idrepo-2" 20:19:32 DEBUG }, 20:19:32 DEBUG "value": [ 20:19:32 DEBUG 1713554372, 20:19:32 DEBUG "0" 20:19:32 DEBUG ] 20:19:32 DEBUG } 20:19:32 DEBUG ] 20:19:32 DEBUG } 20:19:32 DEBUG } 20:19:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713554372" 20:19:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713554372" 20:19:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554372" 20:19:32 INFO [http_cmd]: http status code OK 20:19:32 DEBUG --- status code --- 20:19:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:32 DEBUG --- http response --- 20:19:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:19:32 DEBUG ----- output ----- 20:19:32 DEBUG { 20:19:32 DEBUG "status": "success", 20:19:32 DEBUG "data": { 20:19:32 DEBUG "resultType": "vector", 20:19:32 DEBUG "result": [ 20:19:32 DEBUG { 20:19:32 DEBUG "metric": { 20:19:32 DEBUG "pod": "ds-idrepo-0" 20:19:32 DEBUG }, 20:19:32 DEBUG "value": [ 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG [...] 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG "pod": "ds-cts-2" 20:19:32 DEBUG }, 20:19:32 DEBUG "value": [ 20:19:32 DEBUG 1713554372, 20:19:32 DEBUG "0" 20:19:32 DEBUG ] 20:19:32 DEBUG } 20:19:32 DEBUG ] 20:19:32 DEBUG } 20:19:32 DEBUG } 20:19:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554372" 20:19:32 ERROR [http_cmd]: ERROR 20:19:32 INFO [http_cmd]: http status code OK 20:19:32 ERROR --- status code --- 20:19:32 DEBUG --- status code --- 20:19:32 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:19:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:32 ERROR --- http response --- 20:19:32 DEBUG --- http response --- 20:19:32 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:19:32 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:19:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:19:32 DEBUG ----- output ----- 20:19:32 DEBUG { 20:19:32 DEBUG "status": "success", 20:19:32 DEBUG "data": { 20:19:32 DEBUG "resultType": "vector", 20:19:32 DEBUG "result": [ 20:19:32 DEBUG { 20:19:32 DEBUG "metric": { 20:19:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:19:32 DEBUG }, 20:19:32 DEBUG "value": [ 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG [...] 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:19:32 DEBUG }, 20:19:32 DEBUG "value": [ 20:19:32 DEBUG 1713554372, 20:19:32 DEBUG "0" 20:19:32 DEBUG ] 20:19:32 DEBUG } 20:19:32 DEBUG ] 20:19:32 DEBUG } 20:19:32 DEBUG } 20:19:32 INFO [http_cmd]: http status code OK 20:19:32 DEBUG --- status code --- 20:19:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:32 DEBUG --- http response --- 20:19:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:19:32 DEBUG ----- output ----- 20:19:32 DEBUG { 20:19:32 DEBUG "status": "success", 20:19:32 DEBUG "data": { 20:19:32 DEBUG "resultType": "vector", 20:19:32 DEBUG "result": [ 20:19:32 DEBUG { 20:19:32 DEBUG "metric": { 20:19:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:19:32 DEBUG }, 20:19:32 DEBUG "value": [ 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG [...] 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:19:32 DEBUG }, 20:19:32 DEBUG "value": [ 20:19:32 DEBUG 1713554372, 20:19:32 DEBUG "2" 20:19:32 DEBUG ] 20:19:32 DEBUG } 20:19:32 DEBUG ] 20:19:32 DEBUG } 20:19:32 DEBUG } 20:19:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713554372" 20:19:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554372" 20:19:32 INFO [http_cmd]: http status code OK 20:19:32 DEBUG --- status code --- 20:19:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:32 DEBUG --- http response --- 20:19:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:19:32 DEBUG ----- output ----- 20:19:32 DEBUG { 20:19:32 DEBUG "status": "success", 20:19:32 DEBUG "data": { 20:19:32 DEBUG "resultType": "vector", 20:19:32 DEBUG "result": [ 20:19:32 DEBUG { 20:19:32 DEBUG "metric": { 20:19:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:19:32 DEBUG }, 20:19:32 DEBUG "value": [ 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG [...] 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:19:32 DEBUG }, 20:19:32 DEBUG "value": [ 20:19:32 DEBUG 1713554372, 20:19:32 DEBUG "0.015" 20:19:32 DEBUG ] 20:19:32 DEBUG } 20:19:32 DEBUG ] 20:19:32 DEBUG } 20:19:32 DEBUG } 20:19:32 INFO [http_cmd]: http status code OK 20:19:32 DEBUG --- status code --- 20:19:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:32 DEBUG --- http response --- 20:19:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:19:32 DEBUG ----- output ----- 20:19:32 DEBUG { 20:19:32 DEBUG "status": "success", 20:19:32 DEBUG "data": { 20:19:32 DEBUG "resultType": "vector", 20:19:32 DEBUG "result": [ 20:19:32 DEBUG { 20:19:32 DEBUG "metric": { 20:19:32 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:19:32 DEBUG }, 20:19:32 DEBUG "value": [ 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG [...] 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG "pod": "idm-b85f9d978-txpwv" 20:19:32 DEBUG }, 20:19:32 DEBUG "value": [ 20:19:32 DEBUG 1713554372, 20:19:32 DEBUG "498.7974548665023" 20:19:32 DEBUG ] 20:19:32 DEBUG } 20:19:32 DEBUG ] 20:19:32 DEBUG } 20:19:32 DEBUG } 20:19:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554372" 20:19:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554372" 20:19:32 INFO [http_cmd]: http status code OK 20:19:32 INFO [http_cmd]: http status code OK 20:19:32 DEBUG --- status code --- 20:19:32 DEBUG --- status code --- 20:19:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:32 DEBUG --- http response --- 20:19:32 DEBUG --- http response --- 20:19:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:19:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:19:32 DEBUG ----- output ----- 20:19:32 DEBUG ----- output ----- 20:19:32 DEBUG { 20:19:32 DEBUG { 20:19:32 DEBUG "status": "success", 20:19:32 DEBUG "status": "success", 20:19:32 DEBUG "data": { 20:19:32 DEBUG "data": { 20:19:32 DEBUG "resultType": "vector", 20:19:32 DEBUG "resultType": "vector", 20:19:32 DEBUG "result": [ 20:19:32 DEBUG "result": [ 20:19:32 DEBUG { 20:19:32 DEBUG { 20:19:32 DEBUG "metric": { 20:19:32 DEBUG "metric": { 20:19:32 DEBUG "instance": "10.142.0.107:9100" 20:19:32 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:19:32 DEBUG }, 20:19:32 DEBUG }, 20:19:32 DEBUG "value": [ 20:19:32 DEBUG "value": [ 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG [...] 20:19:32 DEBUG [...] 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG "instance": "10.142.0.43:9100" 20:19:32 DEBUG "pod": "idm-b85f9d978-txpwv" 20:19:32 DEBUG }, 20:19:32 DEBUG }, 20:19:32 DEBUG "value": [ 20:19:32 DEBUG "value": [ 20:19:32 DEBUG 1713554372, 20:19:32 DEBUG 1713554372, 20:19:32 DEBUG "0.004400000000000167" 20:19:32 DEBUG "37668186.19660852" 20:19:32 DEBUG ] 20:19:32 DEBUG ] 20:19:32 DEBUG } 20:19:32 DEBUG } 20:19:32 DEBUG ] 20:19:32 DEBUG ] 20:19:32 DEBUG } 20:19:32 DEBUG } 20:19:32 DEBUG } 20:19:32 DEBUG } 20:19:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713554372" 20:19:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554372" 20:19:32 INFO [http_cmd]: http status code OK 20:19:32 DEBUG --- status code --- 20:19:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:32 DEBUG --- http response --- 20:19:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:19:32 DEBUG ----- output ----- 20:19:32 DEBUG { 20:19:32 DEBUG "status": "success", 20:19:32 DEBUG "data": { 20:19:32 DEBUG "resultType": "vector", 20:19:32 DEBUG "result": [ 20:19:32 DEBUG { 20:19:32 DEBUG "metric": { 20:19:32 DEBUG "pod": "ds-idrepo-0" 20:19:32 DEBUG }, 20:19:32 DEBUG "value": [ 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG [...] 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG "pod": "ds-idrepo-2" 20:19:32 DEBUG }, 20:19:32 DEBUG "value": [ 20:19:32 DEBUG 1713554372, 20:19:32 DEBUG "0" 20:19:32 DEBUG ] 20:19:32 DEBUG } 20:19:32 DEBUG ] 20:19:32 DEBUG } 20:19:32 DEBUG } 20:19:32 INFO [http_cmd]: http status code OK 20:19:32 DEBUG --- status code --- 20:19:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:32 DEBUG --- http response --- 20:19:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:19:32 DEBUG ----- output ----- 20:19:32 DEBUG { 20:19:32 DEBUG "status": "success", 20:19:32 DEBUG "data": { 20:19:32 DEBUG "resultType": "vector", 20:19:32 DEBUG "result": [ 20:19:32 DEBUG { 20:19:32 DEBUG "metric": { 20:19:32 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:19:32 DEBUG }, 20:19:32 DEBUG "value": [ 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG [...] 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:19:32 DEBUG }, 20:19:32 DEBUG "value": [ 20:19:32 DEBUG 1713554372, 20:19:32 DEBUG "1.0328187354352552" 20:19:32 DEBUG ] 20:19:32 DEBUG } 20:19:32 DEBUG ] 20:19:32 DEBUG } 20:19:32 DEBUG } 20:19:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554372" 20:19:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554372" 20:19:32 INFO [http_cmd]: http status code OK 20:19:32 DEBUG --- status code --- 20:19:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:32 DEBUG --- http response --- 20:19:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:19:32 DEBUG ----- output ----- 20:19:32 DEBUG { 20:19:32 DEBUG "status": "success", 20:19:32 DEBUG "data": { 20:19:32 DEBUG "resultType": "vector", 20:19:32 DEBUG "result": [ 20:19:32 DEBUG { 20:19:32 DEBUG "metric": { 20:19:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:19:32 DEBUG }, 20:19:32 DEBUG "value": [ 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG [...] 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:19:32 DEBUG }, 20:19:32 DEBUG "value": [ 20:19:32 DEBUG 1713554372, 20:19:32 DEBUG "0" 20:19:32 DEBUG ] 20:19:32 DEBUG } 20:19:32 DEBUG ] 20:19:32 DEBUG } 20:19:32 DEBUG } 20:19:32 INFO [http_cmd]: http status code OK 20:19:32 DEBUG --- status code --- 20:19:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:32 DEBUG --- http response --- 20:19:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:19:32 DEBUG ----- output ----- 20:19:32 DEBUG { 20:19:32 DEBUG "status": "success", 20:19:32 DEBUG "data": { 20:19:32 DEBUG "resultType": "vector", 20:19:32 DEBUG "result": [ 20:19:32 DEBUG { 20:19:32 DEBUG "metric": { 20:19:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:19:32 DEBUG }, 20:19:32 DEBUG "value": [ 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG [...] 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:19:32 DEBUG }, 20:19:32 DEBUG "value": [ 20:19:32 DEBUG 1713554372, 20:19:32 DEBUG "0" 20:19:32 DEBUG ] 20:19:32 DEBUG } 20:19:32 DEBUG ] 20:19:32 DEBUG } 20:19:32 DEBUG } 20:19:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554372" 20:19:32 INFO [http_cmd]: http status code OK 20:19:32 DEBUG --- status code --- 20:19:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:32 DEBUG --- http response --- 20:19:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:19:32 DEBUG ----- output ----- 20:19:32 DEBUG { 20:19:32 DEBUG "status": "success", 20:19:32 DEBUG "data": { 20:19:32 DEBUG "resultType": "vector", 20:19:32 DEBUG "result": [ 20:19:32 DEBUG { 20:19:32 DEBUG "metric": { 20:19:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:19:32 DEBUG }, 20:19:32 DEBUG "value": [ 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG [...] 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:19:32 DEBUG }, 20:19:32 DEBUG "value": [ 20:19:32 DEBUG 1713554372, 20:19:32 DEBUG "0" 20:19:32 DEBUG ] 20:19:32 DEBUG } 20:19:32 DEBUG ] 20:19:32 DEBUG } 20:19:32 DEBUG } 20:19:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554372" 20:19:32 INFO [http_cmd]: http status code OK 20:19:32 DEBUG --- status code --- 20:19:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:32 DEBUG --- http response --- 20:19:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:19:32 DEBUG ----- output ----- 20:19:32 DEBUG { 20:19:32 DEBUG "status": "success", 20:19:32 DEBUG "data": { 20:19:32 DEBUG "resultType": "vector", 20:19:32 DEBUG "result": [ 20:19:32 DEBUG { 20:19:32 DEBUG "metric": { 20:19:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:19:32 DEBUG }, 20:19:32 DEBUG "value": [ 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG [...] 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:19:32 DEBUG }, 20:19:32 DEBUG "value": [ 20:19:32 DEBUG 1713554372, 20:19:32 DEBUG "0" 20:19:32 DEBUG ] 20:19:32 DEBUG } 20:19:32 DEBUG ] 20:19:32 DEBUG } 20:19:32 DEBUG } 20:19:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554372" 20:19:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554372" 20:19:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554372" 20:19:32 INFO [http_cmd]: http status code OK 20:19:32 INFO [http_cmd]: http status code OK 20:19:32 DEBUG --- status code --- 20:19:32 DEBUG --- status code --- 20:19:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:32 DEBUG --- http response --- 20:19:32 DEBUG --- http response --- 20:19:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:19:32 DEBUG ----- output ----- 20:19:32 DEBUG { 20:19:32 DEBUG "status": "success", 20:19:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:19:32 DEBUG "data": { 20:19:32 DEBUG ----- output ----- 20:19:32 DEBUG "resultType": "vector", 20:19:32 DEBUG { 20:19:32 DEBUG "result": [ 20:19:32 DEBUG "status": "success", 20:19:32 DEBUG { 20:19:32 DEBUG "data": { 20:19:32 DEBUG "metric": { 20:19:32 DEBUG "resultType": "vector", 20:19:32 DEBUG "instance": "10.142.0.107:9100" 20:19:32 DEBUG "result": [ 20:19:32 DEBUG }, 20:19:32 DEBUG { 20:19:32 DEBUG "value": [ 20:19:32 DEBUG "metric": { 20:19:32 DEBUG 20:19:32 DEBUG "instance": "10.142.0.107:9100" 20:19:32 DEBUG 20:19:32 DEBUG }, 20:19:32 DEBUG 20:19:32 DEBUG "value": [ 20:19:32 DEBUG [...] 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 INFO [http_cmd]: http status code OK 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG --- status code --- 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:32 DEBUG [...] 20:19:32 DEBUG "instance": "10.142.0.43:9100" 20:19:32 DEBUG --- http response --- 20:19:32 DEBUG 20:19:32 DEBUG }, 20:19:32 DEBUG 20:19:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:19:32 DEBUG "value": [ 20:19:32 DEBUG 20:19:32 DEBUG ----- output ----- 20:19:32 DEBUG 1713554372, 20:19:32 DEBUG "instance": "10.142.0.43:9100" 20:19:32 DEBUG { 20:19:32 DEBUG "26542.9" 20:19:32 DEBUG }, 20:19:32 DEBUG "status": "success", 20:19:32 DEBUG ] 20:19:32 DEBUG "value": [ 20:19:32 DEBUG "data": { 20:19:32 DEBUG } 20:19:32 DEBUG 1713554372, 20:19:32 DEBUG "resultType": "vector", 20:19:32 DEBUG ] 20:19:32 DEBUG "2063523601408" 20:19:32 DEBUG "result": [ 20:19:32 DEBUG } 20:19:32 DEBUG ] 20:19:32 DEBUG { 20:19:32 DEBUG } 20:19:32 DEBUG } 20:19:32 DEBUG "metric": { 20:19:32 DEBUG ] 20:19:32 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:19:32 DEBUG } 20:19:32 DEBUG }, 20:19:32 DEBUG } 20:19:32 DEBUG "value": [ 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG [...] 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG "pod": "idm-b85f9d978-txpwv" 20:19:32 DEBUG }, 20:19:32 DEBUG "value": [ 20:19:32 DEBUG 1713554372, 20:19:32 DEBUG "0.0033681919201145323" 20:19:32 DEBUG ] 20:19:32 DEBUG } 20:19:32 DEBUG ] 20:19:32 DEBUG } 20:19:32 DEBUG } 20:19:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713554372" 20:19:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554372" 20:19:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554372" 20:19:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554372" 20:19:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554372" 20:19:32 INFO [http_cmd]: http status code OK 20:19:32 DEBUG --- status code --- 20:19:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:32 DEBUG --- http response --- 20:19:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:19:32 DEBUG ----- output ----- 20:19:32 DEBUG { 20:19:32 DEBUG "status": "success", 20:19:32 DEBUG "data": { 20:19:32 DEBUG "resultType": "vector", 20:19:32 DEBUG "result": [ 20:19:32 DEBUG { 20:19:32 DEBUG "metric": { 20:19:32 DEBUG "pod": "ds-cts-0" 20:19:32 DEBUG }, 20:19:32 DEBUG "value": [ 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG [...] 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG "pod": "ds-cts-2" 20:19:32 DEBUG }, 20:19:32 DEBUG "value": [ 20:19:32 DEBUG 1713554372, 20:19:32 DEBUG "29.933333333333334" 20:19:32 DEBUG ] 20:19:32 DEBUG } 20:19:32 DEBUG ] 20:19:32 DEBUG } 20:19:32 DEBUG } 20:19:32 INFO [http_cmd]: http status code OK 20:19:32 DEBUG --- status code --- 20:19:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:32 DEBUG --- http response --- 20:19:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:19:32 DEBUG ----- output ----- 20:19:32 DEBUG { 20:19:32 DEBUG "status": "success", 20:19:32 DEBUG "data": { 20:19:32 DEBUG "resultType": "vector", 20:19:32 DEBUG "result": [ 20:19:32 DEBUG { 20:19:32 DEBUG "metric": { 20:19:32 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:19:32 DEBUG }, 20:19:32 DEBUG "value": [ 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG [...] 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:19:32 DEBUG }, 20:19:32 DEBUG "value": [ 20:19:32 DEBUG 1713554372, 20:19:32 DEBUG "4231782400" 20:19:32 DEBUG ] 20:19:32 DEBUG } 20:19:32 DEBUG ] 20:19:32 DEBUG } 20:19:32 DEBUG } 20:19:32 INFO [http_cmd]: http status code OK 20:19:32 DEBUG --- status code --- 20:19:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:32 DEBUG --- http response --- 20:19:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:19:32 DEBUG ----- output ----- 20:19:32 DEBUG { 20:19:32 DEBUG "status": "success", 20:19:32 INFO [http_cmd]: http status code OK 20:19:32 DEBUG "data": { 20:19:32 DEBUG --- status code --- 20:19:32 DEBUG "resultType": "vector", 20:19:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:32 DEBUG "result": [ 20:19:32 DEBUG --- http response --- 20:19:32 DEBUG { 20:19:32 DEBUG "metric": { 20:19:32 DEBUG "pod": "ds-cts-0" 20:19:32 DEBUG }, 20:19:32 DEBUG "value": [ 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG [...] 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG "pod": "ds-cts-2" 20:19:32 DEBUG }, 20:19:32 DEBUG "value": [ 20:19:32 DEBUG 1713554372, 20:19:32 DEBUG "0" 20:19:32 DEBUG ] 20:19:32 DEBUG } 20:19:32 DEBUG ] 20:19:32 DEBUG } 20:19:32 DEBUG } 20:19:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:19:32 DEBUG ----- output ----- 20:19:32 DEBUG { 20:19:32 DEBUG "status": "success", 20:19:32 DEBUG "data": { 20:19:32 DEBUG "resultType": "vector", 20:19:32 DEBUG "result": [ 20:19:32 DEBUG { 20:19:32 DEBUG "metric": { 20:19:32 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:19:32 DEBUG }, 20:19:32 DEBUG "value": [ 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG [...] 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG "pod": "idm-b85f9d978-txpwv" 20:19:32 DEBUG }, 20:19:32 DEBUG "value": [ 20:19:32 DEBUG 1713554372, 20:19:32 DEBUG "0" 20:19:32 DEBUG ] 20:19:32 DEBUG } 20:19:32 DEBUG ] 20:19:32 DEBUG } 20:19:32 DEBUG } 20:19:32 INFO [http_cmd]: http status code OK 20:19:32 DEBUG --- status code --- 20:19:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:32 DEBUG --- http response --- 20:19:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:19:32 DEBUG ----- output ----- 20:19:32 DEBUG { 20:19:32 DEBUG "status": "success", 20:19:32 DEBUG "data": { 20:19:32 DEBUG "resultType": "vector", 20:19:32 DEBUG "result": [ 20:19:32 DEBUG { 20:19:32 DEBUG "metric": { 20:19:32 DEBUG "instance": "10.142.0.107:9100" 20:19:32 DEBUG }, 20:19:32 DEBUG "value": [ 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG [...] 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG 20:19:32 DEBUG "instance": "10.142.0.43:9100" 20:19:32 DEBUG }, 20:19:32 DEBUG "value": [ 20:19:32 DEBUG 1713554372, 20:19:32 DEBUG "27084.233333333334" 20:19:32 DEBUG ] 20:19:32 DEBUG } 20:19:32 DEBUG ] 20:19:32 DEBUG } 20:19:32 DEBUG } 20:19:34 INFO 20:19:34 INFO [loop_until]: kubectl --namespace=xlou top pods 20:19:34 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:19:34 INFO [loop_until]: OK (rc = 0) 20:19:34 DEBUG --- stdout --- 20:19:34 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 7m 8965Mi am-767f6b67b5-qcqz2 8m 8759Mi am-767f6b67b5-vbltz 8m 8954Mi ds-cts-0 18m 11008Mi ds-cts-1 16m 11270Mi ds-cts-2 18m 11040Mi ds-idrepo-0 15m 3715Mi ds-idrepo-1 12m 3559Mi ds-idrepo-2 14m 3492Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 4m 1217Mi lodemon-f8459c597-6tj6f 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 5Mi overseer-0-5c67bf94db-77mh9 1007m 4040Mi 20:19:34 DEBUG --- stderr --- 20:19:34 DEBUG 20:19:36 INFO 20:19:36 INFO [loop_until]: kubectl --namespace=xlou top node 20:19:36 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:19:36 INFO [loop_until]: OK (rc = 0) 20:19:36 DEBUG --- stdout --- 20:19:36 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 72m 0% 6245Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 77m 0% 7304Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 60m 0% 11695Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 86m 0% 3362Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 75m 0% 14000Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 143m 0% 5165Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 64m 0% 11655Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 79m 0% 2950Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 60m 0% 11190Mi 10% gke-xlou-cdm-ds-0f601751-j488 72m 0% 6728Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 71m 0% 16009Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 95m 0% 7441Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 68m 0% 13477Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 1074m 6% 7867Mi 13% 20:19:36 DEBUG --- stderr --- 20:19:36 DEBUG 20:20:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554432" 20:20:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554432" 20:20:32 INFO [http_cmd]: http status code OK 20:20:32 DEBUG --- status code --- 20:20:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:32 DEBUG --- http response --- 20:20:32 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:20:32 INFO [http_cmd]: http status code OK 20:20:32 DEBUG --- status code --- 20:20:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:32 DEBUG --- http response --- 20:20:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:20:32 DEBUG ----- output ----- 20:20:32 DEBUG { 20:20:32 DEBUG "status": "success", 20:20:32 DEBUG "data": { 20:20:32 DEBUG "resultType": "vector", 20:20:32 DEBUG "result": [ 20:20:32 DEBUG { 20:20:32 DEBUG "metric": { 20:20:32 DEBUG "instance": "10.142.0.107:9100" 20:20:32 DEBUG }, 20:20:32 DEBUG "value": [ 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG [...] 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG "instance": "10.142.0.43:9100" 20:20:32 DEBUG }, 20:20:32 DEBUG "value": [ 20:20:32 DEBUG 1713554432, 20:20:32 DEBUG "0.0026666666666699257" 20:20:32 DEBUG ] 20:20:32 DEBUG } 20:20:32 DEBUG ] 20:20:32 DEBUG } 20:20:32 DEBUG } 20:20:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713554432" 20:20:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554432" 20:20:32 ERROR [http_cmd]: ERROR 20:20:32 INFO [http_cmd]: http status code OK 20:20:32 ERROR --- status code --- 20:20:32 DEBUG --- status code --- 20:20:32 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:20:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554432" 20:20:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:32 ERROR --- http response --- 20:20:32 DEBUG --- http response --- 20:20:32 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:20:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:20:32 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:20:32 DEBUG ----- output ----- 20:20:32 DEBUG { 20:20:32 DEBUG "status": "success", 20:20:32 DEBUG "data": { 20:20:32 DEBUG "resultType": "vector", 20:20:32 DEBUG "result": [ 20:20:32 DEBUG { 20:20:32 DEBUG "metric": { 20:20:32 DEBUG "pod": "ds-idrepo-0" 20:20:32 DEBUG }, 20:20:32 DEBUG "value": [ 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG [...] 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG "pod": "ds-idrepo-2" 20:20:32 DEBUG }, 20:20:32 DEBUG "value": [ 20:20:32 DEBUG 1713554432, 20:20:32 DEBUG "0" 20:20:32 DEBUG ] 20:20:32 DEBUG } 20:20:32 DEBUG ] 20:20:32 DEBUG } 20:20:32 DEBUG } 20:20:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554432" 20:20:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713554432" 20:20:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554432" 20:20:32 INFO [http_cmd]: http status code OK 20:20:32 INFO [http_cmd]: http status code OK 20:20:32 DEBUG --- status code --- 20:20:32 DEBUG --- status code --- 20:20:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:32 DEBUG --- http response --- 20:20:32 DEBUG --- http response --- 20:20:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:20:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:20:32 DEBUG ----- output ----- 20:20:32 DEBUG ----- output ----- 20:20:32 DEBUG { 20:20:32 DEBUG { 20:20:32 DEBUG "status": "success", 20:20:32 DEBUG "status": "success", 20:20:32 DEBUG "data": { 20:20:32 DEBUG "data": { 20:20:32 DEBUG "resultType": "vector", 20:20:32 DEBUG "resultType": "vector", 20:20:32 DEBUG "result": [ 20:20:32 DEBUG "result": [ 20:20:32 DEBUG { 20:20:32 DEBUG { 20:20:32 DEBUG "metric": { 20:20:32 DEBUG "metric": { 20:20:32 DEBUG "pod": "ds-idrepo-0" 20:20:32 DEBUG "pod": "ds-cts-0" 20:20:32 DEBUG }, 20:20:32 DEBUG }, 20:20:32 DEBUG "value": [ 20:20:32 DEBUG "value": [ 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG [...] 20:20:32 DEBUG [...] 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG "pod": "ds-cts-2" 20:20:32 DEBUG "pod": "ds-cts-2" 20:20:32 DEBUG }, 20:20:32 DEBUG }, 20:20:32 DEBUG "value": [ 20:20:32 DEBUG "value": [ 20:20:32 DEBUG 1713554432, 20:20:32 DEBUG 1713554432, 20:20:32 DEBUG "0" 20:20:32 DEBUG "0" 20:20:32 DEBUG ] 20:20:32 DEBUG ] 20:20:32 DEBUG } 20:20:32 DEBUG } 20:20:32 DEBUG ] 20:20:32 DEBUG ] 20:20:32 DEBUG } 20:20:32 DEBUG } 20:20:32 DEBUG } 20:20:32 DEBUG } 20:20:32 INFO [http_cmd]: http status code OK 20:20:32 DEBUG --- status code --- 20:20:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:32 DEBUG --- http response --- 20:20:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:20:32 DEBUG ----- output ----- 20:20:32 DEBUG { 20:20:32 DEBUG "status": "success", 20:20:32 DEBUG "data": { 20:20:32 DEBUG "resultType": "vector", 20:20:32 DEBUG "result": [ 20:20:32 DEBUG { 20:20:32 DEBUG "metric": { 20:20:32 DEBUG "pod": "ds-idrepo-0" 20:20:32 DEBUG }, 20:20:32 DEBUG "value": [ 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG [...] 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG "pod": "ds-idrepo-2" 20:20:32 DEBUG }, 20:20:32 DEBUG "value": [ 20:20:32 DEBUG 1713554432, 20:20:32 DEBUG "0" 20:20:32 DEBUG ] 20:20:32 DEBUG } 20:20:32 DEBUG ] 20:20:32 DEBUG } 20:20:32 DEBUG } 20:20:32 INFO [http_cmd]: http status code OK 20:20:32 DEBUG --- status code --- 20:20:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:32 DEBUG --- http response --- 20:20:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:20:32 DEBUG ----- output ----- 20:20:32 DEBUG { 20:20:32 DEBUG "status": "success", 20:20:32 DEBUG "data": { 20:20:32 DEBUG "resultType": "vector", 20:20:32 DEBUG "result": [ 20:20:32 DEBUG { 20:20:32 DEBUG "metric": { 20:20:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:20:32 DEBUG }, 20:20:32 DEBUG "value": [ 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG [...] 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:20:32 DEBUG }, 20:20:32 DEBUG "value": [ 20:20:32 DEBUG 1713554432, 20:20:32 DEBUG "0" 20:20:32 DEBUG ] 20:20:32 DEBUG } 20:20:32 DEBUG ] 20:20:32 DEBUG } 20:20:32 DEBUG } 20:20:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554432" 20:20:32 INFO [http_cmd]: http status code OK 20:20:32 DEBUG --- status code --- 20:20:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:32 DEBUG --- http response --- 20:20:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:20:32 DEBUG ----- output ----- 20:20:32 DEBUG { 20:20:32 DEBUG "status": "success", 20:20:32 DEBUG "data": { 20:20:32 DEBUG "resultType": "vector", 20:20:32 DEBUG "result": [ 20:20:32 DEBUG { 20:20:32 DEBUG "metric": { 20:20:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:20:32 DEBUG }, 20:20:32 DEBUG "value": [ 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG [...] 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:20:32 DEBUG }, 20:20:32 DEBUG "value": [ 20:20:32 DEBUG 1713554432, 20:20:32 DEBUG "0" 20:20:32 DEBUG ] 20:20:32 DEBUG } 20:20:32 DEBUG ] 20:20:32 DEBUG } 20:20:32 DEBUG } 20:20:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713554432" 20:20:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554432" 20:20:32 INFO [http_cmd]: http status code OK 20:20:32 DEBUG --- status code --- 20:20:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:32 DEBUG --- http response --- 20:20:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:20:32 DEBUG ----- output ----- 20:20:32 DEBUG { 20:20:32 DEBUG "status": "success", 20:20:32 DEBUG "data": { 20:20:32 DEBUG "resultType": "vector", 20:20:32 DEBUG "result": [ 20:20:32 DEBUG { 20:20:32 DEBUG "metric": { 20:20:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:20:32 DEBUG }, 20:20:32 DEBUG "value": [ 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG [...] 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:20:32 DEBUG }, 20:20:32 DEBUG "value": [ 20:20:32 DEBUG 1713554432, 20:20:32 DEBUG "0.016" 20:20:32 DEBUG ] 20:20:32 DEBUG } 20:20:32 DEBUG ] 20:20:32 DEBUG } 20:20:32 DEBUG } 20:20:32 INFO [http_cmd]: http status code OK 20:20:32 DEBUG --- status code --- 20:20:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:32 DEBUG --- http response --- 20:20:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 20:20:32 DEBUG ----- output ----- 20:20:32 DEBUG { 20:20:32 DEBUG "status": "success", 20:20:32 DEBUG "data": { 20:20:32 DEBUG "resultType": "vector", 20:20:32 DEBUG "result": [ 20:20:32 DEBUG { 20:20:32 DEBUG "metric": { 20:20:32 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:20:32 DEBUG }, 20:20:32 DEBUG "value": [ 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG [...] 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG "pod": "ds-idrepo-2" 20:20:32 DEBUG }, 20:20:32 DEBUG "value": [ 20:20:32 DEBUG 1713554432, 20:20:32 DEBUG "3046.3459893269346" 20:20:32 DEBUG ] 20:20:32 DEBUG } 20:20:32 DEBUG ] 20:20:32 DEBUG } 20:20:32 DEBUG } 20:20:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554432" 20:20:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554432" 20:20:32 INFO [http_cmd]: http status code OK 20:20:32 INFO [http_cmd]: http status code OK 20:20:32 DEBUG --- status code --- 20:20:32 DEBUG --- status code --- 20:20:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:32 DEBUG --- http response --- 20:20:32 DEBUG --- http response --- 20:20:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:20:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:20:32 DEBUG ----- output ----- 20:20:32 DEBUG ----- output ----- 20:20:32 DEBUG { 20:20:32 DEBUG { 20:20:32 DEBUG "status": "success", 20:20:32 DEBUG "status": "success", 20:20:32 DEBUG "data": { 20:20:32 DEBUG "data": { 20:20:32 DEBUG "resultType": "vector", 20:20:32 DEBUG "resultType": "vector", 20:20:32 DEBUG "result": [ 20:20:32 DEBUG "result": [ 20:20:32 DEBUG { 20:20:32 DEBUG { 20:20:32 DEBUG "metric": { 20:20:32 DEBUG "metric": { 20:20:32 DEBUG "instance": "10.142.0.107:9100" 20:20:32 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:20:32 DEBUG }, 20:20:32 DEBUG }, 20:20:32 DEBUG "value": [ 20:20:32 DEBUG "value": [ 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG [...] 20:20:32 DEBUG [...] 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG "instance": "10.142.0.43:9100" 20:20:32 DEBUG "pod": "ds-idrepo-2" 20:20:32 DEBUG }, 20:20:32 DEBUG }, 20:20:32 DEBUG "value": [ 20:20:32 DEBUG "value": [ 20:20:32 DEBUG 1713554432, 20:20:32 DEBUG 1713554432, 20:20:32 DEBUG "0.0038999999999930423" 20:20:32 DEBUG "40422.66793529971" 20:20:32 DEBUG ] 20:20:32 DEBUG ] 20:20:32 DEBUG } 20:20:32 DEBUG } 20:20:32 DEBUG ] 20:20:32 DEBUG ] 20:20:32 DEBUG } 20:20:32 DEBUG } 20:20:32 DEBUG } 20:20:32 DEBUG } 20:20:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554432" 20:20:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713554432" 20:20:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554432" 20:20:32 INFO [http_cmd]: http status code OK 20:20:32 DEBUG --- status code --- 20:20:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:32 DEBUG --- http response --- 20:20:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:20:32 DEBUG ----- output ----- 20:20:32 DEBUG { 20:20:32 DEBUG "status": "success", 20:20:32 DEBUG "data": { 20:20:32 DEBUG "resultType": "vector", 20:20:32 DEBUG "result": [ 20:20:32 DEBUG { 20:20:32 DEBUG "metric": { 20:20:32 DEBUG "pod": "ds-idrepo-0" 20:20:32 DEBUG }, 20:20:32 DEBUG "value": [ 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG [...] 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG "pod": "ds-idrepo-2" 20:20:32 DEBUG }, 20:20:32 DEBUG "value": [ 20:20:32 DEBUG 1713554432, 20:20:32 DEBUG "0" 20:20:32 DEBUG ] 20:20:32 DEBUG } 20:20:32 DEBUG ] 20:20:32 DEBUG } 20:20:32 DEBUG } 20:20:32 INFO [http_cmd]: http status code OK 20:20:32 DEBUG --- status code --- 20:20:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:32 DEBUG --- http response --- 20:20:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:20:32 DEBUG ----- output ----- 20:20:32 DEBUG { 20:20:32 DEBUG "status": "success", 20:20:32 DEBUG "data": { 20:20:32 DEBUG "resultType": "vector", 20:20:32 DEBUG "result": [ 20:20:32 DEBUG { 20:20:32 DEBUG "metric": { 20:20:32 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:20:32 DEBUG }, 20:20:32 DEBUG "value": [ 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG [...] 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:20:32 DEBUG }, 20:20:32 DEBUG "value": [ 20:20:32 DEBUG 1713554432, 20:20:32 DEBUG "1.0323497787126035" 20:20:32 DEBUG ] 20:20:32 DEBUG } 20:20:32 DEBUG ] 20:20:32 DEBUG } 20:20:32 DEBUG } 20:20:32 INFO [http_cmd]: http status code OK 20:20:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554432" 20:20:32 DEBUG --- status code --- 20:20:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:32 DEBUG --- http response --- 20:20:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:20:32 DEBUG ----- output ----- 20:20:32 DEBUG { 20:20:32 DEBUG "status": "success", 20:20:32 DEBUG "data": { 20:20:32 DEBUG "resultType": "vector", 20:20:32 DEBUG "result": [ 20:20:32 DEBUG { 20:20:32 DEBUG "metric": { 20:20:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:20:32 DEBUG }, 20:20:32 DEBUG "value": [ 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG [...] 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:20:32 DEBUG }, 20:20:32 DEBUG "value": [ 20:20:32 DEBUG 1713554432, 20:20:32 DEBUG "0" 20:20:32 DEBUG ] 20:20:32 DEBUG } 20:20:32 DEBUG ] 20:20:32 DEBUG } 20:20:32 DEBUG } 20:20:32 INFO [http_cmd]: http status code OK 20:20:32 DEBUG --- status code --- 20:20:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:32 DEBUG --- http response --- 20:20:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:20:32 DEBUG ----- output ----- 20:20:32 DEBUG { 20:20:32 DEBUG "status": "success", 20:20:32 DEBUG "data": { 20:20:32 DEBUG "resultType": "vector", 20:20:32 DEBUG "result": [ 20:20:32 DEBUG { 20:20:32 DEBUG "metric": { 20:20:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:20:32 DEBUG }, 20:20:32 DEBUG "value": [ 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG [...] 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:20:32 DEBUG }, 20:20:32 DEBUG "value": [ 20:20:32 DEBUG 1713554432, 20:20:32 DEBUG "0" 20:20:32 DEBUG ] 20:20:32 DEBUG } 20:20:32 DEBUG ] 20:20:32 DEBUG } 20:20:32 DEBUG } 20:20:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554432" 20:20:32 INFO [http_cmd]: http status code OK 20:20:32 DEBUG --- status code --- 20:20:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:32 DEBUG --- http response --- 20:20:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:20:32 DEBUG ----- output ----- 20:20:32 DEBUG { 20:20:32 DEBUG "status": "success", 20:20:32 DEBUG "data": { 20:20:32 DEBUG "resultType": "vector", 20:20:32 DEBUG "result": [ 20:20:32 DEBUG { 20:20:32 DEBUG "metric": { 20:20:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:20:32 DEBUG }, 20:20:32 DEBUG "value": [ 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG [...] 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:20:32 DEBUG }, 20:20:32 DEBUG "value": [ 20:20:32 DEBUG 1713554432, 20:20:32 DEBUG "0" 20:20:32 DEBUG ] 20:20:32 DEBUG } 20:20:32 DEBUG ] 20:20:32 DEBUG } 20:20:32 DEBUG } 20:20:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554432" 20:20:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554432" 20:20:32 INFO [http_cmd]: http status code OK 20:20:32 DEBUG --- status code --- 20:20:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:32 DEBUG --- http response --- 20:20:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:20:32 DEBUG ----- output ----- 20:20:32 DEBUG { 20:20:32 DEBUG "status": "success", 20:20:32 DEBUG "data": { 20:20:32 DEBUG "resultType": "vector", 20:20:32 DEBUG "result": [ 20:20:32 DEBUG { 20:20:32 DEBUG "metric": { 20:20:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:20:32 DEBUG }, 20:20:32 DEBUG "value": [ 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG [...] 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:20:32 DEBUG }, 20:20:32 DEBUG "value": [ 20:20:32 DEBUG 1713554432, 20:20:32 DEBUG "0" 20:20:32 DEBUG ] 20:20:32 DEBUG } 20:20:32 DEBUG ] 20:20:32 DEBUG } 20:20:32 DEBUG } 20:20:32 INFO [http_cmd]: http status code OK 20:20:32 DEBUG --- status code --- 20:20:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:32 DEBUG --- http response --- 20:20:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:20:32 DEBUG ----- output ----- 20:20:32 DEBUG { 20:20:32 DEBUG "status": "success", 20:20:32 DEBUG "data": { 20:20:32 DEBUG "resultType": "vector", 20:20:32 DEBUG "result": [ 20:20:32 DEBUG { 20:20:32 DEBUG "metric": { 20:20:32 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:20:32 DEBUG }, 20:20:32 DEBUG "value": [ 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG [...] 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG "pod": "ds-idrepo-2" 20:20:32 DEBUG }, 20:20:32 DEBUG "value": [ 20:20:32 DEBUG 1713554432, 20:20:32 DEBUG "0.027993080855551567" 20:20:32 DEBUG ] 20:20:32 DEBUG } 20:20:32 DEBUG ] 20:20:32 DEBUG } 20:20:32 DEBUG } 20:20:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554432" 20:20:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554432" 20:20:32 INFO [http_cmd]: http status code OK 20:20:32 DEBUG --- status code --- 20:20:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:32 DEBUG --- http response --- 20:20:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:20:32 DEBUG ----- output ----- 20:20:32 DEBUG { 20:20:32 DEBUG "status": "success", 20:20:32 DEBUG "data": { 20:20:32 DEBUG "resultType": "vector", 20:20:32 DEBUG "result": [ 20:20:32 DEBUG { 20:20:32 DEBUG "metric": { 20:20:32 DEBUG "instance": "10.142.0.107:9100" 20:20:32 DEBUG }, 20:20:32 DEBUG "value": [ 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG [...] 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG "instance": "10.142.0.43:9100" 20:20:32 DEBUG }, 20:20:32 DEBUG "value": [ 20:20:32 DEBUG 1713554432, 20:20:32 DEBUG "26397.666666666664" 20:20:32 DEBUG ] 20:20:32 DEBUG } 20:20:32 DEBUG ] 20:20:32 DEBUG } 20:20:32 DEBUG } 20:20:32 INFO [http_cmd]: http status code OK 20:20:32 DEBUG --- status code --- 20:20:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:32 DEBUG --- http response --- 20:20:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 20:20:32 DEBUG ----- output ----- 20:20:32 DEBUG { 20:20:32 DEBUG "status": "success", 20:20:32 DEBUG "data": { 20:20:32 DEBUG "resultType": "vector", 20:20:32 DEBUG "result": [ 20:20:32 DEBUG { 20:20:32 DEBUG "metric": { 20:20:32 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:20:32 DEBUG }, 20:20:32 DEBUG "value": [ 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG [...] 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG "pod": "ds-idrepo-2" 20:20:32 DEBUG }, 20:20:32 DEBUG "value": [ 20:20:32 DEBUG 1713554432, 20:20:32 DEBUG "0" 20:20:32 DEBUG ] 20:20:32 DEBUG } 20:20:32 DEBUG ] 20:20:32 DEBUG } 20:20:32 DEBUG } 20:20:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554432" 20:20:32 INFO [http_cmd]: http status code OK 20:20:32 DEBUG --- status code --- 20:20:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:32 DEBUG --- http response --- 20:20:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:20:32 DEBUG ----- output ----- 20:20:32 DEBUG { 20:20:32 DEBUG "status": "success", 20:20:32 DEBUG "data": { 20:20:32 DEBUG "resultType": "vector", 20:20:32 DEBUG "result": [ 20:20:32 DEBUG { 20:20:32 DEBUG "metric": { 20:20:32 DEBUG "instance": "10.142.0.107:9100" 20:20:32 DEBUG }, 20:20:32 DEBUG "value": [ 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG [...] 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG "instance": "10.142.0.43:9100" 20:20:32 DEBUG }, 20:20:32 DEBUG "value": [ 20:20:32 DEBUG 1713554432, 20:20:32 DEBUG "2063532878848" 20:20:32 DEBUG ] 20:20:32 DEBUG } 20:20:32 DEBUG ] 20:20:32 DEBUG } 20:20:32 DEBUG } 20:20:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713554432" 20:20:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554432" 20:20:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554432" 20:20:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554432" 20:20:32 INFO [http_cmd]: http status code OK 20:20:32 DEBUG --- status code --- 20:20:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:32 DEBUG --- http response --- 20:20:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:20:32 DEBUG ----- output ----- 20:20:32 DEBUG { 20:20:32 DEBUG "status": "success", 20:20:32 DEBUG "data": { 20:20:32 DEBUG "resultType": "vector", 20:20:32 DEBUG "result": [ 20:20:32 DEBUG { 20:20:32 DEBUG "metric": { 20:20:32 DEBUG "pod": "ds-cts-0" 20:20:32 DEBUG }, 20:20:32 DEBUG "value": [ 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG [...] 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG "pod": "ds-cts-2" 20:20:32 DEBUG }, 20:20:32 DEBUG "value": [ 20:20:32 DEBUG 1713554432, 20:20:32 DEBUG "29.933333333333334" 20:20:32 DEBUG ] 20:20:32 DEBUG } 20:20:32 DEBUG ] 20:20:32 DEBUG } 20:20:32 DEBUG } 20:20:32 INFO [http_cmd]: http status code OK 20:20:32 DEBUG --- status code --- 20:20:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:32 DEBUG --- http response --- 20:20:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:20:32 DEBUG ----- output ----- 20:20:32 DEBUG { 20:20:32 DEBUG "status": "success", 20:20:32 DEBUG "data": { 20:20:32 DEBUG "resultType": "vector", 20:20:32 DEBUG "result": [ 20:20:32 DEBUG { 20:20:32 DEBUG "metric": { 20:20:32 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:20:32 DEBUG }, 20:20:32 DEBUG "value": [ 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG [...] 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:20:32 DEBUG }, 20:20:32 DEBUG "value": [ 20:20:32 DEBUG 1713554432, 20:20:32 DEBUG "4630097920" 20:20:32 DEBUG ] 20:20:32 DEBUG } 20:20:32 DEBUG ] 20:20:32 DEBUG } 20:20:32 DEBUG } 20:20:32 INFO [http_cmd]: http status code OK 20:20:32 INFO [http_cmd]: http status code OK 20:20:32 DEBUG --- status code --- 20:20:32 DEBUG --- status code --- 20:20:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:32 DEBUG --- http response --- 20:20:32 DEBUG --- http response --- 20:20:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:20:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:20:32 DEBUG ----- output ----- 20:20:32 DEBUG ----- output ----- 20:20:32 DEBUG { 20:20:32 DEBUG { 20:20:32 DEBUG "status": "success", 20:20:32 DEBUG "status": "success", 20:20:32 DEBUG "data": { 20:20:32 DEBUG "data": { 20:20:32 DEBUG "resultType": "vector", 20:20:32 DEBUG "resultType": "vector", 20:20:32 DEBUG "result": [ 20:20:32 DEBUG "result": [ 20:20:32 DEBUG { 20:20:32 DEBUG { 20:20:32 DEBUG "metric": { 20:20:32 DEBUG "metric": { 20:20:32 DEBUG "instance": "10.142.0.107:9100" 20:20:32 DEBUG "pod": "ds-cts-0" 20:20:32 DEBUG }, 20:20:32 DEBUG }, 20:20:32 DEBUG "value": [ 20:20:32 DEBUG "value": [ 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG [...] 20:20:32 DEBUG [...] 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG 20:20:32 DEBUG "instance": "10.142.0.43:9100" 20:20:32 DEBUG "pod": "ds-cts-2" 20:20:32 DEBUG }, 20:20:32 DEBUG }, 20:20:32 DEBUG "value": [ 20:20:32 DEBUG "value": [ 20:20:32 DEBUG 1713554432, 20:20:32 DEBUG 1713554432, 20:20:32 DEBUG "26766.699999999997" 20:20:32 DEBUG "0" 20:20:32 DEBUG ] 20:20:32 DEBUG ] 20:20:32 DEBUG } 20:20:32 DEBUG } 20:20:32 DEBUG ] 20:20:32 DEBUG ] 20:20:32 DEBUG } 20:20:32 DEBUG } 20:20:32 DEBUG } 20:20:32 DEBUG } 20:20:34 INFO 20:20:34 INFO [loop_until]: kubectl --namespace=xlou top pods 20:20:34 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:20:34 INFO [loop_until]: OK (rc = 0) 20:20:34 DEBUG --- stdout --- 20:20:34 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 7m 8952Mi am-767f6b67b5-qcqz2 6m 8750Mi am-767f6b67b5-vbltz 7m 8939Mi ds-cts-0 17m 11006Mi ds-cts-1 17m 11271Mi ds-cts-2 16m 11040Mi ds-idrepo-0 15m 3716Mi ds-idrepo-1 14m 3560Mi ds-idrepo-2 32m 3492Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 982m 4509Mi 20:20:34 DEBUG --- stderr --- 20:20:34 DEBUG 20:20:36 INFO 20:20:36 INFO [loop_until]: kubectl --namespace=xlou top node 20:20:36 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:20:36 INFO [loop_until]: OK (rc = 0) 20:20:36 DEBUG --- stdout --- 20:20:36 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 71m 0% 6247Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 76m 0% 7290Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 52m 0% 11655Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 81m 0% 3357Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 76m 0% 13973Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 170m 0% 5173Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 66m 0% 11637Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 80m 0% 2938Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 65m 0% 11165Mi 10% gke-xlou-cdm-ds-0f601751-j488 73m 0% 6725Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 72m 0% 16012Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 93m 0% 7439Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 71m 0% 13469Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 956m 6% 7986Mi 13% 20:20:36 DEBUG --- stderr --- 20:20:36 DEBUG 20:21:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713554492" 20:21:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554492" 20:21:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554492" 20:21:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554492" 20:21:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713554492" 20:21:32 ERROR [http_cmd]: ERROR 20:21:32 INFO [http_cmd]: http status code OK 20:21:32 ERROR --- status code --- 20:21:32 DEBUG --- status code --- 20:21:32 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:21:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:21:32 ERROR --- http response --- 20:21:32 DEBUG --- http response --- 20:21:32 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:21:32 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:21:32 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:21:32 INFO [http_cmd]: http status code OK 20:21:32 DEBUG --- status code --- 20:21:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:21:32 DEBUG --- http response --- 20:21:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:21:32 DEBUG ----- output ----- 20:21:32 DEBUG { 20:21:32 DEBUG "status": "success", 20:21:32 DEBUG "data": { 20:21:32 DEBUG "resultType": "vector", 20:21:32 DEBUG "result": [ 20:21:32 DEBUG { 20:21:32 DEBUG "metric": { 20:21:32 DEBUG "instance": "10.142.0.107:9100" 20:21:32 DEBUG }, 20:21:32 DEBUG "value": [ 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG [...] 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG "instance": "10.142.0.43:9100" 20:21:32 DEBUG }, 20:21:32 DEBUG "value": [ 20:21:32 DEBUG 1713554492, 20:21:32 DEBUG "0.0016666666666651508" 20:21:32 DEBUG ] 20:21:32 DEBUG } 20:21:32 DEBUG ] 20:21:32 DEBUG } 20:21:32 DEBUG } 20:21:32 INFO [http_cmd]: http status code OK 20:21:32 DEBUG --- status code --- 20:21:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:21:32 DEBUG --- http response --- 20:21:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:21:32 DEBUG ----- output ----- 20:21:32 DEBUG { 20:21:32 DEBUG "status": "success", 20:21:32 DEBUG "data": { 20:21:32 DEBUG "resultType": "vector", 20:21:32 DEBUG "result": [ 20:21:32 DEBUG { 20:21:32 DEBUG "metric": { 20:21:32 DEBUG "pod": "ds-idrepo-0" 20:21:32 DEBUG }, 20:21:32 DEBUG "value": [ 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG [...] 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG "pod": "ds-cts-2" 20:21:32 DEBUG }, 20:21:32 DEBUG "value": [ 20:21:32 DEBUG 1713554492, 20:21:32 DEBUG "0" 20:21:32 DEBUG ] 20:21:32 DEBUG } 20:21:32 DEBUG ] 20:21:32 DEBUG } 20:21:32 DEBUG } 20:21:32 INFO [http_cmd]: http status code OK 20:21:32 DEBUG --- status code --- 20:21:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:21:32 DEBUG --- http response --- 20:21:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:21:32 DEBUG ----- output ----- 20:21:32 DEBUG { 20:21:32 DEBUG "status": "success", 20:21:32 DEBUG "data": { 20:21:32 DEBUG "resultType": "vector", 20:21:32 DEBUG "result": [ 20:21:32 DEBUG { 20:21:32 DEBUG "metric": { 20:21:32 DEBUG "pod": "ds-idrepo-0" 20:21:32 DEBUG }, 20:21:32 DEBUG "value": [ 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG [...] 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG "pod": "ds-idrepo-2" 20:21:32 DEBUG }, 20:21:32 DEBUG "value": [ 20:21:32 DEBUG 1713554492, 20:21:32 DEBUG "0" 20:21:32 DEBUG ] 20:21:32 DEBUG } 20:21:32 DEBUG ] 20:21:32 DEBUG } 20:21:32 DEBUG } 20:21:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554492" 20:21:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554492" 20:21:32 INFO [http_cmd]: http status code OK 20:21:32 DEBUG --- status code --- 20:21:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:21:32 DEBUG --- http response --- 20:21:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:21:32 DEBUG ----- output ----- 20:21:32 DEBUG { 20:21:32 DEBUG "status": "success", 20:21:32 DEBUG "data": { 20:21:32 DEBUG "resultType": "vector", 20:21:32 DEBUG "result": [ 20:21:32 DEBUG { 20:21:32 DEBUG "metric": { 20:21:32 DEBUG "pod": "ds-idrepo-0" 20:21:32 DEBUG }, 20:21:32 DEBUG "value": [ 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG [...] 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG "pod": "ds-idrepo-2" 20:21:32 DEBUG }, 20:21:32 DEBUG "value": [ 20:21:32 DEBUG 1713554492, 20:21:32 DEBUG "0" 20:21:32 DEBUG ] 20:21:32 DEBUG } 20:21:32 DEBUG ] 20:21:32 DEBUG } 20:21:32 DEBUG } 20:21:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554492" 20:21:32 INFO [http_cmd]: http status code OK 20:21:32 INFO [http_cmd]: http status code OK 20:21:32 DEBUG --- status code --- 20:21:32 DEBUG --- status code --- 20:21:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:21:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:21:32 DEBUG --- http response --- 20:21:32 DEBUG --- http response --- 20:21:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:21:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:21:32 DEBUG ----- output ----- 20:21:32 DEBUG ----- output ----- 20:21:32 DEBUG { 20:21:32 DEBUG { 20:21:32 DEBUG "status": "success", 20:21:32 DEBUG "status": "success", 20:21:32 DEBUG "data": { 20:21:32 DEBUG "data": { 20:21:32 DEBUG "resultType": "vector", 20:21:32 DEBUG "resultType": "vector", 20:21:32 DEBUG "result": [ 20:21:32 DEBUG "result": [ 20:21:32 DEBUG { 20:21:32 DEBUG { 20:21:32 DEBUG "metric": { 20:21:32 DEBUG "metric": { 20:21:32 DEBUG "pod": "ds-cts-0" 20:21:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:21:32 DEBUG }, 20:21:32 DEBUG }, 20:21:32 DEBUG "value": [ 20:21:32 DEBUG "value": [ 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG [...] 20:21:32 DEBUG [...] 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG "pod": "ds-cts-2" 20:21:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:21:32 DEBUG }, 20:21:32 DEBUG }, 20:21:32 DEBUG "value": [ 20:21:32 DEBUG "value": [ 20:21:32 DEBUG 1713554492, 20:21:32 DEBUG 1713554492, 20:21:32 DEBUG "0" 20:21:32 DEBUG "0" 20:21:32 DEBUG ] 20:21:32 DEBUG ] 20:21:32 DEBUG } 20:21:32 DEBUG } 20:21:32 DEBUG ] 20:21:32 DEBUG ] 20:21:32 DEBUG } 20:21:32 DEBUG } 20:21:32 DEBUG } 20:21:32 DEBUG } 20:21:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554492" 20:21:32 INFO [http_cmd]: http status code OK 20:21:32 DEBUG --- status code --- 20:21:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:21:32 DEBUG --- http response --- 20:21:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:21:32 DEBUG ----- output ----- 20:21:32 DEBUG { 20:21:32 DEBUG "status": "success", 20:21:32 DEBUG "data": { 20:21:32 DEBUG "resultType": "vector", 20:21:32 DEBUG "result": [ 20:21:32 DEBUG { 20:21:32 DEBUG "metric": { 20:21:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:21:32 DEBUG }, 20:21:32 DEBUG "value": [ 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG [...] 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:21:32 DEBUG }, 20:21:32 DEBUG "value": [ 20:21:32 DEBUG 1713554492, 20:21:32 DEBUG "0" 20:21:32 DEBUG ] 20:21:32 DEBUG } 20:21:32 DEBUG ] 20:21:32 DEBUG } 20:21:32 DEBUG } 20:21:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713554492" 20:21:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554492" 20:21:32 INFO [http_cmd]: http status code OK 20:21:32 DEBUG --- status code --- 20:21:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:21:32 DEBUG --- http response --- 20:21:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:21:32 DEBUG ----- output ----- 20:21:32 DEBUG { 20:21:32 DEBUG "status": "success", 20:21:32 DEBUG "data": { 20:21:32 DEBUG "resultType": "vector", 20:21:32 DEBUG "result": [ 20:21:32 DEBUG { 20:21:32 DEBUG "metric": { 20:21:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:21:32 DEBUG }, 20:21:32 DEBUG "value": [ 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG [...] 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:21:32 DEBUG }, 20:21:32 DEBUG "value": [ 20:21:32 DEBUG 1713554492, 20:21:32 DEBUG "0.0165" 20:21:32 DEBUG ] 20:21:32 DEBUG } 20:21:32 DEBUG ] 20:21:32 DEBUG } 20:21:32 DEBUG } 20:21:32 INFO [http_cmd]: http status code OK 20:21:32 DEBUG --- status code --- 20:21:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:21:32 DEBUG --- http response --- 20:21:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:21:32 DEBUG ----- output ----- 20:21:32 DEBUG { 20:21:32 DEBUG "status": "success", 20:21:32 DEBUG "data": { 20:21:32 DEBUG "resultType": "vector", 20:21:32 DEBUG "result": [ 20:21:32 DEBUG { 20:21:32 DEBUG "metric": { 20:21:32 DEBUG "pod": "ds-idrepo-1" 20:21:32 DEBUG }, 20:21:32 DEBUG "value": [ 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG [...] 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:21:32 DEBUG }, 20:21:32 DEBUG "value": [ 20:21:32 DEBUG 1713554492, 20:21:32 DEBUG "0" 20:21:32 DEBUG ] 20:21:32 DEBUG } 20:21:32 DEBUG ] 20:21:32 DEBUG } 20:21:32 DEBUG } 20:21:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554492" 20:21:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554492" 20:21:32 INFO [http_cmd]: http status code OK 20:21:32 INFO [http_cmd]: http status code OK 20:21:32 DEBUG --- status code --- 20:21:32 DEBUG --- status code --- 20:21:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:21:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:21:32 DEBUG --- http response --- 20:21:32 DEBUG --- http response --- 20:21:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:21:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:21:32 DEBUG ----- output ----- 20:21:32 DEBUG ----- output ----- 20:21:32 DEBUG { 20:21:32 DEBUG { 20:21:32 DEBUG "status": "success", 20:21:32 DEBUG "status": "success", 20:21:32 DEBUG "data": { 20:21:32 DEBUG "data": { 20:21:32 DEBUG "resultType": "vector", 20:21:32 DEBUG "resultType": "vector", 20:21:32 DEBUG "result": [ 20:21:32 DEBUG "result": [ 20:21:32 DEBUG { 20:21:32 DEBUG { 20:21:32 DEBUG "metric": { 20:21:32 DEBUG "metric": { 20:21:32 DEBUG "instance": "10.142.0.107:9100" 20:21:32 DEBUG "pod": "ds-idrepo-1" 20:21:32 DEBUG }, 20:21:32 DEBUG }, 20:21:32 DEBUG "value": [ 20:21:32 DEBUG "value": [ 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG [...] 20:21:32 DEBUG [...] 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG "instance": "10.142.0.43:9100" 20:21:32 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:21:32 DEBUG }, 20:21:32 DEBUG }, 20:21:32 DEBUG "value": [ 20:21:32 DEBUG "value": [ 20:21:32 DEBUG 1713554492, 20:21:32 DEBUG 1713554492, 20:21:32 DEBUG "0.0036666666667400706" 20:21:32 DEBUG "0" 20:21:32 DEBUG ] 20:21:32 DEBUG ] 20:21:32 DEBUG } 20:21:32 DEBUG } 20:21:32 DEBUG ] 20:21:32 DEBUG ] 20:21:32 DEBUG } 20:21:32 DEBUG } 20:21:32 DEBUG } 20:21:32 DEBUG } 20:21:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554492" 20:21:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713554492" 20:21:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554492" 20:21:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554492" 20:21:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554492" 20:21:32 INFO [http_cmd]: http status code OK 20:21:32 INFO [http_cmd]: http status code OK 20:21:32 DEBUG --- status code --- 20:21:32 DEBUG --- status code --- 20:21:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:21:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:21:32 DEBUG --- http response --- 20:21:32 DEBUG --- http response --- 20:21:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:21:32 DEBUG ----- output ----- 20:21:32 DEBUG { 20:21:32 DEBUG "status": "success", 20:21:32 DEBUG "data": { 20:21:32 DEBUG "resultType": "vector", 20:21:32 DEBUG "result": [ 20:21:32 DEBUG { 20:21:32 DEBUG "metric": { 20:21:32 DEBUG "pod": "ds-idrepo-0" 20:21:32 DEBUG }, 20:21:32 DEBUG "value": [ 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG [...] 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG "pod": "ds-idrepo-2" 20:21:32 DEBUG }, 20:21:32 DEBUG "value": [ 20:21:32 DEBUG 1713554492, 20:21:32 DEBUG "0" 20:21:32 DEBUG ] 20:21:32 DEBUG } 20:21:32 DEBUG ] 20:21:32 DEBUG } 20:21:32 DEBUG } 20:21:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:21:32 DEBUG ----- output ----- 20:21:32 INFO [http_cmd]: http status code OK 20:21:32 DEBUG { 20:21:32 DEBUG --- status code --- 20:21:32 DEBUG "status": "success", 20:21:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:21:32 DEBUG "data": { 20:21:32 DEBUG --- http response --- 20:21:32 DEBUG "resultType": "vector", 20:21:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:21:32 DEBUG "result": [ 20:21:32 DEBUG ----- output ----- 20:21:32 INFO [http_cmd]: http status code OK 20:21:32 INFO [http_cmd]: http status code OK 20:21:32 DEBUG { 20:21:32 DEBUG { 20:21:32 DEBUG --- status code --- 20:21:32 DEBUG --- status code --- 20:21:32 DEBUG "metric": { 20:21:32 DEBUG "status": "success", 20:21:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:21:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:21:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:21:32 DEBUG "data": { 20:21:32 DEBUG --- http response --- 20:21:32 DEBUG --- http response --- 20:21:32 DEBUG }, 20:21:32 DEBUG "resultType": "vector", 20:21:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:21:32 DEBUG "value": [ 20:21:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:21:32 DEBUG "result": [ 20:21:32 DEBUG ----- output ----- 20:21:32 DEBUG 20:21:32 DEBUG ----- output ----- 20:21:32 DEBUG { 20:21:32 DEBUG { 20:21:32 DEBUG 20:21:32 DEBUG { 20:21:32 DEBUG "metric": { 20:21:32 DEBUG "status": "success", 20:21:32 DEBUG 20:21:32 DEBUG "status": "success", 20:21:32 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:21:32 DEBUG "data": { 20:21:32 DEBUG [...] 20:21:32 DEBUG "data": { 20:21:32 DEBUG }, 20:21:32 DEBUG "resultType": "vector", 20:21:32 DEBUG 20:21:32 DEBUG "resultType": "vector", 20:21:32 DEBUG "value": [ 20:21:32 DEBUG "result": [ 20:21:32 DEBUG 20:21:32 DEBUG "result": [ 20:21:32 DEBUG 20:21:32 DEBUG { 20:21:32 DEBUG 20:21:32 DEBUG { 20:21:32 DEBUG 20:21:32 DEBUG "metric": { 20:21:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:21:32 DEBUG "metric": { 20:21:32 DEBUG 20:21:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:21:32 DEBUG }, 20:21:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:21:32 DEBUG [...] 20:21:32 DEBUG }, 20:21:32 DEBUG "value": [ 20:21:32 DEBUG }, 20:21:32 DEBUG 20:21:32 DEBUG "value": [ 20:21:32 DEBUG 1713554492, 20:21:32 DEBUG "value": [ 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG "0" 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG ] 20:21:32 DEBUG 20:21:32 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:21:32 DEBUG 20:21:32 DEBUG } 20:21:32 DEBUG 20:21:32 DEBUG }, 20:21:32 DEBUG [...] 20:21:32 DEBUG ] 20:21:32 DEBUG [...] 20:21:32 DEBUG "value": [ 20:21:32 DEBUG 20:21:32 DEBUG } 20:21:32 DEBUG 20:21:32 DEBUG 1713554492, 20:21:32 DEBUG 20:21:32 DEBUG } 20:21:32 DEBUG 20:21:32 DEBUG "0.5181172940978348" 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG ] 20:21:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:21:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:21:32 DEBUG } 20:21:32 DEBUG }, 20:21:32 DEBUG }, 20:21:32 DEBUG ] 20:21:32 DEBUG "value": [ 20:21:32 DEBUG "value": [ 20:21:32 DEBUG } 20:21:32 DEBUG 1713554492, 20:21:32 DEBUG 1713554492, 20:21:32 DEBUG } 20:21:32 DEBUG "0" 20:21:32 DEBUG "0" 20:21:32 DEBUG ] 20:21:32 DEBUG ] 20:21:32 DEBUG } 20:21:32 DEBUG } 20:21:32 DEBUG ] 20:21:32 DEBUG ] 20:21:32 DEBUG } 20:21:32 DEBUG } 20:21:32 DEBUG } 20:21:32 DEBUG } 20:21:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554492" 20:21:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554492" 20:21:32 INFO [http_cmd]: http status code OK 20:21:32 DEBUG --- status code --- 20:21:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:21:32 DEBUG --- http response --- 20:21:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:21:32 DEBUG ----- output ----- 20:21:32 DEBUG { 20:21:32 DEBUG "status": "success", 20:21:32 DEBUG "data": { 20:21:32 DEBUG "resultType": "vector", 20:21:32 DEBUG "result": [ 20:21:32 DEBUG { 20:21:32 DEBUG "metric": { 20:21:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:21:32 DEBUG }, 20:21:32 DEBUG "value": [ 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG [...] 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:21:32 DEBUG }, 20:21:32 DEBUG "value": [ 20:21:32 DEBUG 1713554492, 20:21:32 DEBUG "0" 20:21:32 DEBUG ] 20:21:32 DEBUG } 20:21:32 DEBUG ] 20:21:32 DEBUG } 20:21:32 DEBUG } 20:21:32 INFO [http_cmd]: http status code OK 20:21:32 DEBUG --- status code --- 20:21:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:21:32 DEBUG --- http response --- 20:21:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:21:32 DEBUG ----- output ----- 20:21:32 DEBUG { 20:21:32 DEBUG "status": "success", 20:21:32 DEBUG "data": { 20:21:32 DEBUG "resultType": "vector", 20:21:32 DEBUG "result": [ 20:21:32 DEBUG { 20:21:32 DEBUG "metric": { 20:21:32 DEBUG "pod": "ds-idrepo-1" 20:21:32 DEBUG }, 20:21:32 DEBUG "value": [ 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG [...] 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:21:32 DEBUG }, 20:21:32 DEBUG "value": [ 20:21:32 DEBUG 1713554492, 20:21:32 DEBUG "0.000021509952854880292" 20:21:32 DEBUG ] 20:21:32 DEBUG } 20:21:32 DEBUG ] 20:21:32 DEBUG } 20:21:32 DEBUG } 20:21:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554492" 20:21:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554492" 20:21:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554492" 20:21:32 INFO [http_cmd]: http status code OK 20:21:32 DEBUG --- status code --- 20:21:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:21:32 DEBUG --- http response --- 20:21:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:21:32 DEBUG ----- output ----- 20:21:32 DEBUG { 20:21:32 DEBUG "status": "success", 20:21:32 DEBUG "data": { 20:21:32 DEBUG "resultType": "vector", 20:21:32 DEBUG "result": [ 20:21:32 DEBUG { 20:21:32 DEBUG "metric": { 20:21:32 DEBUG "instance": "10.142.0.107:9100" 20:21:32 DEBUG }, 20:21:32 DEBUG "value": [ 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG [...] 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG "instance": "10.142.0.43:9100" 20:21:32 DEBUG }, 20:21:32 DEBUG "value": [ 20:21:32 DEBUG 1713554492, 20:21:32 DEBUG "26645.466666666667" 20:21:32 DEBUG ] 20:21:32 DEBUG } 20:21:32 DEBUG ] 20:21:32 DEBUG } 20:21:32 DEBUG } 20:21:32 INFO [http_cmd]: http status code OK 20:21:32 DEBUG --- status code --- 20:21:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:21:32 DEBUG --- http response --- 20:21:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:21:32 DEBUG ----- output ----- 20:21:32 DEBUG { 20:21:32 DEBUG "status": "success", 20:21:32 DEBUG "data": { 20:21:32 DEBUG "resultType": "vector", 20:21:32 DEBUG "result": [ 20:21:32 DEBUG { 20:21:32 DEBUG "metric": { 20:21:32 DEBUG "pod": "ds-cts-0" 20:21:32 DEBUG }, 20:21:32 DEBUG "value": [ 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG [...] 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG "pod": "ds-cts-2" 20:21:32 DEBUG }, 20:21:32 DEBUG "value": [ 20:21:32 DEBUG 1713554492, 20:21:32 DEBUG "29.933333333333334" 20:21:32 DEBUG ] 20:21:32 DEBUG } 20:21:32 DEBUG ] 20:21:32 DEBUG } 20:21:32 DEBUG } 20:21:32 INFO [http_cmd]: http status code OK 20:21:32 DEBUG --- status code --- 20:21:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:21:32 DEBUG --- http response --- 20:21:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:21:32 DEBUG ----- output ----- 20:21:32 DEBUG { 20:21:32 DEBUG "status": "success", 20:21:32 DEBUG "data": { 20:21:32 DEBUG "resultType": "vector", 20:21:32 DEBUG "result": [ 20:21:32 DEBUG { 20:21:32 DEBUG "metric": { 20:21:32 DEBUG "pod": "ds-idrepo-1" 20:21:32 DEBUG }, 20:21:32 DEBUG "value": [ 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG [...] 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:21:32 DEBUG }, 20:21:32 DEBUG "value": [ 20:21:32 DEBUG 1713554492, 20:21:32 DEBUG "0" 20:21:32 DEBUG ] 20:21:32 DEBUG } 20:21:32 DEBUG ] 20:21:32 DEBUG } 20:21:32 DEBUG } 20:21:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554492" 20:21:32 INFO [http_cmd]: http status code OK 20:21:32 DEBUG --- status code --- 20:21:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:21:32 DEBUG --- http response --- 20:21:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:21:32 DEBUG ----- output ----- 20:21:32 DEBUG { 20:21:32 DEBUG "status": "success", 20:21:32 DEBUG "data": { 20:21:32 DEBUG "resultType": "vector", 20:21:32 DEBUG "result": [ 20:21:32 DEBUG { 20:21:32 DEBUG "metric": { 20:21:32 DEBUG "instance": "10.142.0.107:9100" 20:21:32 DEBUG }, 20:21:32 DEBUG "value": [ 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG [...] 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG "instance": "10.142.0.43:9100" 20:21:32 DEBUG }, 20:21:32 DEBUG "value": [ 20:21:32 DEBUG 1713554492, 20:21:32 DEBUG "2063537826816" 20:21:32 DEBUG ] 20:21:32 DEBUG } 20:21:32 DEBUG ] 20:21:32 DEBUG } 20:21:32 DEBUG } 20:21:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713554492" 20:21:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554492" 20:21:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554492" 20:21:32 INFO [http_cmd]: http status code OK 20:21:32 DEBUG --- status code --- 20:21:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:21:32 DEBUG --- http response --- 20:21:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:21:32 DEBUG ----- output ----- 20:21:32 DEBUG { 20:21:32 DEBUG "status": "success", 20:21:32 DEBUG "data": { 20:21:32 DEBUG "resultType": "vector", 20:21:32 DEBUG "result": [ 20:21:32 DEBUG { 20:21:32 DEBUG "metric": { 20:21:32 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:21:32 DEBUG }, 20:21:32 DEBUG "value": [ 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG [...] 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:21:32 DEBUG }, 20:21:32 DEBUG "value": [ 20:21:32 DEBUG 1713554492, 20:21:32 DEBUG "4836999168" 20:21:32 DEBUG ] 20:21:32 DEBUG } 20:21:32 DEBUG ] 20:21:32 DEBUG } 20:21:32 DEBUG } 20:21:32 INFO [http_cmd]: http status code OK 20:21:32 DEBUG --- status code --- 20:21:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:21:32 INFO [http_cmd]: http status code OK 20:21:32 DEBUG --- http response --- 20:21:32 DEBUG --- status code --- 20:21:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:21:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:21:32 DEBUG --- http response --- 20:21:32 DEBUG ----- output ----- 20:21:32 DEBUG { 20:21:32 DEBUG "status": "success", 20:21:32 DEBUG "data": { 20:21:32 DEBUG "resultType": "vector", 20:21:32 DEBUG "result": [ 20:21:32 DEBUG { 20:21:32 DEBUG "metric": { 20:21:32 DEBUG "pod": "ds-cts-0" 20:21:32 DEBUG }, 20:21:32 DEBUG "value": [ 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG [...] 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG "pod": "ds-cts-2" 20:21:32 DEBUG }, 20:21:32 DEBUG "value": [ 20:21:32 DEBUG 1713554492, 20:21:32 DEBUG "0" 20:21:32 DEBUG ] 20:21:32 DEBUG } 20:21:32 DEBUG ] 20:21:32 DEBUG } 20:21:32 DEBUG } 20:21:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:21:32 DEBUG ----- output ----- 20:21:32 DEBUG { 20:21:32 DEBUG "status": "success", 20:21:32 DEBUG "data": { 20:21:32 DEBUG "resultType": "vector", 20:21:32 DEBUG "result": [ 20:21:32 DEBUG { 20:21:32 DEBUG "metric": { 20:21:32 DEBUG "instance": "10.142.0.107:9100" 20:21:32 DEBUG }, 20:21:32 DEBUG "value": [ 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG [...] 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG 20:21:32 DEBUG "instance": "10.142.0.43:9100" 20:21:32 DEBUG }, 20:21:32 DEBUG "value": [ 20:21:32 DEBUG 1713554492, 20:21:32 DEBUG "26675.5" 20:21:32 DEBUG ] 20:21:32 DEBUG } 20:21:32 DEBUG ] 20:21:32 DEBUG } 20:21:32 DEBUG } 20:21:34 INFO 20:21:34 INFO [loop_until]: kubectl --namespace=xlou top pods 20:21:34 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:21:35 INFO [loop_until]: OK (rc = 0) 20:21:35 DEBUG --- stdout --- 20:21:35 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 7m 8952Mi am-767f6b67b5-qcqz2 7m 8750Mi am-767f6b67b5-vbltz 7m 8939Mi ds-cts-0 17m 11007Mi ds-cts-1 18m 11270Mi ds-cts-2 17m 11040Mi ds-idrepo-0 14m 3715Mi ds-idrepo-1 36m 3560Mi ds-idrepo-2 13m 3492Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 1085m 4965Mi 20:21:35 DEBUG --- stderr --- 20:21:35 DEBUG 20:21:36 INFO 20:21:36 INFO [loop_until]: kubectl --namespace=xlou top node 20:21:36 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:21:36 INFO [loop_until]: OK (rc = 0) 20:21:36 DEBUG --- stdout --- 20:21:36 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 67m 0% 6256Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 74m 0% 7306Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 56m 0% 11662Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 85m 0% 3367Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 75m 0% 13960Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 156m 0% 5167Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 67m 0% 11626Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 79m 0% 2941Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 62m 0% 11168Mi 10% gke-xlou-cdm-ds-0f601751-j488 70m 0% 6733Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 70m 0% 15998Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 112m 0% 7437Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 73m 0% 13457Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 1139m 7% 8754Mi 14% 20:21:36 DEBUG --- stderr --- 20:21:36 DEBUG 20:22:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554552" 20:22:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554552" 20:22:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713554552" 20:22:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554552" 20:22:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713554552" 20:22:32 INFO [http_cmd]: http status code OK 20:22:32 DEBUG --- status code --- 20:22:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:32 DEBUG --- http response --- 20:22:32 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:22:32 INFO [http_cmd]: http status code OK 20:22:32 DEBUG --- status code --- 20:22:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:32 DEBUG --- http response --- 20:22:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:22:32 DEBUG ----- output ----- 20:22:32 DEBUG { 20:22:32 DEBUG "status": "success", 20:22:32 DEBUG "data": { 20:22:32 DEBUG "resultType": "vector", 20:22:32 DEBUG "result": [ 20:22:32 DEBUG { 20:22:32 DEBUG "metric": { 20:22:32 DEBUG "pod": "ds-idrepo-0" 20:22:32 DEBUG }, 20:22:32 DEBUG "value": [ 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG [...] 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG "pod": "ds-idrepo-2" 20:22:32 DEBUG }, 20:22:32 DEBUG "value": [ 20:22:32 DEBUG 1713554552, 20:22:32 DEBUG "0" 20:22:32 DEBUG ] 20:22:32 DEBUG } 20:22:32 DEBUG ] 20:22:32 DEBUG } 20:22:32 DEBUG } 20:22:32 ERROR [http_cmd]: ERROR 20:22:32 ERROR --- status code --- 20:22:32 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:22:32 ERROR --- http response --- 20:22:32 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:22:32 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:22:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554552" 20:22:32 INFO [http_cmd]: http status code OK 20:22:32 DEBUG --- status code --- 20:22:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:32 DEBUG --- http response --- 20:22:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:22:32 DEBUG ----- output ----- 20:22:32 DEBUG { 20:22:32 DEBUG "status": "success", 20:22:32 DEBUG "data": { 20:22:32 DEBUG "resultType": "vector", 20:22:32 DEBUG "result": [ 20:22:32 DEBUG { 20:22:32 DEBUG "metric": { 20:22:32 DEBUG "pod": "ds-idrepo-0" 20:22:32 DEBUG }, 20:22:32 DEBUG "value": [ 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG [...] 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG "pod": "ds-cts-2" 20:22:32 DEBUG }, 20:22:32 DEBUG "value": [ 20:22:32 DEBUG 1713554552, 20:22:32 DEBUG "0" 20:22:32 DEBUG ] 20:22:32 DEBUG } 20:22:32 DEBUG ] 20:22:32 DEBUG } 20:22:32 DEBUG } 20:22:32 INFO [http_cmd]: http status code OK 20:22:32 DEBUG --- status code --- 20:22:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:32 DEBUG --- http response --- 20:22:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:22:32 DEBUG ----- output ----- 20:22:32 DEBUG { 20:22:32 DEBUG "status": "success", 20:22:32 DEBUG "data": { 20:22:32 DEBUG "resultType": "vector", 20:22:32 DEBUG "result": [ 20:22:32 DEBUG { 20:22:32 DEBUG "metric": { 20:22:32 DEBUG "instance": "10.142.0.107:9100" 20:22:32 DEBUG }, 20:22:32 DEBUG "value": [ 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG [...] 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG "instance": "10.142.0.43:9100" 20:22:32 DEBUG }, 20:22:32 DEBUG "value": [ 20:22:32 DEBUG 1713554552, 20:22:32 DEBUG "0.001999999999998181" 20:22:32 DEBUG ] 20:22:32 DEBUG } 20:22:32 DEBUG ] 20:22:32 DEBUG } 20:22:32 DEBUG } 20:22:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554552" 20:22:32 INFO [http_cmd]: http status code OK 20:22:32 DEBUG --- status code --- 20:22:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:32 DEBUG --- http response --- 20:22:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:22:32 DEBUG ----- output ----- 20:22:32 DEBUG { 20:22:32 DEBUG "status": "success", 20:22:32 DEBUG "data": { 20:22:32 DEBUG "resultType": "vector", 20:22:32 DEBUG "result": [ 20:22:32 DEBUG { 20:22:32 DEBUG "metric": { 20:22:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:22:32 DEBUG }, 20:22:32 DEBUG "value": [ 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG [...] 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:22:32 DEBUG }, 20:22:32 DEBUG "value": [ 20:22:32 DEBUG 1713554552, 20:22:32 DEBUG "0" 20:22:32 DEBUG ] 20:22:32 DEBUG } 20:22:32 DEBUG ] 20:22:32 DEBUG } 20:22:32 DEBUG } 20:22:32 INFO [http_cmd]: http status code OK 20:22:32 DEBUG --- status code --- 20:22:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:32 DEBUG --- http response --- 20:22:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:22:32 DEBUG ----- output ----- 20:22:32 DEBUG { 20:22:32 DEBUG "status": "success", 20:22:32 DEBUG "data": { 20:22:32 DEBUG "resultType": "vector", 20:22:32 DEBUG "result": [ 20:22:32 DEBUG { 20:22:32 DEBUG "metric": { 20:22:32 DEBUG "pod": "ds-idrepo-0" 20:22:32 DEBUG }, 20:22:32 DEBUG "value": [ 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG [...] 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG "pod": "ds-idrepo-2" 20:22:32 DEBUG }, 20:22:32 DEBUG "value": [ 20:22:32 DEBUG 1713554552, 20:22:32 DEBUG "0" 20:22:32 DEBUG ] 20:22:32 DEBUG } 20:22:32 DEBUG ] 20:22:32 DEBUG } 20:22:32 DEBUG } 20:22:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554552" 20:22:32 INFO [http_cmd]: http status code OK 20:22:32 DEBUG --- status code --- 20:22:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:32 DEBUG --- http response --- 20:22:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:22:32 DEBUG ----- output ----- 20:22:32 DEBUG { 20:22:32 DEBUG "status": "success", 20:22:32 DEBUG "data": { 20:22:32 DEBUG "resultType": "vector", 20:22:32 DEBUG "result": [ 20:22:32 DEBUG { 20:22:32 DEBUG "metric": { 20:22:32 DEBUG "pod": "ds-cts-0" 20:22:32 DEBUG }, 20:22:32 DEBUG "value": [ 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG [...] 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG "pod": "ds-cts-2" 20:22:32 DEBUG }, 20:22:32 DEBUG "value": [ 20:22:32 DEBUG 1713554552, 20:22:32 DEBUG "0" 20:22:32 DEBUG ] 20:22:32 DEBUG } 20:22:32 DEBUG ] 20:22:32 DEBUG } 20:22:32 DEBUG } 20:22:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554552" 20:22:32 INFO [http_cmd]: http status code OK 20:22:32 DEBUG --- status code --- 20:22:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:32 DEBUG --- http response --- 20:22:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:22:32 DEBUG ----- output ----- 20:22:32 DEBUG { 20:22:32 DEBUG "status": "success", 20:22:32 DEBUG "data": { 20:22:32 DEBUG "resultType": "vector", 20:22:32 DEBUG "result": [ 20:22:32 DEBUG { 20:22:32 DEBUG "metric": { 20:22:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:22:32 DEBUG }, 20:22:32 DEBUG "value": [ 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG [...] 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:22:32 DEBUG }, 20:22:32 DEBUG "value": [ 20:22:32 DEBUG 1713554552, 20:22:32 DEBUG "0" 20:22:32 DEBUG ] 20:22:32 DEBUG } 20:22:32 DEBUG ] 20:22:32 DEBUG } 20:22:32 DEBUG } 20:22:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554552" 20:22:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713554552" 20:22:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554552" 20:22:32 INFO [http_cmd]: http status code OK 20:22:32 DEBUG --- status code --- 20:22:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:32 DEBUG --- http response --- 20:22:32 INFO [http_cmd]: http status code OK 20:22:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:22:32 DEBUG --- status code --- 20:22:32 DEBUG ----- output ----- 20:22:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:32 DEBUG { 20:22:32 DEBUG --- http response --- 20:22:32 DEBUG "status": "success", 20:22:32 DEBUG "data": { 20:22:32 DEBUG "resultType": "vector", 20:22:32 DEBUG "result": [ 20:22:32 DEBUG { 20:22:32 DEBUG "metric": { 20:22:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:22:32 DEBUG }, 20:22:32 DEBUG "value": [ 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG [...] 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:22:32 DEBUG }, 20:22:32 DEBUG "value": [ 20:22:32 DEBUG 1713554552, 20:22:32 DEBUG "0.017" 20:22:32 DEBUG ] 20:22:32 DEBUG } 20:22:32 DEBUG ] 20:22:32 DEBUG } 20:22:32 DEBUG } 20:22:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:22:32 DEBUG ----- output ----- 20:22:32 DEBUG { 20:22:32 DEBUG "status": "success", 20:22:32 DEBUG "data": { 20:22:32 DEBUG "resultType": "vector", 20:22:32 DEBUG "result": [ 20:22:32 DEBUG { 20:22:32 DEBUG "metric": { 20:22:32 DEBUG "pod": "ds-idrepo-1" 20:22:32 DEBUG }, 20:22:32 DEBUG "value": [ 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG [...] 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 20:22:32 DEBUG }, 20:22:32 DEBUG "value": [ 20:22:32 DEBUG 1713554552, 20:22:32 DEBUG "18144107.48260992" 20:22:32 DEBUG ] 20:22:32 DEBUG } 20:22:32 DEBUG ] 20:22:32 DEBUG } 20:22:32 DEBUG } 20:22:32 INFO [http_cmd]: http status code OK 20:22:32 DEBUG --- status code --- 20:22:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:32 DEBUG --- http response --- 20:22:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:22:32 DEBUG ----- output ----- 20:22:32 DEBUG { 20:22:32 DEBUG "status": "success", 20:22:32 DEBUG "data": { 20:22:32 DEBUG "resultType": "vector", 20:22:32 DEBUG "result": [ 20:22:32 DEBUG { 20:22:32 DEBUG "metric": { 20:22:32 DEBUG "pod": "ds-idrepo-1" 20:22:32 DEBUG }, 20:22:32 DEBUG "value": [ 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG [...] 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 20:22:32 DEBUG }, 20:22:32 DEBUG "value": [ 20:22:32 DEBUG 1713554552, 20:22:32 DEBUG "0" 20:22:32 DEBUG ] 20:22:32 DEBUG } 20:22:32 DEBUG ] 20:22:32 DEBUG } 20:22:32 DEBUG } 20:22:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554552" 20:22:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713554552" 20:22:32 INFO [http_cmd]: http status code OK 20:22:32 DEBUG --- status code --- 20:22:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:32 DEBUG --- http response --- 20:22:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:22:32 DEBUG ----- output ----- 20:22:32 DEBUG { 20:22:32 DEBUG "status": "success", 20:22:32 DEBUG "data": { 20:22:32 DEBUG "resultType": "vector", 20:22:32 DEBUG "result": [ 20:22:32 DEBUG { 20:22:32 DEBUG "metric": { 20:22:32 DEBUG "instance": "10.142.0.107:9100" 20:22:32 DEBUG }, 20:22:32 DEBUG "value": [ 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG [...] 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG "instance": "10.142.0.43:9100" 20:22:32 DEBUG }, 20:22:32 DEBUG "value": [ 20:22:32 DEBUG 1713554552, 20:22:32 DEBUG "0.003533333333392837" 20:22:32 DEBUG ] 20:22:32 DEBUG } 20:22:32 DEBUG ] 20:22:32 DEBUG } 20:22:32 DEBUG } 20:22:32 INFO [http_cmd]: http status code OK 20:22:32 DEBUG --- status code --- 20:22:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:32 DEBUG --- http response --- 20:22:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:22:32 DEBUG ----- output ----- 20:22:32 DEBUG { 20:22:32 DEBUG "status": "success", 20:22:32 DEBUG "data": { 20:22:32 DEBUG "resultType": "vector", 20:22:32 DEBUG "result": [ 20:22:32 DEBUG { 20:22:32 DEBUG "metric": { 20:22:32 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:22:32 DEBUG }, 20:22:32 DEBUG "value": [ 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG [...] 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:22:32 DEBUG }, 20:22:32 DEBUG "value": [ 20:22:32 DEBUG 1713554552, 20:22:32 DEBUG "1.0313057007928457" 20:22:32 DEBUG ] 20:22:32 DEBUG } 20:22:32 DEBUG ] 20:22:32 DEBUG } 20:22:32 DEBUG } 20:22:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554552" 20:22:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554552" 20:22:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554552" 20:22:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554552" 20:22:32 INFO [http_cmd]: http status code OK 20:22:32 DEBUG --- status code --- 20:22:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:32 DEBUG --- http response --- 20:22:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:22:32 DEBUG ----- output ----- 20:22:32 DEBUG { 20:22:32 DEBUG "status": "success", 20:22:32 DEBUG "data": { 20:22:32 DEBUG "resultType": "vector", 20:22:32 DEBUG "result": [ 20:22:32 DEBUG { 20:22:32 DEBUG "metric": { 20:22:32 DEBUG "pod": "ds-idrepo-0" 20:22:32 DEBUG }, 20:22:32 DEBUG "value": [ 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG [...] 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG "pod": "ds-idrepo-2" 20:22:32 DEBUG }, 20:22:32 DEBUG "value": [ 20:22:32 DEBUG 1713554552, 20:22:32 DEBUG "0" 20:22:32 DEBUG ] 20:22:32 DEBUG } 20:22:32 DEBUG ] 20:22:32 DEBUG } 20:22:32 DEBUG } 20:22:32 INFO [http_cmd]: http status code OK 20:22:32 INFO [http_cmd]: http status code OK 20:22:32 DEBUG --- status code --- 20:22:32 DEBUG --- status code --- 20:22:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:32 DEBUG --- http response --- 20:22:32 DEBUG --- http response --- 20:22:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:22:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:22:32 DEBUG ----- output ----- 20:22:32 DEBUG ----- output ----- 20:22:32 DEBUG { 20:22:32 DEBUG { 20:22:32 DEBUG "status": "success", 20:22:32 DEBUG "status": "success", 20:22:32 DEBUG "data": { 20:22:32 DEBUG "data": { 20:22:32 DEBUG "resultType": "vector", 20:22:32 DEBUG "resultType": "vector", 20:22:32 DEBUG "result": [ 20:22:32 DEBUG "result": [ 20:22:32 DEBUG { 20:22:32 DEBUG { 20:22:32 DEBUG "metric": { 20:22:32 DEBUG "metric": { 20:22:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:22:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:22:32 DEBUG }, 20:22:32 DEBUG }, 20:22:32 DEBUG "value": [ 20:22:32 INFO [http_cmd]: http status code OK 20:22:32 DEBUG "value": [ 20:22:32 DEBUG 20:22:32 DEBUG --- status code --- 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG --- http response --- 20:22:32 DEBUG 20:22:32 DEBUG [...] 20:22:32 DEBUG [...] 20:22:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG ----- output ----- 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG { 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG "status": "success", 20:22:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:22:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:22:32 DEBUG "data": { 20:22:32 DEBUG }, 20:22:32 DEBUG }, 20:22:32 DEBUG "resultType": "vector", 20:22:32 DEBUG "value": [ 20:22:32 DEBUG "value": [ 20:22:32 DEBUG "result": [ 20:22:32 DEBUG 1713554552, 20:22:32 DEBUG 1713554552, 20:22:32 DEBUG { 20:22:32 DEBUG "0" 20:22:32 DEBUG "0" 20:22:32 DEBUG "metric": { 20:22:32 DEBUG ] 20:22:32 DEBUG ] 20:22:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:22:32 DEBUG } 20:22:32 DEBUG } 20:22:32 DEBUG }, 20:22:32 DEBUG ] 20:22:32 DEBUG ] 20:22:32 DEBUG "value": [ 20:22:32 DEBUG } 20:22:32 DEBUG } 20:22:32 DEBUG 20:22:32 DEBUG } 20:22:32 DEBUG 20:22:32 DEBUG } 20:22:32 DEBUG 20:22:32 DEBUG [...] 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:22:32 DEBUG }, 20:22:32 DEBUG "value": [ 20:22:32 DEBUG 1713554552, 20:22:32 DEBUG "0" 20:22:32 DEBUG ] 20:22:32 DEBUG } 20:22:32 DEBUG ] 20:22:32 DEBUG } 20:22:32 DEBUG } 20:22:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554552" 20:22:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554552" 20:22:32 INFO [http_cmd]: http status code OK 20:22:32 DEBUG --- status code --- 20:22:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:32 DEBUG --- http response --- 20:22:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:22:32 DEBUG ----- output ----- 20:22:32 DEBUG { 20:22:32 DEBUG "status": "success", 20:22:32 DEBUG "data": { 20:22:32 DEBUG "resultType": "vector", 20:22:32 DEBUG "result": [ 20:22:32 DEBUG { 20:22:32 DEBUG "metric": { 20:22:32 DEBUG "pod": "am-767f6b67b5-9zw22" 20:22:32 DEBUG }, 20:22:32 DEBUG "value": [ 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG [...] 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG "pod": "am-767f6b67b5-vbltz" 20:22:32 DEBUG }, 20:22:32 DEBUG "value": [ 20:22:32 DEBUG 1713554552, 20:22:32 DEBUG "0" 20:22:32 DEBUG ] 20:22:32 DEBUG } 20:22:32 DEBUG ] 20:22:32 DEBUG } 20:22:32 DEBUG } 20:22:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713554552" 20:22:32 INFO [http_cmd]: http status code OK 20:22:32 DEBUG --- status code --- 20:22:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:32 DEBUG --- http response --- 20:22:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:22:32 DEBUG ----- output ----- 20:22:32 DEBUG { 20:22:32 DEBUG "status": "success", 20:22:32 DEBUG "data": { 20:22:32 DEBUG "resultType": "vector", 20:22:32 DEBUG "result": [ 20:22:32 DEBUG { 20:22:32 DEBUG "metric": { 20:22:32 DEBUG "pod": "ds-idrepo-1" 20:22:32 DEBUG }, 20:22:32 DEBUG "value": [ 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG [...] 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 20:22:32 DEBUG }, 20:22:32 DEBUG "value": [ 20:22:32 DEBUG 1713554552, 20:22:32 DEBUG "1.0062273323513689" 20:22:32 DEBUG ] 20:22:32 DEBUG } 20:22:32 DEBUG ] 20:22:32 DEBUG } 20:22:32 DEBUG } 20:22:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554552" 20:22:32 INFO [http_cmd]: http status code OK 20:22:32 DEBUG --- status code --- 20:22:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:32 DEBUG --- http response --- 20:22:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:22:32 DEBUG ----- output ----- 20:22:32 DEBUG { 20:22:32 DEBUG "status": "success", 20:22:32 DEBUG "data": { 20:22:32 DEBUG "resultType": "vector", 20:22:32 DEBUG "result": [ 20:22:32 DEBUG { 20:22:32 DEBUG "metric": { 20:22:32 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:22:32 DEBUG }, 20:22:32 DEBUG "value": [ 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG [...] 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:22:32 DEBUG }, 20:22:32 DEBUG "value": [ 20:22:32 DEBUG 1713554552, 20:22:32 DEBUG "5778063360" 20:22:32 DEBUG ] 20:22:32 DEBUG } 20:22:32 DEBUG ] 20:22:32 DEBUG } 20:22:32 DEBUG } 20:22:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554552" 20:22:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554552" 20:22:32 INFO [http_cmd]: http status code OK 20:22:32 DEBUG --- status code --- 20:22:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:32 DEBUG --- http response --- 20:22:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:22:32 DEBUG ----- output ----- 20:22:32 DEBUG { 20:22:32 DEBUG "status": "success", 20:22:32 DEBUG "data": { 20:22:32 DEBUG "resultType": "vector", 20:22:32 DEBUG "result": [ 20:22:32 DEBUG { 20:22:32 DEBUG "metric": { 20:22:32 DEBUG "instance": "10.142.0.107:9100" 20:22:32 DEBUG }, 20:22:32 DEBUG "value": [ 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG [...] 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG "instance": "10.142.0.43:9100" 20:22:32 DEBUG }, 20:22:32 DEBUG "value": [ 20:22:32 DEBUG 1713554552, 20:22:32 DEBUG "27123.733333333334" 20:22:32 DEBUG ] 20:22:32 DEBUG } 20:22:32 DEBUG ] 20:22:32 DEBUG } 20:22:32 DEBUG } 20:22:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554552" 20:22:32 INFO [http_cmd]: http status code OK 20:22:32 DEBUG --- status code --- 20:22:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:32 DEBUG --- http response --- 20:22:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:22:32 DEBUG ----- output ----- 20:22:32 DEBUG { 20:22:32 DEBUG "status": "success", 20:22:32 DEBUG "data": { 20:22:32 DEBUG "resultType": "vector", 20:22:32 DEBUG "result": [ 20:22:32 DEBUG { 20:22:32 DEBUG "metric": { 20:22:32 DEBUG "pod": "ds-cts-0" 20:22:32 INFO [http_cmd]: http status code OK 20:22:32 DEBUG }, 20:22:32 DEBUG --- status code --- 20:22:32 DEBUG "value": [ 20:22:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:32 DEBUG 20:22:32 DEBUG --- http response --- 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG [...] 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:22:32 DEBUG 20:22:32 DEBUG ----- output ----- 20:22:32 DEBUG "pod": "ds-cts-2" 20:22:32 DEBUG { 20:22:32 DEBUG }, 20:22:32 DEBUG "status": "success", 20:22:32 DEBUG "value": [ 20:22:32 DEBUG "data": { 20:22:32 DEBUG 1713554552, 20:22:32 DEBUG "resultType": "vector", 20:22:32 DEBUG "30" 20:22:32 DEBUG "result": [ 20:22:32 DEBUG ] 20:22:32 DEBUG { 20:22:32 DEBUG } 20:22:32 DEBUG "metric": { 20:22:32 DEBUG ] 20:22:32 DEBUG "instance": "10.142.0.107:9100" 20:22:32 DEBUG } 20:22:32 DEBUG }, 20:22:32 DEBUG } 20:22:32 DEBUG "value": [ 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG [...] 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG "instance": "10.142.0.43:9100" 20:22:32 DEBUG }, 20:22:32 DEBUG "value": [ 20:22:32 DEBUG 1713554552, 20:22:32 DEBUG "26917.7" 20:22:32 DEBUG ] 20:22:32 DEBUG } 20:22:32 DEBUG ] 20:22:32 DEBUG } 20:22:32 DEBUG } 20:22:32 INFO [http_cmd]: http status code OK 20:22:32 DEBUG --- status code --- 20:22:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:32 DEBUG --- http response --- 20:22:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:22:32 DEBUG ----- output ----- 20:22:32 DEBUG { 20:22:32 DEBUG "status": "success", 20:22:32 DEBUG "data": { 20:22:32 DEBUG "resultType": "vector", 20:22:32 DEBUG "result": [ 20:22:32 DEBUG { 20:22:32 DEBUG "metric": { 20:22:32 DEBUG "pod": "ds-idrepo-1" 20:22:32 DEBUG }, 20:22:32 DEBUG "value": [ 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG [...] 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 20:22:32 DEBUG }, 20:22:32 DEBUG "value": [ 20:22:32 DEBUG 1713554552, 20:22:32 DEBUG "0" 20:22:32 DEBUG ] 20:22:32 DEBUG } 20:22:32 DEBUG ] 20:22:32 DEBUG } 20:22:32 DEBUG } 20:22:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713554552" 20:22:32 INFO [http_cmd]: http status code OK 20:22:32 DEBUG --- status code --- 20:22:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:32 DEBUG --- http response --- 20:22:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:22:32 DEBUG ----- output ----- 20:22:32 DEBUG { 20:22:32 DEBUG "status": "success", 20:22:32 DEBUG "data": { 20:22:32 DEBUG "resultType": "vector", 20:22:32 DEBUG "result": [ 20:22:32 DEBUG { 20:22:32 DEBUG "metric": { 20:22:32 DEBUG "instance": "10.142.0.107:9100" 20:22:32 DEBUG }, 20:22:32 DEBUG "value": [ 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG [...] 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG "instance": "10.142.0.43:9100" 20:22:32 DEBUG }, 20:22:32 DEBUG "value": [ 20:22:32 DEBUG 1713554552, 20:22:32 DEBUG "2063542551552" 20:22:32 DEBUG ] 20:22:32 DEBUG } 20:22:32 DEBUG ] 20:22:32 DEBUG } 20:22:32 DEBUG } 20:22:32 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713554552" 20:22:32 INFO [http_cmd]: http status code OK 20:22:32 DEBUG --- status code --- 20:22:32 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:32 DEBUG --- http response --- 20:22:32 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:22:32 DEBUG ----- output ----- 20:22:32 DEBUG { 20:22:32 DEBUG "status": "success", 20:22:32 DEBUG "data": { 20:22:32 DEBUG "resultType": "vector", 20:22:32 DEBUG "result": [ 20:22:32 DEBUG { 20:22:32 DEBUG "metric": { 20:22:32 DEBUG "pod": "ds-cts-0" 20:22:32 DEBUG }, 20:22:32 DEBUG "value": [ 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG [...] 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG 20:22:32 DEBUG "pod": "ds-cts-2" 20:22:32 DEBUG }, 20:22:32 DEBUG "value": [ 20:22:32 DEBUG 1713554552, 20:22:32 DEBUG "0" 20:22:32 DEBUG ] 20:22:32 DEBUG } 20:22:32 DEBUG ] 20:22:32 DEBUG } 20:22:32 DEBUG } 20:22:35 INFO 20:22:35 INFO [loop_until]: kubectl --namespace=xlou top pods 20:22:35 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:22:35 INFO [loop_until]: OK (rc = 0) 20:22:35 DEBUG --- stdout --- 20:22:35 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 7m 8952Mi am-767f6b67b5-qcqz2 7m 8750Mi am-767f6b67b5-vbltz 7m 8940Mi ds-cts-0 18m 11007Mi ds-cts-1 17m 11272Mi ds-cts-2 16m 11040Mi ds-idrepo-0 16m 3715Mi ds-idrepo-1 14m 3560Mi ds-idrepo-2 13m 3492Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-f8459c597-6tj6f 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 1029m 5912Mi 20:22:35 DEBUG --- stderr --- 20:22:35 DEBUG 20:22:36 INFO 20:22:36 INFO [loop_until]: kubectl --namespace=xlou top node 20:22:36 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:22:36 INFO [loop_until]: OK (rc = 0) 20:22:36 DEBUG --- stdout --- 20:22:36 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 66m 0% 6255Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 75m 0% 7304Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 54m 0% 11653Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 84m 0% 3358Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 69m 0% 13966Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 158m 0% 5198Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 65m 0% 11620Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 78m 0% 2945Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 62m 0% 11161Mi 9% gke-xlou-cdm-ds-0f601751-j488 71m 0% 6724Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 71m 0% 15988Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 92m 0% 7435Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 76m 0% 13452Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 1077m 6% 9614Mi 16% 20:22:36 DEBUG --- stderr --- 20:22:36 DEBUG 20:22:55 INFO Finished: True 20:22:55 INFO Waiting for threads to register finish flag 20:23:36 INFO Done. Have a nice day! :) 127.0.0.1 - - [19/Apr/2024 20:23:36] "GET /monitoring/stop HTTP/1.1" 200 - 20:23:40 WARNING File /tmp/lodemon_data-20240419_191228/node_disk_read_bytes_total.json does not exist. Skipping graph generation... WARNING:bokeh.core.validation.check:W-1000 (MISSING_RENDERERS): Plot has no renderers: figure(id='p6521', ...) 127.0.0.1 - - [19/Apr/2024 20:23:44] "GET /monitoring/process HTTP/1.1" 200 -