==================================================================================================== ========================================= Pod describe ========================================= ==================================================================================================== Name: lodemon-6b89d774d5-rf7sk Namespace: xlou Priority: 0 Service Account: k8s-svc-acct-lodemon Node: gke-xlou-cdm-default-pool-1b0769c8-7vmj/10.142.0.111 Start Time: Fri, 19 Apr 2024 03:42:32 +0000 Labels: app=lodemon app.kubernetes.io/name=lodemon pod-template-hash=6b89d774d5 skaffold.dev/run-id=983aace8-2ee2-4ab8-9843-8eb475d81782 Annotations: Status: Running IP: 10.106.42.185 IPs: IP: 10.106.42.185 Controlled By: ReplicaSet/lodemon-6b89d774d5 Containers: lodemon: Container ID: containerd://de558e49b1d8a5d9fed26fc1f539bf95dc78b1e3ec26f1977e528d2450907387 Image: gcr.io/engineeringpit/lodestar-images/lodestarbox:master-stable Image ID: gcr.io/engineeringpit/lodestar-images/lodestarbox@sha256:03f76e6326d713f4d73dc474e0e25fbbbaecc7c1d8c0590bfd94d68ac0d98c97 Port: 8080/TCP Host Port: 0/TCP Command: python3 Args: /lodestar/scripts/lodemon_run.py -W default State: Running Started: Fri, 19 Apr 2024 03:42:50 +0000 Ready: True Restart Count: 0 Limits: cpu: 2 memory: 2Gi Requests: cpu: 1 memory: 1Gi Liveness: exec [cat /tmp/lodemon_alive] delay=20s timeout=1s period=10s #success=1 #failure=3 Readiness: exec [cat /tmp/lodemon_alive] delay=20s timeout=1s period=10s #success=1 #failure=3 Environment: SKAFFOLD_PROFILE: medium Mounts: /lodestar/config/config.yaml from config (rw,path="config.yaml") /lodestar/config/deployments_to_monitor.yaml from deployments-config (rw,path="deployments_to_monitor.yaml") /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-zk2pt (ro) Conditions: Type Status Initialized True Ready True ContainersReady True PodScheduled True Volumes: config: Type: ConfigMap (a volume populated by a ConfigMap) Name: lodemon-config Optional: false deployments-config: Type: ConfigMap (a volume populated by a ConfigMap) Name: lodemon-deployments Optional: false kube-api-access-zk2pt: Type: Projected (a volume that contains injected data from multiple sources) TokenExpirationSeconds: 3607 ConfigMapName: kube-root-ca.crt ConfigMapOptional: DownwardAPI: true QoS Class: Burstable Node-Selectors: Tolerations: node.kubernetes.io/not-ready:NoExecute op=Exists for 300s node.kubernetes.io/unreachable:NoExecute op=Exists for 300s Events: ==================================================================================================== =========================================== Pod logs =========================================== ==================================================================================================== * Serving Flask app 'lodemon_run' * Debug mode: off WARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead. * Running on all addresses (0.0.0.0) * Running on http://127.0.0.1:8080 * Running on http://10.106.42.185:8080 Press CTRL+C to quit 04:43:50 INFO 04:43:50 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 04:43:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:43:50 INFO [loop_until]: OK (rc = 0) 04:43:50 DEBUG --- stdout --- 04:43:50 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 04:43:50 DEBUG --- stderr --- 04:43:50 DEBUG 04:43:50 INFO 04:43:50 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 04:43:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:43:50 INFO [loop_until]: OK (rc = 0) 04:43:50 DEBUG --- stdout --- 04:43:50 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 04:43:50 DEBUG --- stderr --- 04:43:50 DEBUG 04:43:50 INFO 04:43:50 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 04:43:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:43:50 INFO [loop_until]: OK (rc = 0) 04:43:50 DEBUG --- stdout --- 04:43:50 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 04:43:50 DEBUG --- stderr --- 04:43:50 DEBUG 04:43:50 INFO 04:43:50 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 04:43:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:43:50 INFO [loop_until]: OK (rc = 0) 04:43:50 DEBUG --- stdout --- 04:43:50 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 04:43:50 DEBUG --- stderr --- 04:43:50 DEBUG 04:43:50 INFO 04:43:50 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 04:43:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:43:50 INFO [loop_until]: OK (rc = 0) 04:43:50 DEBUG --- stdout --- 04:43:50 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 04:43:50 DEBUG --- stderr --- 04:43:50 DEBUG 04:43:50 INFO 04:43:50 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 04:43:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:43:50 INFO [loop_until]: OK (rc = 0) 04:43:50 DEBUG --- stdout --- 04:43:50 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 04:43:50 DEBUG --- stderr --- 04:43:50 DEBUG 04:43:50 INFO 04:43:50 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 04:43:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:43:51 INFO [loop_until]: OK (rc = 0) 04:43:51 DEBUG --- stdout --- 04:43:51 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 04:43:51 DEBUG --- stderr --- 04:43:51 DEBUG 04:43:51 INFO 04:43:51 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 04:43:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:43:51 INFO [loop_until]: OK (rc = 0) 04:43:51 DEBUG --- stdout --- 04:43:51 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 04:43:51 DEBUG --- stderr --- 04:43:51 DEBUG 04:43:51 INFO 04:43:51 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 04:43:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:43:51 INFO [loop_until]: OK (rc = 0) 04:43:51 DEBUG --- stdout --- 04:43:51 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 04:43:51 DEBUG --- stderr --- 04:43:51 DEBUG 04:43:51 INFO 04:43:51 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 04:43:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:43:51 INFO [loop_until]: OK (rc = 0) 04:43:51 DEBUG --- stdout --- 04:43:51 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 04:43:51 DEBUG --- stderr --- 04:43:51 DEBUG 04:43:51 INFO 04:43:51 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 04:43:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:43:51 INFO [loop_until]: OK (rc = 0) 04:43:51 DEBUG --- stdout --- 04:43:51 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 04:43:51 DEBUG --- stderr --- 04:43:51 DEBUG 04:43:51 INFO 04:43:51 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 04:43:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:43:51 INFO [loop_until]: OK (rc = 0) 04:43:51 DEBUG --- stdout --- 04:43:51 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 04:43:51 DEBUG --- stderr --- 04:43:51 DEBUG 04:43:51 INFO 04:43:51 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 04:43:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:43:51 INFO [loop_until]: OK (rc = 0) 04:43:51 DEBUG --- stdout --- 04:43:51 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 04:43:51 DEBUG --- stderr --- 04:43:51 DEBUG 04:43:51 INFO 04:43:51 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 04:43:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:43:51 INFO [loop_until]: OK (rc = 0) 04:43:51 DEBUG --- stdout --- 04:43:51 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 04:43:51 DEBUG --- stderr --- 04:43:51 DEBUG 04:43:52 INFO 04:43:52 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 04:43:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:43:52 INFO [loop_until]: OK (rc = 0) 04:43:52 DEBUG --- stdout --- 04:43:52 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 04:43:52 DEBUG --- stderr --- 04:43:52 DEBUG 04:43:52 INFO 04:43:52 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 04:43:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:43:52 INFO [loop_until]: OK (rc = 0) 04:43:52 DEBUG --- stdout --- 04:43:52 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 04:43:52 DEBUG --- stderr --- 04:43:52 DEBUG 04:43:52 INFO 04:43:52 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 04:43:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:43:52 INFO [loop_until]: OK (rc = 0) 04:43:52 DEBUG --- stdout --- 04:43:52 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 04:43:52 DEBUG --- stderr --- 04:43:52 DEBUG 04:43:52 INFO 04:43:52 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 04:43:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:43:52 INFO [loop_until]: OK (rc = 0) 04:43:52 DEBUG --- stdout --- 04:43:52 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 04:43:52 DEBUG --- stderr --- 04:43:52 DEBUG 04:43:52 INFO 04:43:52 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 04:43:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:43:52 INFO [loop_until]: OK (rc = 0) 04:43:52 DEBUG --- stdout --- 04:43:52 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 04:43:52 DEBUG --- stderr --- 04:43:52 DEBUG 04:43:52 INFO 04:43:52 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 04:43:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:43:52 INFO [loop_until]: OK (rc = 0) 04:43:52 DEBUG --- stdout --- 04:43:52 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 04:43:52 DEBUG --- stderr --- 04:43:52 DEBUG 04:43:52 INFO 04:43:52 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 04:43:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:43:52 INFO [loop_until]: OK (rc = 0) 04:43:52 DEBUG --- stdout --- 04:43:52 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 04:43:52 DEBUG --- stderr --- 04:43:52 DEBUG 04:43:52 INFO 04:43:52 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 04:43:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:43:53 INFO [loop_until]: OK (rc = 0) 04:43:53 DEBUG --- stdout --- 04:43:53 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 04:43:53 DEBUG --- stderr --- 04:43:53 DEBUG 04:43:53 INFO 04:43:53 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 04:43:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:43:53 INFO [loop_until]: OK (rc = 0) 04:43:53 DEBUG --- stdout --- 04:43:53 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 04:43:53 DEBUG --- stderr --- 04:43:53 DEBUG 04:43:53 INFO 04:43:53 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 04:43:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:43:53 INFO [loop_until]: OK (rc = 0) 04:43:53 DEBUG --- stdout --- 04:43:53 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 04:43:53 DEBUG --- stderr --- 04:43:53 DEBUG 04:43:53 INFO 04:43:53 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 04:43:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:43:53 INFO [loop_until]: OK (rc = 0) 04:43:53 DEBUG --- stdout --- 04:43:53 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 04:43:53 DEBUG --- stderr --- 04:43:53 DEBUG 04:43:53 INFO 04:43:53 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 04:43:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:43:53 INFO [loop_until]: OK (rc = 0) 04:43:53 DEBUG --- stdout --- 04:43:53 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 04:43:53 DEBUG --- stderr --- 04:43:53 DEBUG 04:43:53 INFO 04:43:53 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 04:43:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:43:53 INFO [loop_until]: OK (rc = 0) 04:43:53 DEBUG --- stdout --- 04:43:53 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 04:43:53 DEBUG --- stderr --- 04:43:53 DEBUG 04:43:53 INFO Initializing monitoring instance threads 04:43:53 DEBUG Monitoring instance thread list: [, , , , , , , , , , , , , , , , , , , , , , , , , , , , ] 04:43:53 INFO Starting instance threads 04:43:53 INFO 04:43:53 INFO Thread started 04:43:53 INFO [loop_until]: kubectl --namespace=xlou top node 04:43:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:43:53 INFO 04:43:53 INFO [loop_until]: kubectl --namespace=xlou top pods 04:43:53 INFO Thread started 04:43:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:43:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498233" 04:43:53 INFO Thread started 04:43:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498233" 04:43:53 INFO Thread started 04:43:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498233" 04:43:53 INFO Thread started 04:43:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498233" 04:43:53 INFO Thread started 04:43:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713498233" 04:43:53 INFO Thread started 04:43:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713498233" 04:43:53 INFO Thread started 04:43:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498233" 04:43:53 INFO Thread started 04:43:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498233" 04:43:53 INFO Thread started 04:43:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498233" 04:43:53 INFO Thread started 04:43:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498233" 04:43:53 INFO Thread started 04:43:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498233" 04:43:53 INFO Thread started 04:43:53 INFO [http_cmd]: http status code OK 04:43:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498233" 04:43:53 INFO [http_cmd]: http status code OK 04:43:53 DEBUG --- status code --- 04:43:53 INFO Thread started 04:43:53 DEBUG --- status code --- 04:43:53 INFO [http_cmd]: http status code OK 04:43:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:43:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498233" 04:43:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:43:53 INFO Thread started 04:43:53 DEBUG --- status code --- 04:43:53 DEBUG --- http response --- 04:43:53 DEBUG --- http response --- 04:43:53 INFO [http_cmd]: http status code OK 04:43:53 INFO [http_cmd]: http status code OK 04:43:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:43:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498233" 04:43:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:43:53 INFO Thread started 04:43:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:43:53 DEBUG --- status code --- 04:43:53 DEBUG --- status code --- 04:43:53 DEBUG --- http response --- 04:43:53 DEBUG ----- output ----- 04:43:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498233" 04:43:53 INFO [http_cmd]: http status code OK 04:43:53 INFO [http_cmd]: http status code OK 04:43:53 INFO Thread started 04:43:53 DEBUG ----- output ----- 04:43:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:43:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:43:53 INFO [http_cmd]: http status code OK 04:43:53 INFO [http_cmd]: http status code OK 04:43:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:43:53 INFO [http_cmd]: http status code OK 04:43:53 INFO [http_cmd]: http status code OK 04:43:53 DEBUG { 04:43:53 DEBUG --- status code --- 04:43:53 INFO [http_cmd]: http status code OK 04:43:53 DEBUG --- status code --- 04:43:53 INFO [http_cmd]: http status code OK 04:43:53 INFO [http_cmd]: http status code OK 04:43:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498233" 04:43:53 DEBUG { 04:43:53 DEBUG "status": "success", 04:43:53 DEBUG --- http response --- 04:43:53 DEBUG --- http response --- 04:43:53 DEBUG --- status code --- 04:43:53 DEBUG --- status code --- 04:43:53 DEBUG ----- output ----- 04:43:53 DEBUG --- status code --- 04:43:53 DEBUG --- status code --- 04:43:53 DEBUG "status": "success", 04:43:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:43:53 DEBUG --- status code --- 04:43:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:43:53 DEBUG --- status code --- 04:43:53 DEBUG --- status code --- 04:43:53 INFO Thread started 04:43:53 DEBUG "data": { 04:43:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:43:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:43:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:43:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:43:53 DEBUG { 04:43:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:43:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:43:53 DEBUG "data": { 04:43:53 DEBUG --- http response --- 04:43:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:43:53 DEBUG --- http response --- 04:43:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:43:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:43:53 DEBUG "resultType": "vector", 04:43:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498233" 04:43:53 INFO Thread started 04:43:53 DEBUG ----- output ----- 04:43:53 DEBUG ----- output ----- 04:43:53 DEBUG --- http response --- 04:43:53 DEBUG --- http response --- 04:43:53 DEBUG "status": "success", 04:43:53 DEBUG --- http response --- 04:43:53 DEBUG --- http response --- 04:43:53 DEBUG "resultType": "vector", 04:43:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:43:53 DEBUG --- http response --- 04:43:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:43:53 DEBUG --- http response --- 04:43:53 DEBUG --- http response --- 04:43:53 DEBUG "result": [ 04:43:53 DEBUG { 04:43:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498233" 04:43:53 DEBUG { 04:43:53 INFO Thread started 04:43:53 INFO [loop_until]: OK (rc = 0) 04:43:53 INFO [http_cmd]: http status code OK 04:43:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:43:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:43:53 DEBUG "data": { 04:43:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:43:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:43:53 DEBUG "result": [ 04:43:53 DEBUG ----- output ----- 04:43:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:43:53 DEBUG ----- output ----- 04:43:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:43:53 INFO [http_cmd]: http status code OK 04:43:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:43:53 DEBUG { 04:43:53 DEBUG "status": "success", 04:43:53 DEBUG "status": "success", 04:43:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498233" 04:43:53 DEBUG --- stdout --- 04:43:53 INFO Thread started 04:43:53 DEBUG --- status code --- 04:43:53 DEBUG ----- output ----- 04:43:53 DEBUG ----- output ----- 04:43:53 DEBUG "resultType": "vector", 04:43:53 DEBUG ----- output ----- 04:43:53 DEBUG ----- output ----- 04:43:53 DEBUG { 04:43:53 DEBUG { 04:43:53 DEBUG ----- output ----- 04:43:53 DEBUG { 04:43:53 DEBUG ----- output ----- 04:43:53 DEBUG --- status code --- 04:43:53 DEBUG ----- output ----- 04:43:53 DEBUG "metric": { 04:43:53 DEBUG "data": { 04:43:53 DEBUG "data": { 04:43:53 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 6m 8328Mi am-6fd554f8f7-hvvkw 10m 8815Mi am-6fd554f8f7-pmj4b 6m 8479Mi ds-cts-0 13m 2650Mi ds-cts-1 12m 11889Mi ds-cts-2 12m 12204Mi ds-idrepo-0 12m 3261Mi ds-idrepo-1 11m 3206Mi ds-idrepo-2 10m 3066Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 3m 1215Mi lodemon-6b89d774d5-rf7sk 38m 55Mi login-ui-5fdf49fc66-dlhjd 1m 5Mi overseer-0-5c67bf94db-77mh9 1m 1467Mi 04:43:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498233" 04:43:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:43:53 INFO Thread started 04:43:53 DEBUG { 04:43:53 DEBUG { 04:43:53 DEBUG "result": [ 04:43:53 DEBUG { 04:43:53 DEBUG { 04:43:53 DEBUG "metric": { 04:43:53 DEBUG "status": "success", 04:43:53 DEBUG { 04:43:53 INFO [http_cmd]: http status code OK 04:43:53 DEBUG "status": "success", 04:43:53 DEBUG { 04:43:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:43:53 DEBUG { 04:43:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:43:53 DEBUG "resultType": "vector", 04:43:53 DEBUG "resultType": "vector", 04:43:53 DEBUG --- stderr --- 04:43:53 DEBUG --- http response --- 04:43:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498233" 04:43:53 DEBUG "status": "success", 04:43:53 INFO Thread started 04:43:53 DEBUG "status": "success", 04:43:53 INFO [loop_until]: OK (rc = 0) 04:43:53 DEBUG { 04:43:53 DEBUG "status": "success", 04:43:53 DEBUG "status": "success", 04:43:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:43:53 DEBUG "data": { 04:43:53 DEBUG "status": "success", 04:43:53 DEBUG --- status code --- 04:43:53 DEBUG "data": { 04:43:53 DEBUG "status": "success", 04:43:53 DEBUG --- http response --- 04:43:53 DEBUG "status": "success", 04:43:53 DEBUG }, 04:43:53 DEBUG "result": [ 04:43:53 DEBUG "result": [ 04:43:53 DEBUG 04:43:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:43:53 INFO [http_cmd]: http status code OK 04:43:53 DEBUG "data": { 04:43:53 INFO [http_cmd]: http status code OK 04:43:53 DEBUG "data": { 04:43:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713498233" 04:43:53 DEBUG --- stdout --- 04:43:53 INFO Thread started 04:43:53 DEBUG "metric": { 04:43:53 DEBUG "data": { 04:43:53 DEBUG "data": { 04:43:53 DEBUG }, 04:43:53 DEBUG "resultType": "vector", 04:43:53 DEBUG "data": { 04:43:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:43:53 DEBUG "resultType": "vector", 04:43:53 DEBUG "data": { 04:43:53 DEBUG "data": { 04:43:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:43:53 DEBUG "value": [ 04:43:53 DEBUG { 04:43:53 DEBUG { 04:43:53 DEBUG ----- output ----- 04:43:53 DEBUG --- status code --- 04:43:53 DEBUG "resultType": "vector", 04:43:53 DEBUG --- status code --- 04:43:53 DEBUG "resultType": "vector", 04:43:53 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 85m 0% 6234Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 72m 0% 6683Mi 5% gke-xlou-cdm-default-pool-1b0769c8-nhlf 60m 0% 11515Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 85m 0% 3277Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 73m 0% 5162Mi 4% gke-xlou-cdm-default-pool-1b0769c8-wpwl 163m 0% 5169Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 70m 0% 11081Mi 9% gke-xlou-cdm-ds-0f601751-cf7b 78m 0% 2718Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 61m 0% 10670Mi 9% gke-xlou-cdm-ds-0f601751-j488 63m 0% 6136Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 75m 0% 15984Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 94m 0% 6850Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 68m 0% 14156Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 68m 0% 5253Mi 8% 04:43:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713498233" 04:43:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:43:53 DEBUG "resultType": "vector", 04:43:53 INFO Thread started 04:43:53 DEBUG "resultType": "vector", 04:43:53 DEBUG "value": [ 04:43:53 DEBUG "result": [ 04:43:53 DEBUG "resultType": "vector", 04:43:53 DEBUG --- http response --- 04:43:53 INFO [http_cmd]: http status code OK 04:43:53 DEBUG "result": [ 04:43:53 DEBUG "resultType": "vector", 04:43:53 DEBUG "resultType": "vector", 04:43:53 DEBUG ----- output ----- 04:43:53 DEBUG 04:43:53 DEBUG "metric": { 04:43:53 DEBUG "metric": { 04:43:53 DEBUG { 04:43:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:43:53 DEBUG "result": [ 04:43:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:43:53 DEBUG "result": [ 04:43:53 DEBUG --- stderr --- 04:43:53 DEBUG }, 04:43:53 DEBUG "result": [ 04:43:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713498233" 04:43:53 DEBUG "result": [ 04:43:53 INFO Thread started 04:43:53 DEBUG 04:43:53 DEBUG { 04:43:53 DEBUG "result": [ 04:43:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:43:53 DEBUG --- status code --- 04:43:53 DEBUG { 04:43:53 DEBUG "result": [ 04:43:53 DEBUG "result": [ 04:43:53 DEBUG { 04:43:53 DEBUG 04:43:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:43:53 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 04:43:53 INFO [http_cmd]: http status code OK 04:43:53 DEBUG "status": "success", 04:43:53 DEBUG --- http response --- 04:43:53 DEBUG { 04:43:53 DEBUG --- http response --- 04:43:53 DEBUG { 04:43:53 DEBUG 04:43:53 DEBUG "value": [ 04:43:53 DEBUG { 04:43:53 DEBUG { 04:43:53 DEBUG 04:43:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498233" 04:43:53 DEBUG "metric": { 04:43:53 INFO Thread started 04:43:53 DEBUG { 04:43:53 DEBUG ----- output ----- 04:43:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:43:53 DEBUG "metric": { 04:43:53 DEBUG { 04:43:53 DEBUG { 04:43:53 DEBUG "status": "success", 04:43:53 DEBUG "data": { 04:43:53 DEBUG 04:43:53 DEBUG [...] 04:43:53 DEBUG }, 04:43:53 DEBUG "data": { 04:43:53 DEBUG "metric": { 04:43:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:43:53 INFO [http_cmd]: http status code OK 04:43:53 DEBUG "metric": { 04:43:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:43:53 DEBUG 04:43:53 DEBUG "metric": { 04:43:53 DEBUG "metric": { 04:43:53 DEBUG 04:43:53 DEBUG "instance": "10.142.0.107:9100" 04:43:53 INFO [http_cmd]: http status code OK 04:43:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498233" 04:43:53 DEBUG "metric": { 04:43:53 INFO Thread started 04:43:53 DEBUG { 04:43:53 DEBUG --- http response --- 04:43:53 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 04:43:53 DEBUG "metric": { 04:43:53 DEBUG "metric": { 04:43:53 DEBUG }, 04:43:53 DEBUG "resultType": "vector", 04:43:53 DEBUG --- status code --- 04:43:53 DEBUG 04:43:53 DEBUG "value": [ 04:43:53 DEBUG "resultType": "vector", 04:43:53 DEBUG "instance": "10.142.0.107:9100" 04:43:53 DEBUG ----- output ----- 04:43:53 DEBUG --- status code --- 04:43:53 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:43:53 DEBUG ----- output ----- 04:43:53 DEBUG 04:43:53 DEBUG "pod": "ds-idrepo-0" 04:43:53 DEBUG "instance": "10.142.0.107:9100" 04:43:53 DEBUG [...] 04:43:53 DEBUG 04:43:53 DEBUG --- status code --- 04:43:53 DEBUG "pod": "ds-cts-0" 04:43:53 DEBUG "status": "success", 04:43:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498233" 04:43:53 INFO Thread started 04:43:53 DEBUG }, 04:43:53 DEBUG "pod": "ds-idrepo-0" 04:43:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:43:53 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:43:53 DEBUG "value": [ 04:43:53 DEBUG "result": [ 04:43:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:43:53 DEBUG 04:43:53 DEBUG 04:43:53 ERROR [http_cmd]: ERROR 04:43:53 DEBUG "result": [ 04:43:53 DEBUG }, 04:43:53 DEBUG { 04:43:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:43:53 DEBUG }, 04:43:53 DEBUG { 04:43:53 DEBUG 04:43:53 DEBUG }, 04:43:53 DEBUG }, 04:43:53 DEBUG }, 04:43:53 DEBUG 04:43:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:43:53 DEBUG }, 04:43:53 DEBUG "data": { 04:43:53 INFO All threads has been started 127.0.0.1 - - [19/Apr/2024 04:43:53] "GET /monitoring/start HTTP/1.1" 200 - 04:43:53 DEBUG "value": [ 04:43:53 DEBUG }, 04:43:53 DEBUG ----- output ----- 04:43:53 DEBUG }, 04:43:53 DEBUG 04:43:53 DEBUG { 04:43:53 DEBUG --- http response --- 04:43:53 DEBUG 04:43:53 DEBUG 04:43:53 ERROR --- status code --- 04:43:53 DEBUG { 04:43:53 DEBUG "value": [ 04:43:53 DEBUG "status": "success", 04:43:53 DEBUG --- http response --- 04:43:53 DEBUG "value": [ 04:43:53 DEBUG "status": "success", 04:43:53 DEBUG [...] 04:43:53 DEBUG "value": [ 04:43:53 DEBUG "value": [ 04:43:53 INFO [http_cmd]: http status code OK 04:43:53 DEBUG "value": [ 04:43:53 DEBUG 04:43:53 DEBUG --- http response --- 04:43:53 DEBUG "value": [ 04:43:53 DEBUG "resultType": "vector", 04:43:53 DEBUG 04:43:53 DEBUG "value": [ 04:43:53 DEBUG { 04:43:53 DEBUG "value": [ 04:43:53 DEBUG 04:43:53 DEBUG "metric": { 04:43:53 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 04:43:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:43:53 DEBUG 04:43:53 ERROR http status code is 400 (expected [200, 401, 403, 404]) 04:43:53 ERROR --- http response --- 04:43:53 DEBUG "metric": { 04:43:53 DEBUG "data": { 04:43:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:43:53 DEBUG 04:43:53 DEBUG "data": { 04:43:53 DEBUG "resultType": "vector", 04:43:53 DEBUG "result": [ 04:43:53 DEBUG 04:43:53 DEBUG --- status code --- 04:43:53 DEBUG 04:43:53 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 04:43:53 DEBUG 04:43:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:43:53 DEBUG "result": [ 04:43:53 DEBUG 04:43:53 DEBUG 04:43:53 DEBUG "status": "success", 04:43:53 DEBUG 04:43:53 DEBUG 04:43:53 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:43:53 DEBUG }, 04:43:53 DEBUG ----- output ----- 04:43:53 DEBUG [...] 04:43:53 DEBUG 04:43:53 DEBUG "pod": "ds-idrepo-0" 04:43:53 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 04:43:53 DEBUG "resultType": "vector", 04:43:53 INFO [http_cmd]: http status code OK 04:43:53 DEBUG ----- output ----- 04:43:53 DEBUG 04:43:53 DEBUG 04:43:53 DEBUG 04:43:53 DEBUG { 04:43:53 DEBUG 04:43:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:43:53 DEBUG 04:43:53 DEBUG }, 04:43:53 DEBUG 04:43:53 DEBUG ----- output ----- 04:43:53 DEBUG { 04:43:53 DEBUG 04:43:53 DEBUG 04:43:53 DEBUG "data": { 04:43:53 DEBUG 04:43:53 DEBUG [...] 04:43:53 DEBUG }, 04:43:53 INFO [http_cmd]: http status code OK 04:43:53 DEBUG "value": [ 04:43:53 DEBUG { 04:43:53 DEBUG 04:43:53 DEBUG 04:43:53 DEBUG }, 04:43:53 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 04:43:53 DEBUG "result": [ 04:43:53 DEBUG --- status code --- 04:43:53 DEBUG { 04:43:53 DEBUG 04:43:53 DEBUG 04:43:53 DEBUG 04:43:53 DEBUG 04:43:53 DEBUG [...] 04:43:53 DEBUG --- http response --- 04:43:53 DEBUG 04:43:53 DEBUG "value": [ 04:43:53 DEBUG 04:43:53 DEBUG { 04:43:53 DEBUG "status": "success", 04:43:53 DEBUG "data": { 04:43:53 DEBUG "metric": { 04:43:53 DEBUG "resultType": "vector", 04:43:53 DEBUG 04:43:53 DEBUG 04:43:53 DEBUG "value": [ 04:43:53 DEBUG --- status code --- 04:43:53 DEBUG 1713498233, 04:43:53 DEBUG "status": "success", 04:43:53 DEBUG 04:43:53 DEBUG 04:43:53 DEBUG "value": [ 04:43:53 DEBUG { 04:43:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:43:53 DEBUG "status": "success", 04:43:53 DEBUG [...] 04:43:53 DEBUG 04:43:53 DEBUG "metric": { 04:43:53 DEBUG "pod": "ds-cts-0" 04:43:53 DEBUG 04:43:53 DEBUG [...] 04:43:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:43:53 DEBUG 1713498233, 04:43:53 DEBUG "0" 04:43:53 DEBUG 04:43:53 DEBUG [...] 04:43:53 DEBUG "resultType": "vector", 04:43:53 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:43:53 DEBUG "result": [ 04:43:53 DEBUG { 04:43:53 DEBUG 04:43:53 DEBUG 04:43:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:43:53 DEBUG "0" 04:43:53 DEBUG "data": { 04:43:53 DEBUG 04:43:53 DEBUG [...] 04:43:53 DEBUG 04:43:53 DEBUG "metric": { 04:43:53 DEBUG --- http response --- 04:43:53 DEBUG "data": { 04:43:53 DEBUG 04:43:53 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 04:43:53 DEBUG 04:43:53 DEBUG }, 04:43:53 DEBUG 04:43:53 DEBUG 04:43:53 DEBUG ----- output ----- 04:43:53 DEBUG [...] 04:43:53 DEBUG ] 04:43:53 DEBUG [...] 04:43:53 DEBUG 04:43:53 DEBUG "result": [ 04:43:53 DEBUG }, 04:43:53 DEBUG [...] 04:43:53 DEBUG "metric": { 04:43:53 DEBUG 04:43:53 DEBUG 04:43:53 DEBUG --- http response --- 04:43:53 DEBUG ] 04:43:53 DEBUG "resultType": "vector", 04:43:53 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 04:43:53 DEBUG 04:43:53 DEBUG 04:43:53 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:43:53 DEBUG "resultType": "vector", 04:43:53 DEBUG 04:43:53 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 04:43:53 DEBUG }, 04:43:53 DEBUG [...] 04:43:53 DEBUG "value": [ 04:43:53 DEBUG 04:43:53 DEBUG 04:43:53 DEBUG { 04:43:53 DEBUG 04:43:53 DEBUG } 04:43:53 DEBUG 04:43:53 DEBUG 04:43:53 DEBUG { 04:43:53 DEBUG "value": [ 04:43:53 DEBUG 04:43:53 DEBUG "pod": "ds-cts-0" 04:43:53 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 04:43:53 DEBUG 04:43:53 DEBUG } 04:43:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:43:53 DEBUG "result": [ 04:43:53 DEBUG }, 04:43:53 DEBUG 04:43:53 DEBUG 04:43:53 DEBUG }, 04:43:53 DEBUG "result": [ 04:43:53 DEBUG 04:43:53 DEBUG "value": [ 04:43:53 DEBUG 04:43:53 DEBUG 04:43:53 DEBUG "pod": "ds-idrepo-2" 04:43:53 DEBUG }, 04:43:53 DEBUG "status": "success", 04:43:53 DEBUG 04:43:53 DEBUG ] 04:43:53 DEBUG 04:43:53 DEBUG 04:43:53 DEBUG "metric": { 04:43:53 DEBUG 04:43:53 DEBUG 04:43:53 DEBUG }, 04:43:53 DEBUG }, 04:43:53 DEBUG [...] 04:43:53 DEBUG ] 04:43:53 DEBUG ----- output ----- 04:43:53 DEBUG { 04:43:53 DEBUG "metric": { 04:43:53 DEBUG 04:43:53 DEBUG [...] 04:43:53 DEBUG "value": [ 04:43:53 DEBUG { 04:43:53 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:43:53 DEBUG 1713498233, 04:43:53 DEBUG 04:43:53 DEBUG 04:43:53 DEBUG 04:43:53 DEBUG "value": [ 04:43:53 DEBUG "data": { 04:43:53 DEBUG 04:43:53 DEBUG } 04:43:53 DEBUG 04:43:53 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 04:43:53 DEBUG "pod": "ds-idrepo-0" 04:43:53 DEBUG 04:43:53 DEBUG 04:43:53 DEBUG "value": [ 04:43:53 DEBUG "value": [ 04:43:53 DEBUG 04:43:53 DEBUG } 04:43:53 DEBUG { 04:43:53 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:43:53 DEBUG }, 04:43:53 DEBUG "value": [ 04:43:53 DEBUG 1713498233, 04:43:53 DEBUG "0.027360346906091026" 04:43:53 DEBUG 04:43:53 DEBUG }, 04:43:53 DEBUG "0.00001972998029898336" 04:43:53 DEBUG 04:43:53 DEBUG 04:43:53 DEBUG "instance": "10.142.0.43:9100" 04:43:53 DEBUG 1713498233, 04:43:53 DEBUG "resultType": "vector", 04:43:53 DEBUG "pod": "ds-cts-2" 04:43:53 DEBUG }, 04:43:53 DEBUG "pod": "ds-idrepo-2" 04:43:53 DEBUG }, 04:43:53 DEBUG }, 04:43:53 DEBUG 04:43:53 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:43:53 DEBUG 04:43:53 DEBUG 1713498233, 04:43:53 DEBUG 04:43:53 DEBUG } 04:43:53 DEBUG "status": "success", 04:43:53 DEBUG "instance": "10.142.0.43:9100" 04:43:53 DEBUG "value": [ 04:43:53 DEBUG 04:43:53 DEBUG "metric": { 04:43:53 DEBUG ] 04:43:53 DEBUG 04:43:53 DEBUG "value": [ 04:43:53 DEBUG ] 04:43:53 DEBUG "instance": "10.142.0.43:9100" 04:43:53 DEBUG [...] 04:43:53 DEBUG }, 04:43:53 DEBUG "0" 04:43:53 DEBUG "result": [ 04:43:53 DEBUG "value": [ 04:43:53 DEBUG } 04:43:53 DEBUG }, 04:43:53 DEBUG "value": [ 04:43:53 DEBUG "value": [ 04:43:53 DEBUG [...] 04:43:53 DEBUG }, 04:43:53 DEBUG 04:43:53 DEBUG "0" 04:43:53 DEBUG 04:43:53 DEBUG "data": { 04:43:53 DEBUG }, 04:43:53 DEBUG 04:43:53 DEBUG 04:43:53 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:43:53 DEBUG } 04:43:53 DEBUG 04:43:53 DEBUG 1713498233, 04:43:53 DEBUG } 04:43:53 DEBUG }, 04:43:53 DEBUG 04:43:53 DEBUG "value": [ 04:43:53 DEBUG ] 04:43:53 DEBUG { 04:43:53 DEBUG 1713498233, 04:43:53 DEBUG "value": [ 04:43:53 DEBUG 1713498233, 04:43:53 DEBUG 04:43:53 DEBUG 04:43:53 DEBUG "value": [ 04:43:53 DEBUG 04:43:53 DEBUG ] 04:43:53 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:43:53 DEBUG "resultType": "vector", 04:43:53 DEBUG "value": [ 04:43:53 DEBUG 04:43:53 DEBUG 04:43:53 DEBUG [...] 04:43:53 DEBUG }, 04:43:53 DEBUG "pod": "ds-idrepo-2" 04:43:53 DEBUG "0" 04:43:53 DEBUG ] 04:43:53 DEBUG "value": [ 04:43:53 DEBUG 04:43:53 DEBUG 1713498233, 04:43:53 DEBUG } 04:43:53 DEBUG "metric": { 04:43:53 DEBUG "0" 04:43:53 DEBUG 1713498233, 04:43:53 DEBUG "0" 04:43:53 DEBUG 04:43:53 DEBUG 04:43:53 DEBUG 1713498233, 04:43:53 DEBUG "0" 04:43:53 DEBUG } 04:43:53 DEBUG }, 04:43:53 DEBUG "result": [ 04:43:53 DEBUG 1713498233, 04:43:53 DEBUG 04:43:53 DEBUG ] 04:43:53 DEBUG 04:43:53 DEBUG 04:43:53 DEBUG "value": [ 04:43:53 DEBUG ] 04:43:53 DEBUG } 04:43:53 DEBUG } 04:43:53 DEBUG 04:43:53 DEBUG "28504.8" 04:43:53 DEBUG ] 04:43:53 DEBUG } 04:43:53 DEBUG "instance": "10.142.0.107:9100" 04:43:53 DEBUG "1635917824" 04:43:53 DEBUG ] 04:43:53 DEBUG 04:43:53 DEBUG 04:43:53 DEBUG [...] 04:43:53 DEBUG ] 04:43:53 DEBUG ] 04:43:53 DEBUG } 04:43:53 DEBUG { 04:43:53 DEBUG "0.00033333333333350387" 04:43:53 DEBUG [...] 04:43:53 DEBUG } 04:43:53 DEBUG }, 04:43:53 DEBUG 04:43:53 DEBUG 04:43:53 DEBUG 1713498233, 04:43:53 DEBUG ] 04:43:53 DEBUG } 04:43:53 DEBUG "pod": "ds-cts-2" 04:43:53 DEBUG ] 04:43:53 DEBUG ] 04:43:53 DEBUG } 04:43:53 DEBUG }, 04:43:53 DEBUG ] 04:43:53 DEBUG } 04:43:53 DEBUG [...] 04:43:53 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:43:53 DEBUG 04:43:53 DEBUG } 04:43:53 DEBUG "value": [ 04:43:53 DEBUG } 04:43:53 DEBUG "metric": { 04:43:53 DEBUG ] 04:43:53 DEBUG 04:43:53 DEBUG } 04:43:53 DEBUG "value": [ 04:43:53 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:43:53 DEBUG 04:43:53 DEBUG "33747.86666666667" 04:43:53 DEBUG } 04:43:53 DEBUG }, 04:43:53 DEBUG } 04:43:53 DEBUG } 04:43:53 DEBUG "value": [ 04:43:53 DEBUG } 04:43:53 DEBUG ] 04:43:53 DEBUG 04:43:53 DEBUG }, 04:43:53 DEBUG 04:43:53 DEBUG ] 04:43:53 DEBUG 1713498233, 04:43:53 DEBUG "instance": "10.142.0.107:9100" 04:43:53 DEBUG } 04:43:53 DEBUG 04:43:53 DEBUG 1713498233, 04:43:53 DEBUG }, 04:43:53 DEBUG 04:43:53 DEBUG ] 04:43:53 DEBUG } 04:43:53 DEBUG "value": [ 04:43:53 DEBUG ] 04:43:53 DEBUG } 04:43:53 DEBUG 04:43:53 DEBUG ] 04:43:53 DEBUG } 04:43:53 DEBUG 04:43:53 DEBUG "value": [ 04:43:53 DEBUG 04:43:53 DEBUG } 04:43:53 DEBUG "0" 04:43:53 DEBUG }, 04:43:53 DEBUG ] 04:43:53 DEBUG 04:43:53 DEBUG "0" 04:43:53 DEBUG "value": [ 04:43:53 DEBUG [...] 04:43:53 DEBUG } 04:43:53 DEBUG 1713498233, 04:43:53 DEBUG "0" 04:43:53 DEBUG } 04:43:53 DEBUG 04:43:53 DEBUG } 04:43:53 DEBUG } 04:43:53 DEBUG 04:43:53 DEBUG 1713498233, 04:43:53 DEBUG "pod": "ds-cts-2" 04:43:53 DEBUG } 04:43:53 DEBUG ] 04:43:53 DEBUG "value": [ 04:43:53 DEBUG } 04:43:53 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:43:53 DEBUG ] 04:43:53 DEBUG 1713498233, 04:43:53 DEBUG 04:43:53 DEBUG ] 04:43:53 DEBUG ] 04:43:53 DEBUG ] 04:43:53 DEBUG 04:43:53 DEBUG } 04:43:53 DEBUG "pod": "ds-cts-2" 04:43:53 DEBUG "0" 04:43:53 DEBUG }, 04:43:53 DEBUG "value": [ 04:43:53 DEBUG 04:43:53 DEBUG } 04:43:53 DEBUG }, 04:43:53 DEBUG } 04:43:53 DEBUG "2" 04:43:53 DEBUG 04:43:53 DEBUG } 04:43:53 DEBUG } 04:43:53 DEBUG } 04:43:53 DEBUG [...] 04:43:53 DEBUG }, 04:43:53 DEBUG ] 04:43:53 DEBUG } 04:43:53 DEBUG ] 04:43:53 DEBUG 04:43:53 DEBUG "value": [ 04:43:53 DEBUG ] 04:43:53 DEBUG ] 04:43:53 DEBUG 04:43:53 DEBUG } 04:43:53 DEBUG } 04:43:53 DEBUG ] 04:43:53 DEBUG } 04:43:53 DEBUG "value": [ 04:43:53 DEBUG } 04:43:53 DEBUG 1713498233, 04:43:53 DEBUG } 04:43:53 DEBUG 04:43:53 DEBUG 1713498233, 04:43:53 DEBUG } 04:43:53 DEBUG } 04:43:53 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:43:53 DEBUG 04:43:53 DEBUG } 04:43:53 DEBUG 1713498233, 04:43:53 DEBUG ] 04:43:53 DEBUG "0" 04:43:53 DEBUG } 04:43:53 DEBUG [...] 04:43:53 DEBUG "0" 04:43:53 DEBUG } 04:43:53 DEBUG ] 04:43:53 DEBUG }, 04:43:53 DEBUG 04:43:53 DEBUG "0" 04:43:53 DEBUG } 04:43:53 DEBUG ] 04:43:53 DEBUG 04:43:53 DEBUG ] 04:43:53 DEBUG } 04:43:53 DEBUG "value": [ 04:43:53 DEBUG 04:43:53 DEBUG ] 04:43:53 DEBUG } 04:43:53 DEBUG } 04:43:53 DEBUG 04:43:53 DEBUG } 04:43:53 DEBUG } 04:43:53 DEBUG 1713498233, 04:43:53 DEBUG "instance": "10.142.0.43:9100" 04:43:53 DEBUG } 04:43:53 DEBUG ] 04:43:53 DEBUG 04:43:53 DEBUG ] 04:43:53 DEBUG "55.171" 04:43:53 DEBUG }, 04:43:53 DEBUG ] 04:43:53 DEBUG } 04:43:53 DEBUG "instance": "10.142.0.43:9100" 04:43:53 DEBUG } 04:43:53 DEBUG ] 04:43:53 DEBUG "value": [ 04:43:53 DEBUG } 04:43:53 DEBUG } 04:43:53 DEBUG }, 04:43:53 DEBUG } 04:43:53 DEBUG } 04:43:53 DEBUG 1713498233, 04:43:53 DEBUG } 04:43:53 DEBUG "value": [ 04:43:53 DEBUG ] 04:43:53 DEBUG "387204615168" 04:43:53 DEBUG 1713498233, 04:43:53 DEBUG } 04:43:53 DEBUG ] 04:43:53 DEBUG "0.000366666666676944" 04:43:53 DEBUG } 04:43:53 DEBUG } 04:43:53 DEBUG ] 04:43:53 DEBUG ] 04:43:53 DEBUG } 04:43:53 DEBUG } 04:43:53 DEBUG ] 04:43:53 DEBUG } 04:43:53 DEBUG } 04:43:53 DEBUG } 04:44:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713498293" 04:44:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498293" 04:44:53 ERROR [http_cmd]: ERROR 04:44:53 ERROR --- status code --- 04:44:53 ERROR http status code is 400 (expected [200, 401, 403, 404]) 04:44:53 ERROR --- http response --- 04:44:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498293" 04:44:53 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 04:44:53 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 04:44:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498293" 04:44:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498293" 04:44:53 INFO 04:44:53 INFO 04:44:53 INFO [loop_until]: kubectl --namespace=xlou top pods 04:44:53 INFO [loop_until]: kubectl --namespace=xlou top node 04:44:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:44:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:44:53 INFO [http_cmd]: http status code OK 04:44:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713498293" 04:44:53 INFO [http_cmd]: http status code OK 04:44:53 DEBUG --- status code --- 04:44:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498293" 04:44:53 DEBUG --- status code --- 04:44:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713498293" 04:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:44:53 INFO [http_cmd]: http status code OK 04:44:53 DEBUG --- http response --- 04:44:53 DEBUG --- http response --- 04:44:53 DEBUG --- status code --- 04:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:44:53 DEBUG ----- output ----- 04:44:53 DEBUG --- http response --- 04:44:53 DEBUG ----- output ----- 04:44:53 DEBUG { 04:44:53 DEBUG { 04:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:44:53 DEBUG "status": "success", 04:44:53 DEBUG "status": "success", 04:44:53 DEBUG ----- output ----- 04:44:53 DEBUG "data": { 04:44:53 DEBUG "data": { 04:44:53 DEBUG { 04:44:53 DEBUG "resultType": "vector", 04:44:53 DEBUG "resultType": "vector", 04:44:53 DEBUG "status": "success", 04:44:53 DEBUG "result": [ 04:44:53 DEBUG "result": [ 04:44:53 DEBUG "data": { 04:44:53 DEBUG { 04:44:53 DEBUG { 04:44:53 DEBUG "resultType": "vector", 04:44:53 DEBUG "metric": { 04:44:53 DEBUG "metric": { 04:44:53 DEBUG "result": [ 04:44:53 DEBUG "pod": "ds-cts-0" 04:44:53 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:44:53 DEBUG { 04:44:53 DEBUG }, 04:44:53 DEBUG }, 04:44:53 DEBUG "metric": { 04:44:53 DEBUG "value": [ 04:44:53 DEBUG "value": [ 04:44:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG }, 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG "value": [ 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG [...] 04:44:53 DEBUG [...] 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG [...] 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG "pod": "ds-cts-2" 04:44:53 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:44:53 DEBUG 04:44:53 DEBUG }, 04:44:53 DEBUG }, 04:44:53 DEBUG 04:44:53 DEBUG "value": [ 04:44:53 DEBUG "value": [ 04:44:53 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 04:44:53 DEBUG 1713498293, 04:44:53 DEBUG 1713498293, 04:44:53 DEBUG }, 04:44:53 DEBUG "0" 04:44:53 DEBUG "468.93333333333334" 04:44:53 DEBUG "value": [ 04:44:53 DEBUG ] 04:44:53 DEBUG ] 04:44:53 DEBUG 1713498293, 04:44:53 DEBUG } 04:44:53 DEBUG } 04:44:53 DEBUG "7549.854125843674" 04:44:53 DEBUG ] 04:44:53 DEBUG ] 04:44:53 DEBUG ] 04:44:53 DEBUG } 04:44:53 DEBUG } 04:44:53 DEBUG } 04:44:53 DEBUG } 04:44:53 DEBUG } 04:44:53 DEBUG ] 04:44:53 INFO [http_cmd]: http status code OK 04:44:53 INFO [http_cmd]: http status code OK 04:44:53 DEBUG } 04:44:53 DEBUG --- status code --- 04:44:53 DEBUG --- status code --- 04:44:53 DEBUG } 04:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:44:53 DEBUG --- http response --- 04:44:53 INFO [http_cmd]: http status code OK 04:44:53 DEBUG --- http response --- 04:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:44:53 DEBUG --- status code --- 04:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:44:53 DEBUG ----- output ----- 04:44:53 INFO [http_cmd]: http status code OK 04:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:44:53 DEBUG ----- output ----- 04:44:53 DEBUG { 04:44:53 DEBUG --- status code --- 04:44:53 DEBUG --- http response --- 04:44:53 DEBUG { 04:44:53 DEBUG "status": "success", 04:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:44:53 DEBUG "status": "success", 04:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:44:53 DEBUG "data": { 04:44:53 DEBUG --- http response --- 04:44:53 DEBUG "data": { 04:44:53 DEBUG ----- output ----- 04:44:53 DEBUG "resultType": "vector", 04:44:53 DEBUG "resultType": "vector", 04:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:44:53 DEBUG { 04:44:53 DEBUG "result": [ 04:44:53 DEBUG "result": [ 04:44:53 DEBUG ----- output ----- 04:44:53 DEBUG "status": "success", 04:44:53 DEBUG { 04:44:53 DEBUG { 04:44:53 DEBUG { 04:44:53 DEBUG "data": { 04:44:53 DEBUG "metric": { 04:44:53 DEBUG "metric": { 04:44:53 DEBUG "status": "success", 04:44:53 DEBUG "resultType": "vector", 04:44:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:44:53 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:44:53 DEBUG "data": { 04:44:53 DEBUG "result": [ 04:44:53 DEBUG }, 04:44:53 DEBUG }, 04:44:53 DEBUG "resultType": "vector", 04:44:53 DEBUG { 04:44:53 DEBUG "value": [ 04:44:53 DEBUG "value": [ 04:44:53 DEBUG "result": [ 04:44:53 DEBUG "metric": { 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG { 04:44:53 DEBUG "pod": "ds-idrepo-0" 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG "metric": { 04:44:53 DEBUG }, 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 04:44:53 DEBUG "value": [ 04:44:53 DEBUG [...] 04:44:53 DEBUG [...] 04:44:53 DEBUG }, 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG "value": [ 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG [...] 04:44:53 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 04:44:53 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG }, 04:44:53 DEBUG }, 04:44:53 DEBUG [...] 04:44:53 DEBUG 04:44:53 DEBUG "value": [ 04:44:53 DEBUG "value": [ 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 1713498293, 04:44:53 DEBUG 1713498293, 04:44:53 DEBUG 04:44:53 DEBUG "pod": "ds-cts-2" 04:44:53 DEBUG "0" 04:44:53 DEBUG "468.43333333333334" 04:44:53 DEBUG 04:44:53 DEBUG }, 04:44:53 DEBUG ] 04:44:53 DEBUG ] 04:44:53 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 04:44:53 DEBUG "value": [ 04:44:53 DEBUG } 04:44:53 DEBUG } 04:44:53 DEBUG }, 04:44:53 DEBUG 1713498293, 04:44:53 DEBUG ] 04:44:53 DEBUG ] 04:44:53 DEBUG "value": [ 04:44:53 DEBUG "0" 04:44:53 DEBUG } 04:44:53 DEBUG } 04:44:53 DEBUG 1713498293, 04:44:53 DEBUG ] 04:44:53 DEBUG } 04:44:53 DEBUG } 04:44:53 DEBUG "2084425728" 04:44:53 DEBUG } 04:44:53 DEBUG ] 04:44:53 DEBUG ] 04:44:53 DEBUG } 04:44:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498293" 04:44:53 DEBUG } 04:44:53 DEBUG ] 04:44:53 DEBUG } 04:44:53 DEBUG } 04:44:53 DEBUG } 04:44:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498293" 04:44:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498293" 04:44:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498293" 04:44:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713498293" 04:44:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498293" 04:44:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498293" 04:44:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498293" 04:44:53 INFO [http_cmd]: http status code OK 04:44:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498293" 04:44:53 INFO [http_cmd]: http status code OK 04:44:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498293" 04:44:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498293" 04:44:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498293" 04:44:53 DEBUG --- status code --- 04:44:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498293" 04:44:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498293" 04:44:53 DEBUG --- status code --- 04:44:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498293" 04:44:53 INFO [http_cmd]: http status code OK 04:44:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498293" 04:44:53 INFO [http_cmd]: http status code OK 04:44:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498293" 04:44:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498293" 04:44:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713498293" 04:44:53 INFO [http_cmd]: http status code OK 04:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:44:53 INFO [http_cmd]: http status code OK 04:44:53 INFO [http_cmd]: http status code OK 04:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:44:53 DEBUG --- status code --- 04:44:53 INFO [http_cmd]: http status code OK 04:44:53 DEBUG --- status code --- 04:44:53 INFO [http_cmd]: http status code OK 04:44:53 INFO [http_cmd]: http status code OK 04:44:53 INFO [http_cmd]: http status code OK 04:44:53 INFO [http_cmd]: http status code OK 04:44:53 DEBUG --- status code --- 04:44:53 INFO [http_cmd]: http status code OK 04:44:53 DEBUG --- http response --- 04:44:53 INFO [http_cmd]: http status code OK 04:44:53 DEBUG --- status code --- 04:44:53 DEBUG --- status code --- 04:44:53 DEBUG --- http response --- 04:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:44:53 DEBUG --- status code --- 04:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:44:53 DEBUG --- status code --- 04:44:53 DEBUG --- status code --- 04:44:53 INFO [http_cmd]: http status code OK 04:44:53 INFO [http_cmd]: http status code OK 04:44:53 DEBUG --- status code --- 04:44:53 DEBUG --- status code --- 04:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:44:53 DEBUG --- status code --- 04:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 04:44:53 DEBUG --- status code --- 04:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:44:53 DEBUG --- http response --- 04:44:53 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 04:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:44:53 DEBUG --- http response --- 04:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:44:53 DEBUG --- status code --- 04:44:53 DEBUG --- status code --- 04:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:44:53 DEBUG --- http response --- 04:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:44:53 DEBUG ----- output ----- 04:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:44:53 DEBUG --- http response --- 04:44:53 DEBUG --- http response --- 04:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:44:53 INFO [http_cmd]: http status code OK 04:44:53 DEBUG --- status code --- 04:44:53 DEBUG --- http response --- 04:44:53 DEBUG --- http response --- 04:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:44:53 DEBUG --- http response --- 04:44:53 DEBUG --- http response --- 04:44:53 DEBUG --- http response --- 04:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:44:53 INFO [http_cmd]: http status code OK 04:44:53 INFO [http_cmd]: http status code OK 04:44:53 DEBUG { 04:44:53 DEBUG --- http response --- 04:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:44:53 DEBUG ----- output ----- 04:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:44:53 DEBUG --- http response --- 04:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:44:53 DEBUG ----- output ----- 04:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:44:53 DEBUG --- http response --- 04:44:53 DEBUG --- http response --- 04:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:44:53 DEBUG ----- output ----- 04:44:53 DEBUG --- status code --- 04:44:53 DEBUG --- status code --- 04:44:53 DEBUG "status": "success", 04:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:44:53 DEBUG ----- output ----- 04:44:53 DEBUG ----- output ----- 04:44:53 DEBUG { 04:44:53 DEBUG --- http response --- 04:44:53 DEBUG ----- output ----- 04:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 04:44:53 DEBUG { 04:44:53 DEBUG ----- output ----- 04:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:44:53 DEBUG ----- output ----- 04:44:53 DEBUG ----- output ----- 04:44:53 DEBUG ----- output ----- 04:44:53 DEBUG { 04:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:44:53 DEBUG "data": { 04:44:53 DEBUG ----- output ----- 04:44:53 DEBUG { 04:44:53 DEBUG { 04:44:53 DEBUG "status": "success", 04:44:53 DEBUG { 04:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:44:53 DEBUG ----- output ----- 04:44:53 DEBUG "status": "success", 04:44:53 DEBUG { 04:44:53 DEBUG ----- output ----- 04:44:53 DEBUG ----- output ----- 04:44:53 DEBUG { 04:44:53 DEBUG { 04:44:53 DEBUG { 04:44:53 DEBUG "status": "success", 04:44:53 DEBUG --- http response --- 04:44:53 DEBUG --- http response --- 04:44:53 DEBUG "resultType": "vector", 04:44:53 DEBUG { 04:44:53 DEBUG "status": "success", 04:44:53 DEBUG "status": "success", 04:44:53 DEBUG "data": { 04:44:53 DEBUG "status": "success", 04:44:53 DEBUG ----- output ----- 04:44:53 DEBUG { 04:44:53 DEBUG "data": { 04:44:53 DEBUG "status": "success", 04:44:53 DEBUG { 04:44:53 DEBUG { 04:44:53 DEBUG "status": "success", 04:44:53 DEBUG "status": "success", 04:44:53 DEBUG "status": "success", 04:44:53 DEBUG "data": { 04:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:44:53 DEBUG "result": [ 04:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:44:53 DEBUG "status": "success", 04:44:53 DEBUG "data": { 04:44:53 DEBUG "data": { 04:44:53 DEBUG "resultType": "vector", 04:44:53 DEBUG "data": { 04:44:53 DEBUG { 04:44:53 DEBUG "status": "success", 04:44:53 DEBUG "resultType": "vector", 04:44:53 DEBUG "data": { 04:44:53 DEBUG "status": "success", 04:44:53 DEBUG "status": "success", 04:44:53 DEBUG "data": { 04:44:53 DEBUG "data": { 04:44:53 DEBUG "data": { 04:44:53 DEBUG "resultType": "vector", 04:44:53 DEBUG ----- output ----- 04:44:53 DEBUG { 04:44:53 DEBUG ----- output ----- 04:44:53 DEBUG "data": { 04:44:53 DEBUG "resultType": "vector", 04:44:53 DEBUG "result": [ 04:44:53 DEBUG "resultType": "vector", 04:44:53 DEBUG "resultType": "vector", 04:44:53 DEBUG "status": "success", 04:44:53 DEBUG "data": { 04:44:53 DEBUG "result": [ 04:44:53 DEBUG "resultType": "vector", 04:44:53 DEBUG "data": { 04:44:53 DEBUG "data": { 04:44:53 DEBUG "resultType": "vector", 04:44:53 DEBUG "resultType": "vector", 04:44:53 DEBUG "resultType": "vector", 04:44:53 DEBUG "result": [ 04:44:53 DEBUG { 04:44:53 DEBUG "metric": { 04:44:53 DEBUG { 04:44:53 DEBUG "resultType": "vector", 04:44:53 DEBUG "result": [ 04:44:53 DEBUG { 04:44:53 DEBUG "result": [ 04:44:53 DEBUG "result": [ 04:44:53 DEBUG "data": { 04:44:53 DEBUG "resultType": "vector", 04:44:53 DEBUG { 04:44:53 DEBUG "result": [ 04:44:53 DEBUG "resultType": "vector", 04:44:53 DEBUG "resultType": "vector", 04:44:53 DEBUG "result": [ 04:44:53 DEBUG "result": [ 04:44:53 DEBUG "result": [ 04:44:53 DEBUG { 04:44:53 DEBUG "status": "success", 04:44:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:44:53 DEBUG "status": "success", 04:44:53 DEBUG "result": [ 04:44:53 DEBUG { 04:44:53 DEBUG "metric": { 04:44:53 DEBUG { 04:44:53 DEBUG { 04:44:53 DEBUG "resultType": "vector", 04:44:53 DEBUG "result": [ 04:44:53 DEBUG "metric": { 04:44:53 DEBUG { 04:44:53 DEBUG "result": [ 04:44:53 DEBUG "result": [ 04:44:53 DEBUG { 04:44:53 DEBUG { 04:44:53 DEBUG { 04:44:53 DEBUG "metric": { 04:44:53 DEBUG "data": { 04:44:53 DEBUG }, 04:44:53 DEBUG "data": { 04:44:53 DEBUG { 04:44:53 DEBUG "metric": { 04:44:53 DEBUG "instance": "10.142.0.107:9100" 04:44:53 DEBUG "metric": { 04:44:53 DEBUG "metric": { 04:44:53 DEBUG "result": [ 04:44:53 DEBUG { 04:44:53 DEBUG "instance": "10.142.0.107:9100" 04:44:53 DEBUG "metric": { 04:44:53 DEBUG { 04:44:53 DEBUG { 04:44:53 DEBUG "metric": { 04:44:53 DEBUG "metric": { 04:44:53 DEBUG "metric": { 04:44:53 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 04:44:53 DEBUG "resultType": "vector", 04:44:53 DEBUG "value": [ 04:44:53 DEBUG "resultType": "vector", 04:44:53 DEBUG "metric": { 04:44:53 DEBUG "pod": "ds-idrepo-0" 04:44:53 DEBUG }, 04:44:53 DEBUG "pod": "ds-idrepo-0" 04:44:53 DEBUG "pod": "ds-cts-0" 04:44:53 DEBUG { 04:44:53 DEBUG "metric": { 04:44:53 DEBUG }, 04:44:53 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:44:53 DEBUG "metric": { 04:44:53 DEBUG "metric": { 04:44:53 DEBUG "pod": "ds-idrepo-0" 04:44:53 DEBUG "instance": "10.142.0.107:9100" 04:44:53 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:44:53 DEBUG }, 04:44:53 DEBUG "result": [ 04:44:53 DEBUG 04:44:53 DEBUG "result": [ 04:44:53 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:44:53 DEBUG }, 04:44:53 DEBUG "value": [ 04:44:53 DEBUG }, 04:44:53 DEBUG }, 04:44:53 DEBUG "metric": { 04:44:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:44:53 DEBUG "value": [ 04:44:53 DEBUG }, 04:44:53 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:44:53 DEBUG "pod": "ds-cts-0" 04:44:53 DEBUG }, 04:44:53 DEBUG }, 04:44:53 DEBUG }, 04:44:53 DEBUG "value": [ 04:44:53 DEBUG { 04:44:53 DEBUG 04:44:53 DEBUG { 04:44:53 DEBUG }, 04:44:53 DEBUG "value": [ 04:44:53 DEBUG 04:44:53 DEBUG "value": [ 04:44:53 DEBUG "value": [ 04:44:53 DEBUG "instance": "10.142.0.107:9100" 04:44:53 DEBUG }, 04:44:53 DEBUG 04:44:53 DEBUG "value": [ 04:44:53 DEBUG }, 04:44:53 DEBUG }, 04:44:53 DEBUG "value": [ 04:44:53 DEBUG "value": [ 04:44:53 DEBUG "value": [ 04:44:53 DEBUG 04:44:53 DEBUG "metric": { 04:44:53 DEBUG 04:44:53 DEBUG "metric": { 04:44:53 DEBUG "value": [ 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG }, 04:44:53 DEBUG "value": [ 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG "value": [ 04:44:53 DEBUG "value": [ 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:44:53 DEBUG [...] 04:44:53 DEBUG "instance": "10.142.0.107:9100" 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG "value": [ 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG }, 04:44:53 DEBUG 04:44:53 DEBUG }, 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 INFO [loop_until]: OK (rc = 0) 04:44:53 DEBUG [...] 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG [...] 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG [...] 04:44:53 DEBUG "value": [ 04:44:53 DEBUG 04:44:53 DEBUG "value": [ 04:44:53 DEBUG 04:44:53 DEBUG [...] 04:44:53 DEBUG --- stdout --- 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG [...] 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG [...] 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG [...] 04:44:53 DEBUG [...] 04:44:53 DEBUG [...] 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG [...] 04:44:53 DEBUG 04:44:53 DEBUG [...] 04:44:53 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 6m 8328Mi am-6fd554f8f7-hvvkw 295m 8830Mi am-6fd554f8f7-pmj4b 6m 8479Mi ds-cts-0 337m 2735Mi ds-cts-1 11m 11888Mi ds-cts-2 72m 12206Mi ds-idrepo-0 11m 3261Mi ds-idrepo-1 114m 3206Mi ds-idrepo-2 224m 3065Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1172Mi idm-b85f9d978-wnhxb 4m 1215Mi lodemon-6b89d774d5-rf7sk 156m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 2382m 1894Mi 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG [...] 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG [...] 04:44:53 DEBUG [...] 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG --- stderr --- 04:44:53 DEBUG "instance": "10.142.0.43:9100" 04:44:53 DEBUG 04:44:53 DEBUG [...] 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG }, 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG }, 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG "instance": "10.142.0.43:9100" 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 04:44:53 DEBUG [...] 04:44:53 DEBUG "value": [ 04:44:53 DEBUG [...] 04:44:53 DEBUG 04:44:53 DEBUG "pod": "ds-idrepo-2" 04:44:53 DEBUG 04:44:53 DEBUG "value": [ 04:44:53 DEBUG "pod": "ds-cts-2" 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG }, 04:44:53 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:44:53 DEBUG 04:44:53 DEBUG 04:44:53 DEBUG "pod": "ds-idrepo-2" 04:44:53 DEBUG "instance": "10.142.0.43:9100" 04:44:53 DEBUG }, 04:44:53 DEBUG }, 04:44:53 DEBUG 04:44:53 DEBUG 1713498293, 04:44:53 DEBUG 04:44:53 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:44:53 DEBUG }, 04:44:53 DEBUG "pod": "ds-idrepo-2" 04:44:53 DEBUG 1713498293, 04:44:53 DEBUG }, 04:44:53 DEBUG 04:44:53 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 04:44:53 DEBUG "value": [ 04:44:53 DEBUG }, 04:44:53 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:44:53 DEBUG "pod": "ds-cts-2" 04:44:53 DEBUG }, 04:44:53 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:44:53 DEBUG "value": [ 04:44:53 DEBUG "value": [ 04:44:53 DEBUG 04:44:53 DEBUG "2668722.436403222" 04:44:53 DEBUG 04:44:53 DEBUG }, 04:44:53 DEBUG "value": [ 04:44:53 DEBUG }, 04:44:53 DEBUG "13198790.966666665" 04:44:53 DEBUG "value": [ 04:44:53 DEBUG "instance": "10.142.0.43:9100" 04:44:53 DEBUG }, 04:44:53 DEBUG 1713498293, 04:44:53 DEBUG "value": [ 04:44:53 DEBUG }, 04:44:53 DEBUG }, 04:44:53 DEBUG "value": [ 04:44:53 DEBUG }, 04:44:53 DEBUG 1713498293, 04:44:53 DEBUG "0.012999999999998598" 04:44:53 DEBUG 04:44:53 DEBUG ] 04:44:53 DEBUG 04:44:53 DEBUG "value": [ 04:44:53 DEBUG 1713498293, 04:44:53 DEBUG "0" 04:44:53 DEBUG ] 04:44:53 DEBUG 1713498293, 04:44:53 DEBUG }, 04:44:53 DEBUG "value": [ 04:44:53 DEBUG "13203761.6" 04:44:53 DEBUG 1713498293, 04:44:53 DEBUG "value": [ 04:44:53 DEBUG "value": [ 04:44:53 DEBUG 1713498293, 04:44:53 DEBUG "value": [ 04:44:53 DEBUG 1713498293, 04:44:53 DEBUG ] 04:44:53 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:44:53 DEBUG } 04:44:53 DEBUG "instance": "10.142.0.43:9100" 04:44:53 DEBUG 1713498293, 04:44:53 DEBUG "value": [ 04:44:53 DEBUG ] 04:44:53 DEBUG } 04:44:53 DEBUG "0" 04:44:53 DEBUG "value": [ 04:44:53 DEBUG 1713498293, 04:44:53 DEBUG ] 04:44:53 DEBUG "468.8666666666667" 04:44:53 DEBUG 1713498293, 04:44:53 DEBUG 1713498293, 04:44:53 DEBUG "0" 04:44:53 DEBUG 1713498293, 04:44:53 DEBUG "2.44855177864055" 04:44:53 DEBUG } 04:44:53 DEBUG }, 04:44:53 DEBUG ] 04:44:53 DEBUG }, 04:44:53 DEBUG "7022.5" 04:44:53 DEBUG 1713498293, 04:44:53 DEBUG } 04:44:53 DEBUG ] 04:44:53 DEBUG ] 04:44:53 DEBUG 1713498293, 04:44:53 DEBUG "1.7676777091734126" 04:44:53 DEBUG } 04:44:53 DEBUG ] 04:44:53 DEBUG "0" 04:44:53 DEBUG "0.00003321555555555555" 04:44:53 DEBUG ] 04:44:53 DEBUG "0" 04:44:53 DEBUG ] 04:44:53 DEBUG ] 04:44:53 DEBUG "value": [ 04:44:53 DEBUG } 04:44:53 DEBUG "value": [ 04:44:53 DEBUG ] 04:44:53 DEBUG "0" 04:44:53 DEBUG ] 04:44:53 DEBUG } 04:44:53 DEBUG } 04:44:53 DEBUG ] 04:44:53 DEBUG ] 04:44:53 DEBUG ] 04:44:53 DEBUG } 04:44:53 DEBUG ] 04:44:53 DEBUG ] 04:44:53 DEBUG } 04:44:53 DEBUG ] 04:44:53 DEBUG } 04:44:53 DEBUG } 04:44:53 DEBUG } 04:44:53 DEBUG } 04:44:53 DEBUG 1713498293, 04:44:53 DEBUG } 04:44:53 DEBUG ] 04:44:53 DEBUG } 04:44:53 DEBUG } 04:44:53 DEBUG "0.017099999999999227" 04:44:53 DEBUG } 04:44:53 DEBUG } 04:44:53 DEBUG } 04:44:53 DEBUG ] 04:44:53 DEBUG } 04:44:53 DEBUG } 04:44:53 DEBUG ] 04:44:53 DEBUG } 04:44:53 DEBUG ] 04:44:53 DEBUG 1713498293, 04:44:53 DEBUG "387335676928" 04:44:53 DEBUG ] 04:44:53 DEBUG } 04:44:53 DEBUG } 04:44:53 DEBUG ] 04:44:53 DEBUG } 04:44:53 DEBUG ] 04:44:53 DEBUG } 04:44:53 DEBUG } 04:44:53 DEBUG ] 04:44:53 DEBUG ] 04:44:53 DEBUG } 04:44:53 DEBUG ] 04:44:53 DEBUG } 04:44:53 DEBUG "55.171" 04:44:53 DEBUG ] 04:44:53 DEBUG } 04:44:53 DEBUG ] 04:44:53 DEBUG } 04:44:53 DEBUG } 04:44:53 DEBUG } 04:44:53 DEBUG } 04:44:53 DEBUG } 04:44:53 DEBUG } 04:44:53 INFO [loop_until]: OK (rc = 0) 04:44:53 DEBUG } 04:44:53 DEBUG } 04:44:53 DEBUG ] 04:44:53 DEBUG } 04:44:53 DEBUG } 04:44:53 DEBUG } 04:44:53 DEBUG ] 04:44:53 DEBUG } 04:44:53 DEBUG } 04:44:53 DEBUG } 04:44:53 DEBUG --- stdout --- 04:44:53 DEBUG } 04:44:53 DEBUG } 04:44:53 DEBUG ] 04:44:53 DEBUG } 04:44:53 DEBUG } 04:44:54 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 170m 0% 6245Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 90m 0% 6676Mi 5% gke-xlou-cdm-default-pool-1b0769c8-nhlf 163m 0% 11517Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 76m 0% 3266Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 517m 1% 5210Mi 4% gke-xlou-cdm-default-pool-1b0769c8-wpwl 157m 0% 5179Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 210m 0% 11099Mi 9% gke-xlou-cdm-ds-0f601751-cf7b 403m 1% 2699Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 643m 2% 10716Mi 9% gke-xlou-cdm-ds-0f601751-j488 81m 0% 6133Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 76m 0% 15873Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 94m 0% 6853Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 218m 0% 14144Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 2356m 14% 5678Mi 9% 04:44:54 DEBUG ] 04:44:54 DEBUG } 04:44:54 DEBUG } 04:44:54 DEBUG --- stderr --- 04:44:54 DEBUG } 04:44:54 DEBUG } 04:44:54 DEBUG 04:44:54 DEBUG } 04:45:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713498353" 04:45:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498353" 04:45:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498353" 04:45:53 ERROR [http_cmd]: ERROR 04:45:53 ERROR --- status code --- 04:45:53 ERROR http status code is 400 (expected [200, 401, 403, 404]) 04:45:53 ERROR --- http response --- 04:45:53 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 04:45:53 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 04:45:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713498353" 04:45:53 INFO [http_cmd]: http status code OK 04:45:53 DEBUG --- status code --- 04:45:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:45:53 DEBUG --- http response --- 04:45:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:45:53 DEBUG ----- output ----- 04:45:53 DEBUG { 04:45:53 DEBUG "status": "success", 04:45:53 DEBUG "data": { 04:45:53 DEBUG "resultType": "vector", 04:45:53 DEBUG "result": [ 04:45:53 DEBUG { 04:45:53 DEBUG "metric": { 04:45:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:45:53 DEBUG }, 04:45:53 DEBUG "value": [ 04:45:53 DEBUG 04:45:53 DEBUG 04:45:53 DEBUG 04:45:53 DEBUG [...] 04:45:53 DEBUG 04:45:53 DEBUG 04:45:53 DEBUG 04:45:53 DEBUG "pod": "idm-b85f9d978-txpwv" 04:45:53 DEBUG }, 04:45:53 DEBUG "value": [ 04:45:53 DEBUG 1713498353, 04:45:53 DEBUG "781.7166849563433" 04:45:53 DEBUG ] 04:45:53 DEBUG } 04:45:53 DEBUG ] 04:45:53 DEBUG } 04:45:53 DEBUG } 04:45:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498353" 04:45:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498353" 04:45:53 INFO [http_cmd]: http status code OK 04:45:53 DEBUG --- status code --- 04:45:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:45:53 DEBUG --- http response --- 04:45:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:45:53 DEBUG ----- output ----- 04:45:53 DEBUG { 04:45:53 DEBUG "status": "success", 04:45:53 DEBUG "data": { 04:45:53 DEBUG "resultType": "vector", 04:45:53 DEBUG "result": [ 04:45:53 DEBUG { 04:45:53 DEBUG "metric": { 04:45:53 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:45:53 DEBUG }, 04:45:53 DEBUG "value": [ 04:45:53 DEBUG 04:45:53 DEBUG 04:45:53 DEBUG 04:45:53 DEBUG [...] 04:45:53 DEBUG 04:45:53 DEBUG 04:45:53 DEBUG 04:45:53 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:45:53 DEBUG }, 04:45:53 DEBUG "value": [ 04:45:53 DEBUG 1713498353, 04:45:53 DEBUG "1999.2666666666667" 04:45:53 DEBUG ] 04:45:53 DEBUG } 04:45:53 DEBUG ] 04:45:53 DEBUG } 04:45:53 DEBUG } 04:45:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498353" 04:45:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713498353" 04:45:53 INFO [http_cmd]: http status code OK 04:45:53 INFO [http_cmd]: http status code OK 04:45:53 DEBUG --- status code --- 04:45:53 DEBUG --- status code --- 04:45:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:45:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:45:53 DEBUG --- http response --- 04:45:53 DEBUG --- http response --- 04:45:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:45:53 DEBUG ----- output ----- 04:45:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:45:53 DEBUG { 04:45:53 DEBUG ----- output ----- 04:45:53 DEBUG "status": "success", 04:45:53 DEBUG { 04:45:53 DEBUG "data": { 04:45:53 DEBUG "status": "success", 04:45:53 DEBUG "resultType": "vector", 04:45:53 DEBUG "data": { 04:45:53 DEBUG "result": [ 04:45:53 DEBUG "resultType": "vector", 04:45:53 DEBUG { 04:45:53 DEBUG "result": [ 04:45:53 DEBUG "metric": { 04:45:53 DEBUG { 04:45:53 DEBUG "pod": "ds-cts-0" 04:45:53 DEBUG "metric": { 04:45:53 DEBUG }, 04:45:53 DEBUG "pod": "ds-idrepo-0" 04:45:53 DEBUG "value": [ 04:45:53 DEBUG }, 04:45:53 DEBUG 04:45:53 DEBUG "value": [ 04:45:53 DEBUG 04:45:53 DEBUG 04:45:53 DEBUG 04:45:53 DEBUG 04:45:53 DEBUG [...] 04:45:53 DEBUG 04:45:53 DEBUG 04:45:53 DEBUG [...] 04:45:53 DEBUG 04:45:53 DEBUG 04:45:53 DEBUG 04:45:53 DEBUG 04:45:53 DEBUG "pod": "ds-cts-2" 04:45:53 DEBUG 04:45:53 DEBUG }, 04:45:53 DEBUG "pod": "ds-cts-2" 04:45:53 DEBUG "value": [ 04:45:53 DEBUG }, 04:45:53 DEBUG 1713498353, 04:45:53 DEBUG "value": [ 04:45:53 DEBUG "0" 04:45:53 DEBUG 1713498353, 04:45:53 DEBUG ] 04:45:53 DEBUG "0" 04:45:53 DEBUG } 04:45:53 DEBUG ] 04:45:53 DEBUG ] 04:45:53 DEBUG } 04:45:53 DEBUG } 04:45:53 DEBUG ] 04:45:53 DEBUG } 04:45:53 DEBUG } 04:45:53 DEBUG } 04:45:53 INFO [http_cmd]: http status code OK 04:45:53 DEBUG --- status code --- 04:45:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:45:53 DEBUG --- http response --- 04:45:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:45:53 DEBUG ----- output ----- 04:45:53 DEBUG { 04:45:53 DEBUG "status": "success", 04:45:53 DEBUG "data": { 04:45:53 DEBUG "resultType": "vector", 04:45:53 DEBUG "result": [ 04:45:53 DEBUG { 04:45:53 DEBUG "metric": { 04:45:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:45:53 DEBUG }, 04:45:53 DEBUG "value": [ 04:45:53 DEBUG 04:45:53 DEBUG 04:45:53 DEBUG 04:45:53 DEBUG [...] 04:45:53 DEBUG 04:45:53 DEBUG 04:45:53 DEBUG 04:45:53 DEBUG "pod": "idm-b85f9d978-txpwv" 04:45:53 DEBUG }, 04:45:53 DEBUG "value": [ 04:45:53 DEBUG 1713498353, 04:45:53 DEBUG "0" 04:45:53 DEBUG ] 04:45:53 DEBUG } 04:45:53 DEBUG ] 04:45:53 DEBUG } 04:45:53 DEBUG } 04:45:53 INFO [http_cmd]: http status code OK 04:45:53 DEBUG --- status code --- 04:45:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:45:53 DEBUG --- http response --- 04:45:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:45:53 DEBUG ----- output ----- 04:45:53 DEBUG { 04:45:53 DEBUG "status": "success", 04:45:53 DEBUG "data": { 04:45:53 DEBUG "resultType": "vector", 04:45:53 DEBUG "result": [ 04:45:53 DEBUG { 04:45:53 DEBUG "metric": { 04:45:53 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:45:53 DEBUG }, 04:45:53 DEBUG "value": [ 04:45:53 DEBUG 04:45:53 DEBUG 04:45:53 DEBUG 04:45:53 DEBUG [...] 04:45:53 DEBUG 04:45:53 DEBUG 04:45:53 DEBUG 04:45:53 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:45:53 DEBUG }, 04:45:53 DEBUG "value": [ 04:45:53 DEBUG 1713498353, 04:45:53 DEBUG "1997.7" 04:45:53 DEBUG ] 04:45:53 DEBUG } 04:45:53 DEBUG ] 04:45:53 DEBUG } 04:45:53 DEBUG } 04:45:53 INFO [http_cmd]: http status code OK 04:45:53 DEBUG --- status code --- 04:45:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:45:53 DEBUG --- http response --- 04:45:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:45:53 DEBUG ----- output ----- 04:45:53 DEBUG { 04:45:53 DEBUG "status": "success", 04:45:53 DEBUG "data": { 04:45:53 DEBUG "resultType": "vector", 04:45:53 DEBUG "result": [ 04:45:53 DEBUG { 04:45:53 DEBUG "metric": { 04:45:53 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 04:45:53 DEBUG }, 04:45:53 DEBUG "value": [ 04:45:53 DEBUG 04:45:53 DEBUG 04:45:53 DEBUG 04:45:53 DEBUG [...] 04:45:53 DEBUG 04:45:53 DEBUG 04:45:53 DEBUG 04:45:53 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 04:45:53 DEBUG }, 04:45:53 DEBUG "value": [ 04:45:53 DEBUG 1713498353, 04:45:53 DEBUG "2130771968" 04:45:53 DEBUG ] 04:45:53 DEBUG } 04:45:53 DEBUG ] 04:45:53 DEBUG } 04:45:53 DEBUG } 04:45:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713498354" 04:45:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498354" 04:45:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498354" 04:45:54 INFO [http_cmd]: http status code OK 04:45:54 DEBUG --- status code --- 04:45:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:45:54 DEBUG --- http response --- 04:45:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:45:54 DEBUG ----- output ----- 04:45:54 DEBUG { 04:45:54 DEBUG "status": "success", 04:45:54 DEBUG "data": { 04:45:54 DEBUG "resultType": "vector", 04:45:54 DEBUG "result": [ 04:45:54 DEBUG { 04:45:54 DEBUG "metric": { 04:45:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 04:45:54 DEBUG }, 04:45:54 DEBUG "value": [ 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG [...] 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 04:45:54 DEBUG }, 04:45:54 DEBUG "value": [ 04:45:54 DEBUG 1713498354, 04:45:54 DEBUG "6.529827739623806" 04:45:54 DEBUG ] 04:45:54 DEBUG } 04:45:54 DEBUG ] 04:45:54 DEBUG } 04:45:54 DEBUG } 04:45:54 INFO [http_cmd]: http status code OK 04:45:54 DEBUG --- status code --- 04:45:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:45:54 DEBUG --- http response --- 04:45:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:45:54 DEBUG ----- output ----- 04:45:54 DEBUG { 04:45:54 DEBUG "status": "success", 04:45:54 DEBUG "data": { 04:45:54 DEBUG "resultType": "vector", 04:45:54 DEBUG "result": [ 04:45:54 DEBUG { 04:45:54 DEBUG "metric": { 04:45:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:45:54 DEBUG }, 04:45:54 DEBUG "value": [ 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG [...] 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:45:54 DEBUG }, 04:45:54 DEBUG "value": [ 04:45:54 DEBUG 1713498354, 04:45:54 DEBUG "0" 04:45:54 DEBUG ] 04:45:54 DEBUG } 04:45:54 DEBUG ] 04:45:54 DEBUG } 04:45:54 DEBUG } 04:45:54 INFO 04:45:54 INFO [loop_until]: kubectl --namespace=xlou top node 04:45:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:45:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498354" 04:45:54 INFO [http_cmd]: http status code OK 04:45:54 DEBUG --- status code --- 04:45:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:45:54 DEBUG --- http response --- 04:45:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:45:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498354" 04:45:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498354" 04:45:54 DEBUG ----- output ----- 04:45:54 DEBUG { 04:45:54 DEBUG "status": "success", 04:45:54 DEBUG "data": { 04:45:54 DEBUG "resultType": "vector", 04:45:54 DEBUG "result": [ 04:45:54 DEBUG { 04:45:54 DEBUG "metric": { 04:45:54 DEBUG "instance": "10.142.0.107:9100" 04:45:54 DEBUG }, 04:45:54 DEBUG "value": [ 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG [...] 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG "instance": "10.142.0.43:9100" 04:45:54 DEBUG }, 04:45:54 DEBUG "value": [ 04:45:54 DEBUG 1713498354, 04:45:54 DEBUG "0.0359999999999971" 04:45:54 DEBUG ] 04:45:54 DEBUG } 04:45:54 DEBUG ] 04:45:54 DEBUG } 04:45:54 DEBUG } 04:45:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498354" 04:45:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498354" 04:45:54 INFO [http_cmd]: http status code OK 04:45:54 DEBUG --- status code --- 04:45:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:45:54 DEBUG --- http response --- 04:45:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:45:54 DEBUG ----- output ----- 04:45:54 DEBUG { 04:45:54 DEBUG "status": "success", 04:45:54 DEBUG "data": { 04:45:54 DEBUG "resultType": "vector", 04:45:54 DEBUG "result": [ 04:45:54 DEBUG { 04:45:54 DEBUG "metric": { 04:45:54 DEBUG "instance": "10.142.0.107:9100" 04:45:54 DEBUG }, 04:45:54 DEBUG "value": [ 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG [...] 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG "instance": "10.142.0.43:9100" 04:45:54 DEBUG }, 04:45:54 DEBUG "value": [ 04:45:54 DEBUG 1713498354, 04:45:54 DEBUG "0.04749999999999564" 04:45:54 DEBUG ] 04:45:54 DEBUG } 04:45:54 DEBUG ] 04:45:54 DEBUG } 04:45:54 DEBUG } 04:45:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713498354" 04:45:54 INFO [http_cmd]: http status code OK 04:45:54 DEBUG --- status code --- 04:45:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:45:54 DEBUG --- http response --- 04:45:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:45:54 DEBUG ----- output ----- 04:45:54 DEBUG { 04:45:54 DEBUG "status": "success", 04:45:54 DEBUG "data": { 04:45:54 DEBUG "resultType": "vector", 04:45:54 DEBUG "result": [ 04:45:54 DEBUG { 04:45:54 DEBUG "metric": { 04:45:54 DEBUG "pod": "ds-cts-0" 04:45:54 DEBUG }, 04:45:54 DEBUG "value": [ 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG [...] 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG "pod": "ds-cts-2" 04:45:54 DEBUG }, 04:45:54 DEBUG "value": [ 04:45:54 DEBUG 1713498354, 04:45:54 DEBUG "0" 04:45:54 DEBUG ] 04:45:54 DEBUG } 04:45:54 DEBUG ] 04:45:54 DEBUG } 04:45:54 DEBUG } 04:45:54 INFO [http_cmd]: http status code OK 04:45:54 DEBUG --- status code --- 04:45:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:45:54 DEBUG --- http response --- 04:45:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 04:45:54 INFO [http_cmd]: http status code OK 04:45:54 DEBUG --- status code --- 04:45:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:45:54 DEBUG --- http response --- 04:45:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:45:54 DEBUG ----- output ----- 04:45:54 INFO [http_cmd]: http status code OK 04:45:54 DEBUG { 04:45:54 DEBUG --- status code --- 04:45:54 DEBUG "status": "success", 04:45:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:45:54 DEBUG "data": { 04:45:54 DEBUG --- http response --- 04:45:54 DEBUG "resultType": "vector", 04:45:54 DEBUG "result": [ 04:45:54 DEBUG { 04:45:54 DEBUG "metric": { 04:45:54 DEBUG "pod": "ds-cts-0" 04:45:54 DEBUG }, 04:45:54 DEBUG "value": [ 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG [...] 04:45:54 INFO 04:45:54 DEBUG 04:45:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:45:54 INFO [loop_until]: kubectl --namespace=xlou top pods 04:45:54 DEBUG 04:45:54 DEBUG ----- output ----- 04:45:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:45:54 DEBUG 04:45:54 DEBUG { 04:45:54 DEBUG "pod": "ds-cts-2" 04:45:54 DEBUG "status": "success", 04:45:54 DEBUG }, 04:45:54 DEBUG "data": { 04:45:54 DEBUG "value": [ 04:45:54 DEBUG "resultType": "vector", 04:45:54 DEBUG 1713498354, 04:45:54 DEBUG "result": [ 04:45:54 DEBUG "0" 04:45:54 DEBUG { 04:45:54 DEBUG ] 04:45:54 DEBUG "metric": { 04:45:54 DEBUG } 04:45:54 DEBUG "pod": "ds-idrepo-0" 04:45:54 DEBUG ] 04:45:54 DEBUG }, 04:45:54 DEBUG } 04:45:54 DEBUG "value": [ 04:45:54 DEBUG } 04:45:54 DEBUG 04:45:54 INFO [http_cmd]: http status code OK 04:45:54 DEBUG 04:45:54 DEBUG --- status code --- 04:45:54 DEBUG 04:45:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:45:54 DEBUG [...] 04:45:54 DEBUG --- http response --- 04:45:54 DEBUG 04:45:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:45:54 DEBUG 04:45:54 DEBUG ----- output ----- 04:45:54 DEBUG 04:45:54 DEBUG { 04:45:54 DEBUG "pod": "ds-idrepo-2" 04:45:54 DEBUG "status": "success", 04:45:54 DEBUG }, 04:45:54 DEBUG "data": { 04:45:54 DEBUG "value": [ 04:45:54 DEBUG "resultType": "vector", 04:45:54 DEBUG 1713498354, 04:45:54 DEBUG "result": [ 04:45:54 DEBUG "0" 04:45:54 DEBUG { 04:45:54 DEBUG ] 04:45:54 DEBUG "metric": { 04:45:54 DEBUG } 04:45:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:45:54 DEBUG ] 04:45:54 DEBUG }, 04:45:54 DEBUG } 04:45:54 DEBUG "value": [ 04:45:54 DEBUG } 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG [...] 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:45:54 DEBUG }, 04:45:54 DEBUG "value": [ 04:45:54 DEBUG 1713498354, 04:45:54 DEBUG "55.171" 04:45:54 DEBUG ] 04:45:54 DEBUG } 04:45:54 DEBUG ] 04:45:54 DEBUG } 04:45:54 DEBUG } 04:45:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498354" 04:45:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498354" 04:45:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498354" 04:45:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498354" 04:45:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498354" 04:45:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498354" 04:45:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498354" 04:45:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498354" 04:45:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498354" 04:45:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498354" 04:45:54 INFO [http_cmd]: http status code OK 04:45:54 INFO [http_cmd]: http status code OK 04:45:54 DEBUG --- status code --- 04:45:54 INFO [http_cmd]: http status code OK 04:45:54 DEBUG --- status code --- 04:45:54 INFO [http_cmd]: http status code OK 04:45:54 INFO [http_cmd]: http status code OK 04:45:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:45:54 DEBUG --- status code --- 04:45:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:45:54 DEBUG --- status code --- 04:45:54 DEBUG --- status code --- 04:45:54 DEBUG --- http response --- 04:45:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:45:54 DEBUG --- http response --- 04:45:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:45:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:45:54 INFO [http_cmd]: http status code OK 04:45:54 DEBUG --- http response --- 04:45:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 04:45:54 INFO [http_cmd]: http status code OK 04:45:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 04:45:54 DEBUG --- http response --- 04:45:54 INFO [http_cmd]: http status code OK 04:45:54 INFO [http_cmd]: http status code OK 04:45:54 DEBUG --- http response --- 04:45:54 DEBUG --- status code --- 04:45:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:45:54 DEBUG ----- output ----- 04:45:54 DEBUG --- status code --- 04:45:54 DEBUG ----- output ----- 04:45:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:45:54 DEBUG --- status code --- 04:45:54 DEBUG --- status code --- 04:45:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:45:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:45:54 DEBUG ----- output ----- 04:45:54 DEBUG { 04:45:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:45:54 DEBUG { 04:45:54 DEBUG ----- output ----- 04:45:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:45:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:45:54 DEBUG --- http response --- 04:45:54 DEBUG ----- output ----- 04:45:54 DEBUG { 04:45:54 DEBUG "status": "success", 04:45:54 DEBUG --- http response --- 04:45:54 DEBUG "status": "success", 04:45:54 DEBUG { 04:45:54 DEBUG --- http response --- 04:45:54 DEBUG --- http response --- 04:45:54 DEBUG { 04:45:54 DEBUG "status": "success", 04:45:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:45:54 DEBUG "data": { 04:45:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:45:54 DEBUG "data": { 04:45:54 DEBUG "status": "success", 04:45:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:45:54 DEBUG "status": "success", 04:45:54 INFO [http_cmd]: http status code OK 04:45:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:45:54 DEBUG "data": { 04:45:54 DEBUG ----- output ----- 04:45:54 DEBUG "resultType": "vector", 04:45:54 DEBUG ----- output ----- 04:45:54 DEBUG "resultType": "vector", 04:45:54 DEBUG "data": { 04:45:54 DEBUG ----- output ----- 04:45:54 DEBUG "data": { 04:45:54 DEBUG --- status code --- 04:45:54 DEBUG ----- output ----- 04:45:54 DEBUG "resultType": "vector", 04:45:54 DEBUG { 04:45:54 DEBUG "result": [ 04:45:54 DEBUG { 04:45:54 DEBUG "result": [ 04:45:54 DEBUG "resultType": "vector", 04:45:54 DEBUG { 04:45:54 DEBUG "resultType": "vector", 04:45:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:45:54 DEBUG { 04:45:54 DEBUG "result": [ 04:45:54 DEBUG "status": "success", 04:45:54 DEBUG { 04:45:54 DEBUG "status": "success", 04:45:54 DEBUG { 04:45:54 DEBUG "result": [ 04:45:54 DEBUG "status": "success", 04:45:54 DEBUG "result": [ 04:45:54 DEBUG --- http response --- 04:45:54 DEBUG "status": "success", 04:45:54 DEBUG { 04:45:54 DEBUG "data": { 04:45:54 DEBUG "metric": { 04:45:54 DEBUG "data": { 04:45:54 DEBUG "metric": { 04:45:54 DEBUG { 04:45:54 DEBUG "data": { 04:45:54 DEBUG { 04:45:54 DEBUG "data": { 04:45:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:45:54 DEBUG "metric": { 04:45:54 DEBUG "resultType": "vector", 04:45:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:45:54 DEBUG "resultType": "vector", 04:45:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:45:54 DEBUG "metric": { 04:45:54 DEBUG "resultType": "vector", 04:45:54 DEBUG "metric": { 04:45:54 DEBUG "resultType": "vector", 04:45:54 DEBUG ----- output ----- 04:45:54 DEBUG "instance": "10.142.0.107:9100" 04:45:54 DEBUG "result": [ 04:45:54 DEBUG }, 04:45:54 DEBUG "result": [ 04:45:54 DEBUG }, 04:45:54 DEBUG "instance": "10.142.0.107:9100" 04:45:54 DEBUG "result": [ 04:45:54 DEBUG "pod": "ds-idrepo-0" 04:45:54 DEBUG "result": [ 04:45:54 DEBUG { 04:45:54 DEBUG }, 04:45:54 DEBUG { 04:45:54 DEBUG "value": [ 04:45:54 DEBUG { 04:45:54 DEBUG "value": [ 04:45:54 DEBUG }, 04:45:54 DEBUG { 04:45:54 DEBUG }, 04:45:54 DEBUG { 04:45:54 DEBUG "status": "success", 04:45:54 DEBUG "value": [ 04:45:54 DEBUG "metric": { 04:45:54 DEBUG 04:45:54 DEBUG "metric": { 04:45:54 DEBUG 04:45:54 DEBUG "value": [ 04:45:54 DEBUG "metric": { 04:45:54 DEBUG "value": [ 04:45:54 DEBUG "metric": { 04:45:54 DEBUG "data": { 04:45:54 DEBUG 04:45:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:45:54 DEBUG 04:45:54 DEBUG "instance": "10.142.0.107:9100" 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:45:54 DEBUG 04:45:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:45:54 DEBUG "resultType": "vector", 04:45:54 DEBUG 04:45:54 DEBUG }, 04:45:54 DEBUG 04:45:54 DEBUG }, 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG }, 04:45:54 DEBUG 04:45:54 DEBUG }, 04:45:54 DEBUG "result": [ 04:45:54 DEBUG 04:45:54 DEBUG "value": [ 04:45:54 DEBUG [...] 04:45:54 DEBUG "value": [ 04:45:54 DEBUG [...] 04:45:54 DEBUG 04:45:54 DEBUG "value": [ 04:45:54 DEBUG 04:45:54 DEBUG "value": [ 04:45:54 DEBUG { 04:45:54 DEBUG [...] 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG [...] 04:45:54 DEBUG 04:45:54 DEBUG [...] 04:45:54 DEBUG 04:45:54 DEBUG "metric": { 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG "pod": "ds-idrepo-0" 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG }, 04:45:54 DEBUG 04:45:54 DEBUG [...] 04:45:54 DEBUG "pod": "ds-cts-0" 04:45:54 DEBUG [...] 04:45:54 DEBUG "pod": "ds-cts-0" 04:45:54 DEBUG 04:45:54 DEBUG [...] 04:45:54 DEBUG 04:45:54 DEBUG [...] 04:45:54 DEBUG "value": [ 04:45:54 DEBUG "instance": "10.142.0.43:9100" 04:45:54 DEBUG 04:45:54 DEBUG }, 04:45:54 DEBUG 04:45:54 DEBUG }, 04:45:54 DEBUG "instance": "10.142.0.43:9100" 04:45:54 DEBUG 04:45:54 DEBUG "pod": "ds-idrepo-2" 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG }, 04:45:54 DEBUG 04:45:54 DEBUG "value": [ 04:45:54 DEBUG 04:45:54 DEBUG "value": [ 04:45:54 DEBUG }, 04:45:54 DEBUG 04:45:54 DEBUG }, 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG "value": [ 04:45:54 DEBUG 04:45:54 DEBUG 1713498354, 04:45:54 DEBUG 04:45:54 DEBUG 1713498354, 04:45:54 DEBUG "value": [ 04:45:54 DEBUG 04:45:54 DEBUG "value": [ 04:45:54 DEBUG 04:45:54 DEBUG 04:45:54 DEBUG 1713498354, 04:45:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:45:54 DEBUG "0" 04:45:54 DEBUG "instance": "10.142.0.43:9100" 04:45:54 DEBUG "0" 04:45:54 DEBUG 1713498354, 04:45:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:45:54 DEBUG 1713498354, 04:45:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:45:54 DEBUG [...] 04:45:54 DEBUG "47958675.43333334" 04:45:54 DEBUG }, 04:45:54 DEBUG ] 04:45:54 DEBUG }, 04:45:54 DEBUG ] 04:45:54 DEBUG "47954697.83333333" 04:45:54 DEBUG }, 04:45:54 DEBUG "0" 04:45:54 DEBUG }, 04:45:54 DEBUG 04:45:54 DEBUG ] 04:45:54 DEBUG "value": [ 04:45:54 DEBUG } 04:45:54 DEBUG "value": [ 04:45:54 DEBUG } 04:45:54 DEBUG ] 04:45:54 DEBUG "value": [ 04:45:54 DEBUG ] 04:45:54 DEBUG "value": [ 04:45:54 DEBUG 04:45:54 DEBUG } 04:45:54 DEBUG 1713498354, 04:45:54 DEBUG ] 04:45:54 DEBUG 1713498354, 04:45:54 DEBUG ] 04:45:54 DEBUG } 04:45:54 DEBUG 1713498354, 04:45:54 DEBUG } 04:45:54 INFO [loop_until]: OK (rc = 0) 04:45:54 DEBUG 1713498354, 04:45:54 DEBUG 04:45:54 DEBUG ] 04:45:54 DEBUG "1998.6" 04:45:54 DEBUG } 04:45:54 DEBUG "389061647360" 04:45:54 DEBUG } 04:45:54 DEBUG ] 04:45:54 DEBUG "101225.5" 04:45:54 DEBUG ] 04:45:54 DEBUG --- stdout --- 04:45:54 DEBUG "0" 04:45:54 DEBUG "pod": "ds-idrepo-2" 04:45:54 DEBUG } 04:45:54 DEBUG ] 04:45:54 DEBUG } 04:45:54 DEBUG ] 04:45:54 DEBUG } 04:45:54 DEBUG } 04:45:54 DEBUG } 04:45:54 DEBUG ] 04:45:54 DEBUG ] 04:45:54 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 79m 0% 6257Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 4569m 15% 6703Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9461m 31% 11593Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 80m 0% 3274Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 5790m 19% 5942Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 158m 0% 5190Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 9788m 32% 11140Mi 9% gke-xlou-cdm-ds-0f601751-cf7b 10292m 34% 2716Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 10917m 36% 10746Mi 9% gke-xlou-cdm-ds-0f601751-j488 4884m 16% 6150Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 4961m 16% 15327Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 4761m 15% 6874Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 5107m 17% 13672Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 6697m 42% 5715Mi 9% 04:45:54 DEBUG }, 04:45:54 DEBUG } 04:45:54 DEBUG } 04:45:54 DEBUG } 04:45:54 DEBUG } 04:45:54 DEBUG } 04:45:54 DEBUG } 04:45:54 DEBUG --- stderr --- 04:45:54 DEBUG "value": [ 04:45:54 DEBUG ] 04:45:54 DEBUG ] 04:45:54 DEBUG } 04:45:54 DEBUG ] 04:45:54 DEBUG ] 04:45:54 DEBUG 04:45:54 DEBUG 1713498354, 04:45:54 DEBUG } 04:45:54 DEBUG } 04:45:54 DEBUG } 04:45:54 DEBUG } 04:45:54 DEBUG "0" 04:45:54 DEBUG } 04:45:54 DEBUG } 04:45:54 DEBUG } 04:45:54 DEBUG } 04:45:54 DEBUG ] 04:45:54 DEBUG } 04:45:54 DEBUG ] 04:45:54 DEBUG } 04:45:54 DEBUG } 04:45:54 INFO [loop_until]: OK (rc = 0) 04:45:54 DEBUG --- stdout --- 04:45:54 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 9913m 8360Mi am-6fd554f8f7-hvvkw 9642m 8868Mi am-6fd554f8f7-pmj4b 10110m 8544Mi ds-cts-0 5781m 3281Mi ds-cts-1 4866m 11260Mi ds-cts-2 5164m 11655Mi ds-idrepo-0 4624m 3262Mi ds-idrepo-1 4778m 3206Mi ds-idrepo-2 4518m 3066Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 3m 1215Mi lodemon-6b89d774d5-rf7sk 9m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 6556m 1937Mi 04:45:54 DEBUG --- stderr --- 04:45:54 DEBUG 04:46:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713498414" 04:46:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713498414" 04:46:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498414" 04:46:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498414" 04:46:54 ERROR [http_cmd]: ERROR 04:46:54 ERROR --- status code --- 04:46:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 04:46:54 ERROR --- http response --- 04:46:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 04:46:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 04:46:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498414" 04:46:54 INFO [http_cmd]: http status code OK 04:46:54 DEBUG --- status code --- 04:46:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:46:54 DEBUG --- http response --- 04:46:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 04:46:54 DEBUG ----- output ----- 04:46:54 DEBUG { 04:46:54 DEBUG "status": "success", 04:46:54 DEBUG "data": { 04:46:54 DEBUG "resultType": "vector", 04:46:54 DEBUG "result": [ 04:46:54 DEBUG { 04:46:54 DEBUG "metric": { 04:46:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:46:54 DEBUG }, 04:46:54 DEBUG "value": [ 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG [...] 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:46:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498414" 04:46:54 DEBUG }, 04:46:54 INFO [http_cmd]: http status code OK 04:46:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713498414" 04:46:54 DEBUG "value": [ 04:46:54 DEBUG --- status code --- 04:46:54 DEBUG 1713498414, 04:46:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:46:54 DEBUG "425.6111937510613" 04:46:54 DEBUG --- http response --- 04:46:54 DEBUG ] 04:46:54 DEBUG } 04:46:54 DEBUG ] 04:46:54 DEBUG } 04:46:54 DEBUG } 04:46:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:46:54 DEBUG ----- output ----- 04:46:54 DEBUG { 04:46:54 DEBUG "status": "success", 04:46:54 DEBUG "data": { 04:46:54 DEBUG "resultType": "vector", 04:46:54 DEBUG "result": [ 04:46:54 DEBUG { 04:46:54 DEBUG "metric": { 04:46:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 04:46:54 DEBUG }, 04:46:54 DEBUG "value": [ 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG [...] 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 04:46:54 DEBUG }, 04:46:54 DEBUG "value": [ 04:46:54 DEBUG 1713498414, 04:46:54 DEBUG "6.4991007953086735" 04:46:54 DEBUG ] 04:46:54 DEBUG } 04:46:54 DEBUG ] 04:46:54 DEBUG } 04:46:54 DEBUG } 04:46:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713498414" 04:46:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498414" 04:46:54 INFO [http_cmd]: http status code OK 04:46:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498414" 04:46:54 DEBUG --- status code --- 04:46:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:46:54 DEBUG --- http response --- 04:46:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:46:54 INFO [http_cmd]: http status code OK 04:46:54 INFO [http_cmd]: http status code OK 04:46:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498414" 04:46:54 DEBUG ----- output ----- 04:46:54 INFO [http_cmd]: http status code OK 04:46:54 DEBUG --- status code --- 04:46:54 DEBUG --- status code --- 04:46:54 DEBUG { 04:46:54 DEBUG --- status code --- 04:46:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:46:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:46:54 DEBUG "status": "success", 04:46:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:46:54 DEBUG --- http response --- 04:46:54 DEBUG --- http response --- 04:46:54 DEBUG "data": { 04:46:54 DEBUG --- http response --- 04:46:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 04:46:54 DEBUG "resultType": "vector", 04:46:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:46:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:46:54 DEBUG ----- output ----- 04:46:54 DEBUG "result": [ 04:46:54 DEBUG ----- output ----- 04:46:54 DEBUG ----- output ----- 04:46:54 DEBUG { 04:46:54 DEBUG { 04:46:54 DEBUG { 04:46:54 DEBUG { 04:46:54 DEBUG "status": "success", 04:46:54 DEBUG "metric": { 04:46:54 DEBUG "status": "success", 04:46:54 DEBUG "status": "success", 04:46:54 DEBUG "data": { 04:46:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:46:54 DEBUG "data": { 04:46:54 DEBUG "data": { 04:46:54 DEBUG "resultType": "vector", 04:46:54 DEBUG }, 04:46:54 DEBUG "resultType": "vector", 04:46:54 DEBUG "resultType": "vector", 04:46:54 DEBUG "result": [ 04:46:54 DEBUG "value": [ 04:46:54 DEBUG "result": [ 04:46:54 DEBUG "result": [ 04:46:54 DEBUG { 04:46:54 DEBUG 04:46:54 DEBUG { 04:46:54 DEBUG { 04:46:54 DEBUG "metric": { 04:46:54 DEBUG 04:46:54 DEBUG "metric": { 04:46:54 DEBUG "metric": { 04:46:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:46:54 DEBUG 04:46:54 DEBUG "pod": "ds-idrepo-0" 04:46:54 DEBUG "pod": "ds-cts-0" 04:46:54 DEBUG }, 04:46:54 DEBUG [...] 04:46:54 INFO [http_cmd]: http status code OK 04:46:54 DEBUG }, 04:46:54 DEBUG }, 04:46:54 DEBUG "value": [ 04:46:54 DEBUG 04:46:54 DEBUG --- status code --- 04:46:54 DEBUG "value": [ 04:46:54 DEBUG "value": [ 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG --- http response --- 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:46:54 INFO [http_cmd]: http status code OK 04:46:54 INFO [http_cmd]: http status code OK 04:46:54 DEBUG [...] 04:46:54 DEBUG }, 04:46:54 DEBUG [...] 04:46:54 DEBUG [...] 04:46:54 DEBUG ----- output ----- 04:46:54 DEBUG --- status code --- 04:46:54 DEBUG --- status code --- 04:46:54 DEBUG 04:46:54 DEBUG "value": [ 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG { 04:46:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:46:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:46:54 DEBUG 04:46:54 DEBUG 1713498414, 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG "status": "success", 04:46:54 DEBUG --- http response --- 04:46:54 DEBUG --- http response --- 04:46:54 DEBUG 04:46:54 INFO [http_cmd]: http status code OK 04:46:54 DEBUG "1995.4666666666667" 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG "data": { 04:46:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:46:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:46:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:46:54 DEBUG --- status code --- 04:46:54 DEBUG ] 04:46:54 DEBUG "pod": "ds-cts-2" 04:46:54 DEBUG "pod": "ds-cts-2" 04:46:54 DEBUG "resultType": "vector", 04:46:54 DEBUG ----- output ----- 04:46:54 DEBUG }, 04:46:54 DEBUG ----- output ----- 04:46:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:46:54 DEBUG } 04:46:54 DEBUG }, 04:46:54 DEBUG }, 04:46:54 DEBUG "result": [ 04:46:54 DEBUG { 04:46:54 DEBUG "value": [ 04:46:54 DEBUG { 04:46:54 DEBUG --- http response --- 04:46:54 DEBUG ] 04:46:54 DEBUG "value": [ 04:46:54 DEBUG "value": [ 04:46:54 DEBUG { 04:46:54 DEBUG "status": "success", 04:46:54 DEBUG 1713498414, 04:46:54 DEBUG "status": "success", 04:46:54 DEBUG } 04:46:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:46:54 DEBUG 1713498414, 04:46:54 DEBUG 1713498414, 04:46:54 DEBUG "metric": { 04:46:54 DEBUG "data": { 04:46:54 DEBUG "0" 04:46:54 DEBUG "data": { 04:46:54 DEBUG } 04:46:54 DEBUG ----- output ----- 04:46:54 DEBUG "0" 04:46:54 DEBUG "0" 04:46:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 04:46:54 DEBUG "resultType": "vector", 04:46:54 DEBUG ] 04:46:54 DEBUG "resultType": "vector", 04:46:54 DEBUG { 04:46:54 DEBUG ] 04:46:54 DEBUG ] 04:46:54 DEBUG }, 04:46:54 DEBUG "result": [ 04:46:54 DEBUG } 04:46:54 DEBUG "result": [ 04:46:54 DEBUG "status": "success", 04:46:54 DEBUG } 04:46:54 DEBUG } 04:46:54 DEBUG "value": [ 04:46:54 DEBUG { 04:46:54 DEBUG ] 04:46:54 DEBUG { 04:46:54 DEBUG "data": { 04:46:54 DEBUG ] 04:46:54 DEBUG ] 04:46:54 DEBUG 04:46:54 DEBUG "metric": { 04:46:54 DEBUG } 04:46:54 DEBUG "metric": { 04:46:54 DEBUG "resultType": "vector", 04:46:54 DEBUG } 04:46:54 DEBUG } 04:46:54 DEBUG 04:46:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498414" 04:46:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:46:54 DEBUG } 04:46:54 DEBUG "instance": "10.142.0.107:9100" 04:46:54 DEBUG "result": [ 04:46:54 DEBUG } 04:46:54 DEBUG } 04:46:54 DEBUG 04:46:54 DEBUG }, 04:46:54 DEBUG }, 04:46:54 DEBUG { 04:46:54 DEBUG [...] 04:46:54 DEBUG "value": [ 04:46:54 DEBUG "value": [ 04:46:54 DEBUG "metric": { 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG "pod": "ds-cts-0" 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG }, 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG "value": [ 04:46:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 04:46:54 DEBUG [...] 04:46:54 DEBUG [...] 04:46:54 DEBUG 04:46:54 DEBUG }, 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG "value": [ 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 1713498414, 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG [...] 04:46:54 DEBUG "2132983808" 04:46:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:46:54 DEBUG "instance": "10.142.0.43:9100" 04:46:54 DEBUG 04:46:54 DEBUG ] 04:46:54 DEBUG }, 04:46:54 DEBUG }, 04:46:54 DEBUG 04:46:54 DEBUG } 04:46:54 DEBUG "value": [ 04:46:54 DEBUG "value": [ 04:46:54 DEBUG 04:46:54 DEBUG ] 04:46:54 DEBUG 1713498414, 04:46:54 DEBUG 1713498414, 04:46:54 DEBUG "pod": "ds-cts-2" 04:46:54 DEBUG } 04:46:54 DEBUG "1995.8666666666666" 04:46:54 DEBUG "0.042866666666667656" 04:46:54 DEBUG }, 04:46:54 DEBUG } 04:46:54 DEBUG ] 04:46:54 DEBUG ] 04:46:54 DEBUG "value": [ 04:46:54 DEBUG } 04:46:54 DEBUG } 04:46:54 DEBUG 1713498414, 04:46:54 DEBUG ] 04:46:54 DEBUG ] 04:46:54 DEBUG "0" 04:46:54 DEBUG } 04:46:54 DEBUG } 04:46:54 DEBUG ] 04:46:54 DEBUG } 04:46:54 DEBUG } 04:46:54 DEBUG } 04:46:54 DEBUG ] 04:46:54 DEBUG } 04:46:54 DEBUG } 04:46:54 INFO [http_cmd]: http status code OK 04:46:54 DEBUG --- status code --- 04:46:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:46:54 DEBUG --- http response --- 04:46:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:46:54 DEBUG ----- output ----- 04:46:54 DEBUG { 04:46:54 DEBUG "status": "success", 04:46:54 DEBUG "data": { 04:46:54 DEBUG "resultType": "vector", 04:46:54 DEBUG "result": [ 04:46:54 DEBUG { 04:46:54 DEBUG "metric": { 04:46:54 DEBUG "pod": "ds-cts-0" 04:46:54 DEBUG }, 04:46:54 DEBUG "value": [ 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG [...] 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG "pod": "ds-cts-2" 04:46:54 DEBUG }, 04:46:54 DEBUG "value": [ 04:46:54 DEBUG 1713498414, 04:46:54 DEBUG "0" 04:46:54 DEBUG ] 04:46:54 DEBUG } 04:46:54 DEBUG ] 04:46:54 DEBUG } 04:46:54 DEBUG } 04:46:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498414" 04:46:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498414" 04:46:54 INFO [http_cmd]: http status code OK 04:46:54 DEBUG --- status code --- 04:46:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:46:54 DEBUG --- http response --- 04:46:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:46:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498414" 04:46:54 DEBUG ----- output ----- 04:46:54 DEBUG { 04:46:54 DEBUG "status": "success", 04:46:54 DEBUG "data": { 04:46:54 DEBUG "resultType": "vector", 04:46:54 DEBUG "result": [ 04:46:54 DEBUG { 04:46:54 DEBUG "metric": { 04:46:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:46:54 DEBUG }, 04:46:54 DEBUG "value": [ 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG [...] 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:46:54 DEBUG }, 04:46:54 DEBUG "value": [ 04:46:54 DEBUG 1713498414, 04:46:54 DEBUG "0" 04:46:54 DEBUG ] 04:46:54 DEBUG } 04:46:54 DEBUG ] 04:46:54 DEBUG } 04:46:54 DEBUG } 04:46:54 INFO [http_cmd]: http status code OK 04:46:54 DEBUG --- status code --- 04:46:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:46:54 DEBUG --- http response --- 04:46:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:46:54 DEBUG ----- output ----- 04:46:54 DEBUG { 04:46:54 DEBUG "status": "success", 04:46:54 DEBUG "data": { 04:46:54 DEBUG "resultType": "vector", 04:46:54 DEBUG "result": [ 04:46:54 DEBUG { 04:46:54 DEBUG "metric": { 04:46:54 DEBUG "pod": "ds-idrepo-0" 04:46:54 DEBUG }, 04:46:54 DEBUG "value": [ 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG [...] 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG "pod": "ds-idrepo-2" 04:46:54 DEBUG }, 04:46:54 DEBUG "value": [ 04:46:54 DEBUG 1713498414, 04:46:54 DEBUG "0" 04:46:54 DEBUG ] 04:46:54 DEBUG } 04:46:54 DEBUG ] 04:46:54 DEBUG } 04:46:54 DEBUG } 04:46:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498414" 04:46:54 INFO [http_cmd]: http status code OK 04:46:54 DEBUG --- status code --- 04:46:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:46:54 DEBUG --- http response --- 04:46:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:46:54 DEBUG ----- output ----- 04:46:54 DEBUG { 04:46:54 DEBUG "status": "success", 04:46:54 DEBUG "data": { 04:46:54 DEBUG "resultType": "vector", 04:46:54 DEBUG "result": [ 04:46:54 DEBUG { 04:46:54 DEBUG "metric": { 04:46:54 DEBUG "instance": "10.142.0.107:9100" 04:46:54 DEBUG }, 04:46:54 DEBUG "value": [ 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG [...] 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG "instance": "10.142.0.43:9100" 04:46:54 DEBUG }, 04:46:54 DEBUG "value": [ 04:46:54 DEBUG 1713498414, 04:46:54 DEBUG "0.03199999999999979" 04:46:54 DEBUG ] 04:46:54 DEBUG } 04:46:54 DEBUG ] 04:46:54 DEBUG } 04:46:54 DEBUG } 04:46:54 INFO [http_cmd]: http status code OK 04:46:54 DEBUG --- status code --- 04:46:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:46:54 DEBUG --- http response --- 04:46:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 04:46:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713498414" 04:46:54 INFO [http_cmd]: http status code OK 04:46:54 DEBUG --- status code --- 04:46:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:46:54 DEBUG --- http response --- 04:46:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:46:54 DEBUG ----- output ----- 04:46:54 DEBUG { 04:46:54 DEBUG "status": "success", 04:46:54 DEBUG "data": { 04:46:54 DEBUG "resultType": "vector", 04:46:54 DEBUG "result": [ 04:46:54 DEBUG { 04:46:54 DEBUG "metric": { 04:46:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:46:54 DEBUG }, 04:46:54 DEBUG "value": [ 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG [...] 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:46:54 DEBUG }, 04:46:54 DEBUG "value": [ 04:46:54 DEBUG 1713498414, 04:46:54 DEBUG "55.171" 04:46:54 DEBUG ] 04:46:54 DEBUG } 04:46:54 DEBUG ] 04:46:54 DEBUG } 04:46:54 DEBUG } 04:46:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498414" 04:46:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498414" 04:46:54 INFO [http_cmd]: http status code OK 04:46:54 DEBUG --- status code --- 04:46:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:46:54 DEBUG --- http response --- 04:46:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:46:54 DEBUG ----- output ----- 04:46:54 DEBUG { 04:46:54 DEBUG "status": "success", 04:46:54 DEBUG "data": { 04:46:54 DEBUG "resultType": "vector", 04:46:54 DEBUG "result": [ 04:46:54 DEBUG { 04:46:54 DEBUG "metric": { 04:46:54 DEBUG "instance": "10.142.0.107:9100" 04:46:54 DEBUG }, 04:46:54 DEBUG "value": [ 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG [...] 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG "instance": "10.142.0.43:9100" 04:46:54 DEBUG }, 04:46:54 DEBUG "value": [ 04:46:54 DEBUG 1713498414, 04:46:54 DEBUG "47820715.5" 04:46:54 DEBUG ] 04:46:54 DEBUG } 04:46:54 DEBUG ] 04:46:54 DEBUG } 04:46:54 DEBUG } 04:46:54 INFO [http_cmd]: http status code OK 04:46:54 DEBUG --- status code --- 04:46:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:46:54 DEBUG --- http response --- 04:46:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:46:54 DEBUG ----- output ----- 04:46:54 DEBUG { 04:46:54 DEBUG "status": "success", 04:46:54 DEBUG "data": { 04:46:54 DEBUG "resultType": "vector", 04:46:54 DEBUG "result": [ 04:46:54 DEBUG { 04:46:54 DEBUG "metric": { 04:46:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:46:54 DEBUG }, 04:46:54 DEBUG "value": [ 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG [...] 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:46:54 DEBUG }, 04:46:54 DEBUG "value": [ 04:46:54 DEBUG 1713498414, 04:46:54 DEBUG "221046" 04:46:54 DEBUG ] 04:46:54 DEBUG } 04:46:54 DEBUG ] 04:46:54 DEBUG } 04:46:54 DEBUG } 04:46:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498414" 04:46:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498414" 04:46:54 INFO 04:46:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498414" 04:46:54 INFO 04:46:54 INFO [loop_until]: kubectl --namespace=xlou top node 04:46:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498414" 04:46:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498414" 04:46:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498414" 04:46:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498414" 04:46:54 INFO [loop_until]: kubectl --namespace=xlou top pods 04:46:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498414" 04:46:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:46:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:46:54 INFO [http_cmd]: http status code OK 04:46:54 INFO [http_cmd]: http status code OK 04:46:54 INFO [http_cmd]: http status code OK 04:46:54 DEBUG --- status code --- 04:46:54 DEBUG --- status code --- 04:46:54 DEBUG --- status code --- 04:46:54 INFO [http_cmd]: http status code OK 04:46:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:46:54 INFO [http_cmd]: http status code OK 04:46:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:46:54 INFO [http_cmd]: http status code OK 04:46:54 INFO [http_cmd]: http status code OK 04:46:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:46:54 DEBUG --- status code --- 04:46:54 DEBUG --- http response --- 04:46:54 INFO [http_cmd]: http status code OK 04:46:54 DEBUG --- status code --- 04:46:54 DEBUG --- http response --- 04:46:54 DEBUG --- status code --- 04:46:54 DEBUG --- status code --- 04:46:54 DEBUG --- http response --- 04:46:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:46:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 04:46:54 DEBUG --- status code --- 04:46:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:46:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 04:46:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:46:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:46:54 DEBUG --- http response --- 04:46:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:46:54 DEBUG ----- output ----- 04:46:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:46:54 DEBUG --- http response --- 04:46:54 DEBUG ----- output ----- 04:46:54 DEBUG --- http response --- 04:46:54 DEBUG --- http response --- 04:46:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:46:54 DEBUG ----- output ----- 04:46:54 DEBUG { 04:46:54 DEBUG --- http response --- 04:46:54 DEBUG { 04:46:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:46:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:46:54 DEBUG ----- output ----- 04:46:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:46:54 DEBUG { 04:46:54 DEBUG "status": "success", 04:46:54 DEBUG "status": "success", 04:46:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:46:54 DEBUG ----- output ----- 04:46:54 DEBUG ----- output ----- 04:46:54 DEBUG { 04:46:54 DEBUG ----- output ----- 04:46:54 DEBUG "status": "success", 04:46:54 DEBUG "data": { 04:46:54 DEBUG "data": { 04:46:54 DEBUG ----- output ----- 04:46:54 DEBUG { 04:46:54 DEBUG { 04:46:54 DEBUG "status": "success", 04:46:54 DEBUG { 04:46:54 DEBUG "data": { 04:46:54 DEBUG "resultType": "vector", 04:46:54 DEBUG "resultType": "vector", 04:46:54 DEBUG { 04:46:54 DEBUG "status": "success", 04:46:54 DEBUG "status": "success", 04:46:54 DEBUG "data": { 04:46:54 DEBUG "status": "success", 04:46:54 DEBUG "resultType": "vector", 04:46:54 DEBUG "result": [ 04:46:54 DEBUG "result": [ 04:46:54 DEBUG "status": "success", 04:46:54 DEBUG "data": { 04:46:54 DEBUG "data": { 04:46:54 DEBUG "resultType": "vector", 04:46:54 DEBUG "data": { 04:46:54 DEBUG "result": [ 04:46:54 DEBUG { 04:46:54 DEBUG { 04:46:54 DEBUG "data": { 04:46:54 DEBUG "resultType": "vector", 04:46:54 DEBUG "resultType": "vector", 04:46:54 DEBUG "result": [ 04:46:54 DEBUG "resultType": "vector", 04:46:54 DEBUG { 04:46:54 DEBUG "metric": { 04:46:54 DEBUG "metric": { 04:46:54 DEBUG "resultType": "vector", 04:46:54 DEBUG "result": [ 04:46:54 DEBUG "result": [ 04:46:54 DEBUG { 04:46:54 DEBUG "result": [ 04:46:54 DEBUG "metric": { 04:46:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:46:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:46:54 DEBUG "result": [ 04:46:54 DEBUG { 04:46:54 DEBUG { 04:46:54 DEBUG "metric": { 04:46:54 DEBUG { 04:46:54 DEBUG "instance": "10.142.0.107:9100" 04:46:54 DEBUG }, 04:46:54 DEBUG }, 04:46:54 DEBUG { 04:46:54 DEBUG "metric": { 04:46:54 DEBUG "metric": { 04:46:54 DEBUG "pod": "ds-idrepo-0" 04:46:54 DEBUG "metric": { 04:46:54 DEBUG }, 04:46:54 DEBUG "value": [ 04:46:54 DEBUG "value": [ 04:46:54 DEBUG "metric": { 04:46:54 DEBUG "instance": "10.142.0.107:9100" 04:46:54 DEBUG "pod": "ds-idrepo-0" 04:46:54 DEBUG }, 04:46:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:46:54 DEBUG "value": [ 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:46:54 DEBUG }, 04:46:54 DEBUG }, 04:46:54 DEBUG "value": [ 04:46:54 DEBUG }, 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG }, 04:46:54 DEBUG "value": [ 04:46:54 DEBUG "value": [ 04:46:54 DEBUG 04:46:54 DEBUG "value": [ 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG "value": [ 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG [...] 04:46:54 DEBUG [...] 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG [...] 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG [...] 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG [...] 04:46:54 DEBUG [...] 04:46:54 DEBUG 04:46:54 DEBUG [...] 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG [...] 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 04:46:54 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG "instance": "10.142.0.43:9100" 04:46:54 DEBUG }, 04:46:54 DEBUG }, 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG 04:46:54 DEBUG "pod": "ds-idrepo-2" 04:46:54 DEBUG 04:46:54 DEBUG }, 04:46:54 DEBUG "value": [ 04:46:54 DEBUG "value": [ 04:46:54 DEBUG 04:46:54 DEBUG "instance": "10.142.0.43:9100" 04:46:54 DEBUG "pod": "ds-idrepo-2" 04:46:54 DEBUG }, 04:46:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:46:54 DEBUG "value": [ 04:46:54 DEBUG 1713498414, 04:46:54 DEBUG 1713498414, 04:46:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:46:54 DEBUG }, 04:46:54 DEBUG }, 04:46:54 DEBUG "value": [ 04:46:54 DEBUG }, 04:46:54 DEBUG 1713498414, 04:46:54 DEBUG "0" 04:46:54 DEBUG "0" 04:46:54 DEBUG }, 04:46:54 DEBUG "value": [ 04:46:54 DEBUG "value": [ 04:46:54 DEBUG 1713498414, 04:46:54 DEBUG "value": [ 04:46:54 DEBUG "47818298.9" 04:46:54 DEBUG ] 04:46:54 DEBUG ] 04:46:54 DEBUG "value": [ 04:46:54 DEBUG 1713498414, 04:46:54 DEBUG 1713498414, 04:46:54 DEBUG "0" 04:46:54 DEBUG 1713498414, 04:46:54 DEBUG ] 04:46:54 DEBUG } 04:46:54 DEBUG } 04:46:54 DEBUG 1713498414, 04:46:54 DEBUG "391285480448" 04:46:54 DEBUG "0" 04:46:54 DEBUG ] 04:46:54 DEBUG "1995.6" 04:46:54 DEBUG } 04:46:54 DEBUG ] 04:46:54 DEBUG ] 04:46:54 DEBUG "0" 04:46:54 DEBUG ] 04:46:54 DEBUG ] 04:46:54 DEBUG } 04:46:54 DEBUG ] 04:46:54 DEBUG ] 04:46:54 DEBUG } 04:46:54 DEBUG } 04:46:54 DEBUG ] 04:46:54 DEBUG } 04:46:54 DEBUG } 04:46:54 DEBUG ] 04:46:54 DEBUG } 04:46:54 DEBUG } 04:46:54 DEBUG } 04:46:54 DEBUG } 04:46:54 DEBUG } 04:46:54 DEBUG ] 04:46:54 DEBUG ] 04:46:54 DEBUG } 04:46:54 DEBUG ] 04:46:54 DEBUG } 04:46:54 DEBUG ] 04:46:54 DEBUG } 04:46:54 DEBUG } 04:46:54 DEBUG } 04:46:54 DEBUG } 04:46:54 DEBUG } 04:46:54 DEBUG } 04:46:54 DEBUG } 04:46:54 DEBUG } 04:46:54 DEBUG } 04:46:54 INFO [loop_until]: OK (rc = 0) 04:46:54 DEBUG --- stdout --- 04:46:54 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 9500m 8361Mi am-6fd554f8f7-hvvkw 9120m 8869Mi am-6fd554f8f7-pmj4b 10001m 8546Mi ds-cts-0 5599m 4393Mi ds-cts-1 4987m 10866Mi ds-cts-2 5029m 11344Mi ds-idrepo-0 4577m 3262Mi ds-idrepo-1 4756m 3206Mi ds-idrepo-2 4624m 3065Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 4m 1215Mi lodemon-6b89d774d5-rf7sk 14m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 6652m 1940Mi 04:46:54 DEBUG --- stderr --- 04:46:54 DEBUG 04:46:54 INFO [loop_until]: OK (rc = 0) 04:46:54 DEBUG --- stdout --- 04:46:54 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 78m 0% 6255Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 4576m 15% 6700Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9333m 31% 11586Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 87m 0% 3261Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 5694m 19% 7186Mi 6% gke-xlou-cdm-default-pool-1b0769c8-wpwl 154m 0% 5172Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 9670m 32% 11135Mi 9% gke-xlou-cdm-ds-0f601751-cf7b 10263m 34% 2729Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 10826m 36% 10753Mi 9% gke-xlou-cdm-ds-0f601751-j488 4860m 16% 6158Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 4882m 16% 14941Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 5014m 16% 6878Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 5048m 16% 13363Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 6443m 40% 5713Mi 9% 04:46:54 DEBUG --- stderr --- 04:46:54 DEBUG 04:47:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498474" 04:47:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498474" 04:47:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713498474" 04:47:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498474" 04:47:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498474" 04:47:54 INFO [http_cmd]: http status code OK 04:47:54 DEBUG --- status code --- 04:47:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:47:54 DEBUG --- http response --- 04:47:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:47:54 DEBUG ----- output ----- 04:47:54 DEBUG { 04:47:54 DEBUG "status": "success", 04:47:54 DEBUG "data": { 04:47:54 DEBUG "resultType": "vector", 04:47:54 DEBUG "result": [ 04:47:54 DEBUG { 04:47:54 DEBUG "metric": { 04:47:54 DEBUG "pod": "ds-cts-0" 04:47:54 DEBUG }, 04:47:54 DEBUG "value": [ 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG [...] 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG "pod": "ds-cts-2" 04:47:54 DEBUG }, 04:47:54 DEBUG "value": [ 04:47:54 DEBUG 1713498474, 04:47:54 DEBUG "0" 04:47:54 DEBUG ] 04:47:54 DEBUG } 04:47:54 DEBUG ] 04:47:54 DEBUG } 04:47:54 DEBUG } 04:47:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713498474" 04:47:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713498474" 04:47:54 INFO [http_cmd]: http status code OK 04:47:54 INFO [http_cmd]: http status code OK 04:47:54 DEBUG --- status code --- 04:47:54 INFO [http_cmd]: http status code OK 04:47:54 DEBUG --- status code --- 04:47:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:47:54 DEBUG --- status code --- 04:47:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:47:54 DEBUG --- http response --- 04:47:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:47:54 DEBUG --- http response --- 04:47:54 DEBUG --- http response --- 04:47:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:47:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:47:54 DEBUG ----- output ----- 04:47:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:47:54 DEBUG ----- output ----- 04:47:54 DEBUG { 04:47:54 DEBUG ----- output ----- 04:47:54 DEBUG { 04:47:54 DEBUG "status": "success", 04:47:54 DEBUG { 04:47:54 DEBUG "status": "success", 04:47:54 DEBUG "data": { 04:47:54 DEBUG "status": "success", 04:47:54 DEBUG "data": { 04:47:54 DEBUG "resultType": "vector", 04:47:54 DEBUG "data": { 04:47:54 DEBUG "resultType": "vector", 04:47:54 DEBUG "result": [ 04:47:54 DEBUG "resultType": "vector", 04:47:54 DEBUG "result": [ 04:47:54 DEBUG { 04:47:54 DEBUG "result": [ 04:47:54 DEBUG { 04:47:54 DEBUG "metric": { 04:47:54 DEBUG { 04:47:54 DEBUG "metric": { 04:47:54 DEBUG "pod": "ds-cts-0" 04:47:54 DEBUG "metric": { 04:47:54 DEBUG "pod": "ds-cts-0" 04:47:54 DEBUG }, 04:47:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 04:47:54 DEBUG }, 04:47:54 DEBUG "value": [ 04:47:54 DEBUG }, 04:47:54 DEBUG "value": [ 04:47:54 DEBUG 04:47:54 DEBUG "value": [ 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG [...] 04:47:54 DEBUG 04:47:54 DEBUG [...] 04:47:54 DEBUG 04:47:54 DEBUG [...] 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG "pod": "ds-cts-2" 04:47:54 DEBUG 04:47:54 DEBUG "pod": "ds-cts-2" 04:47:54 DEBUG }, 04:47:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 04:47:54 DEBUG }, 04:47:54 DEBUG "value": [ 04:47:54 DEBUG }, 04:47:54 DEBUG "value": [ 04:47:54 DEBUG 1713498474, 04:47:54 DEBUG "value": [ 04:47:54 DEBUG 1713498474, 04:47:54 DEBUG "0" 04:47:54 DEBUG 1713498474, 04:47:54 DEBUG "0.00020262666666666666" 04:47:54 DEBUG ] 04:47:54 DEBUG "6.518631305334972" 04:47:54 DEBUG ] 04:47:54 DEBUG } 04:47:54 DEBUG ] 04:47:54 DEBUG } 04:47:54 DEBUG ] 04:47:54 DEBUG } 04:47:54 DEBUG ] 04:47:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498474" 04:47:54 DEBUG } 04:47:54 DEBUG ] 04:47:54 INFO [http_cmd]: http status code OK 04:47:54 DEBUG } 04:47:54 DEBUG } 04:47:54 DEBUG } 04:47:54 DEBUG --- status code --- 04:47:54 DEBUG } 04:47:54 DEBUG } 04:47:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:47:54 DEBUG --- http response --- 04:47:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:47:54 DEBUG ----- output ----- 04:47:54 DEBUG { 04:47:54 DEBUG "status": "success", 04:47:54 DEBUG "data": { 04:47:54 DEBUG "resultType": "vector", 04:47:54 DEBUG "result": [ 04:47:54 DEBUG { 04:47:54 DEBUG "metric": { 04:47:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:47:54 DEBUG }, 04:47:54 DEBUG "value": [ 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG [...] 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:47:54 DEBUG }, 04:47:54 DEBUG "value": [ 04:47:54 DEBUG 1713498474, 04:47:54 DEBUG "1992.6" 04:47:54 DEBUG ] 04:47:54 DEBUG } 04:47:54 DEBUG ] 04:47:54 DEBUG } 04:47:54 DEBUG } 04:47:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498474" 04:47:54 ERROR [http_cmd]: ERROR 04:47:54 INFO [http_cmd]: http status code OK 04:47:54 ERROR --- status code --- 04:47:54 DEBUG --- status code --- 04:47:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 04:47:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:47:54 ERROR --- http response --- 04:47:54 DEBUG --- http response --- 04:47:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 04:47:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 04:47:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:47:54 DEBUG ----- output ----- 04:47:54 DEBUG { 04:47:54 DEBUG "status": "success", 04:47:54 DEBUG "data": { 04:47:54 DEBUG "resultType": "vector", 04:47:54 DEBUG "result": [ 04:47:54 DEBUG { 04:47:54 DEBUG "metric": { 04:47:54 DEBUG "pod": "ds-idrepo-0" 04:47:54 DEBUG }, 04:47:54 DEBUG "value": [ 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG [...] 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG "pod": "ds-cts-2" 04:47:54 DEBUG }, 04:47:54 DEBUG "value": [ 04:47:54 DEBUG 1713498474, 04:47:54 DEBUG "0" 04:47:54 DEBUG ] 04:47:54 DEBUG } 04:47:54 DEBUG ] 04:47:54 DEBUG } 04:47:54 DEBUG } 04:47:54 INFO [http_cmd]: http status code OK 04:47:54 DEBUG --- status code --- 04:47:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:47:54 DEBUG --- http response --- 04:47:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498474" 04:47:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 04:47:54 DEBUG ----- output ----- 04:47:54 DEBUG { 04:47:54 DEBUG "status": "success", 04:47:54 DEBUG "data": { 04:47:54 DEBUG "resultType": "vector", 04:47:54 DEBUG "result": [ 04:47:54 DEBUG { 04:47:54 DEBUG "metric": { 04:47:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:47:54 DEBUG }, 04:47:54 DEBUG "value": [ 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG [...] 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 04:47:54 INFO [http_cmd]: http status code OK 04:47:54 DEBUG }, 04:47:54 DEBUG --- status code --- 04:47:54 DEBUG "value": [ 04:47:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:47:54 DEBUG 1713498474, 04:47:54 DEBUG --- http response --- 04:47:54 DEBUG "0" 04:47:54 DEBUG ] 04:47:54 DEBUG } 04:47:54 DEBUG ] 04:47:54 DEBUG } 04:47:54 DEBUG } 04:47:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:47:54 DEBUG ----- output ----- 04:47:54 DEBUG { 04:47:54 DEBUG "status": "success", 04:47:54 DEBUG "data": { 04:47:54 DEBUG "resultType": "vector", 04:47:54 DEBUG "result": [ 04:47:54 DEBUG { 04:47:54 DEBUG "metric": { 04:47:54 DEBUG "instance": "10.142.0.107:9100" 04:47:54 DEBUG }, 04:47:54 DEBUG "value": [ 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG [...] 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG "instance": "10.142.0.43:9100" 04:47:54 DEBUG }, 04:47:54 DEBUG "value": [ 04:47:54 DEBUG 1713498474, 04:47:54 DEBUG "0.031000000000002602" 04:47:54 DEBUG ] 04:47:54 DEBUG } 04:47:54 DEBUG ] 04:47:54 DEBUG } 04:47:54 DEBUG } 04:47:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498474" 04:47:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498474" 04:47:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713498474" 04:47:54 INFO [http_cmd]: http status code OK 04:47:54 DEBUG --- status code --- 04:47:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:47:54 DEBUG --- http response --- 04:47:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:47:54 DEBUG ----- output ----- 04:47:54 DEBUG { 04:47:54 DEBUG "status": "success", 04:47:54 DEBUG "data": { 04:47:54 DEBUG "resultType": "vector", 04:47:54 DEBUG "result": [ 04:47:54 DEBUG { 04:47:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498474" 04:47:54 DEBUG "metric": { 04:47:54 DEBUG "pod": "ds-idrepo-0" 04:47:54 DEBUG }, 04:47:54 DEBUG "value": [ 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG [...] 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG "pod": "ds-idrepo-2" 04:47:54 DEBUG }, 04:47:54 DEBUG "value": [ 04:47:54 DEBUG 1713498474, 04:47:54 DEBUG "0" 04:47:54 DEBUG ] 04:47:54 DEBUG } 04:47:54 DEBUG ] 04:47:54 DEBUG } 04:47:54 DEBUG } 04:47:54 INFO [http_cmd]: http status code OK 04:47:54 DEBUG --- status code --- 04:47:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:47:54 DEBUG --- http response --- 04:47:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:47:54 DEBUG ----- output ----- 04:47:54 DEBUG { 04:47:54 DEBUG "status": "success", 04:47:54 DEBUG "data": { 04:47:54 DEBUG "resultType": "vector", 04:47:54 DEBUG "result": [ 04:47:54 DEBUG { 04:47:54 DEBUG "metric": { 04:47:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:47:54 DEBUG }, 04:47:54 DEBUG "value": [ 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG [...] 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:47:54 DEBUG }, 04:47:54 DEBUG "value": [ 04:47:54 DEBUG 1713498474, 04:47:54 DEBUG "1992.7333333333333" 04:47:54 DEBUG ] 04:47:54 DEBUG } 04:47:54 DEBUG ] 04:47:54 DEBUG } 04:47:54 DEBUG } 04:47:54 INFO [http_cmd]: http status code OK 04:47:54 DEBUG --- status code --- 04:47:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:47:54 DEBUG --- http response --- 04:47:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 04:47:54 DEBUG ----- output ----- 04:47:54 DEBUG { 04:47:54 DEBUG "status": "success", 04:47:54 DEBUG "data": { 04:47:54 DEBUG "resultType": "vector", 04:47:54 DEBUG "result": [ 04:47:54 DEBUG { 04:47:54 DEBUG "metric": { 04:47:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:47:54 DEBUG }, 04:47:54 DEBUG "value": [ 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG [...] 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 04:47:54 DEBUG }, 04:47:54 DEBUG "value": [ 04:47:54 DEBUG 1713498474, 04:47:54 DEBUG "0" 04:47:54 DEBUG ] 04:47:54 DEBUG } 04:47:54 DEBUG ] 04:47:54 DEBUG } 04:47:54 DEBUG } 04:47:54 INFO [http_cmd]: http status code OK 04:47:54 DEBUG --- status code --- 04:47:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:47:54 DEBUG --- http response --- 04:47:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:47:54 DEBUG ----- output ----- 04:47:54 DEBUG { 04:47:54 DEBUG "status": "success", 04:47:54 DEBUG "data": { 04:47:54 DEBUG "resultType": "vector", 04:47:54 DEBUG "result": [ 04:47:54 DEBUG { 04:47:54 DEBUG "metric": { 04:47:54 DEBUG "instance": "10.142.0.107:9100" 04:47:54 DEBUG }, 04:47:54 DEBUG "value": [ 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG [...] 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG "instance": "10.142.0.43:9100" 04:47:54 DEBUG }, 04:47:54 DEBUG "value": [ 04:47:54 DEBUG 1713498474, 04:47:54 DEBUG "0.042300000000012494" 04:47:54 DEBUG ] 04:47:54 DEBUG } 04:47:54 DEBUG ] 04:47:54 DEBUG } 04:47:54 DEBUG } 04:47:54 INFO [http_cmd]: http status code OK 04:47:54 DEBUG --- status code --- 04:47:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:47:54 DEBUG --- http response --- 04:47:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:47:54 DEBUG ----- output ----- 04:47:54 DEBUG { 04:47:54 DEBUG "status": "success", 04:47:54 DEBUG "data": { 04:47:54 DEBUG "resultType": "vector", 04:47:54 DEBUG "result": [ 04:47:54 DEBUG { 04:47:54 DEBUG "metric": { 04:47:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 04:47:54 DEBUG }, 04:47:54 DEBUG "value": [ 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG [...] 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 04:47:54 DEBUG }, 04:47:54 DEBUG "value": [ 04:47:54 DEBUG 1713498474, 04:47:54 DEBUG "2134147072" 04:47:54 DEBUG ] 04:47:54 DEBUG } 04:47:54 DEBUG ] 04:47:54 DEBUG } 04:47:54 DEBUG } 04:47:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498474" 04:47:54 INFO [http_cmd]: http status code OK 04:47:54 DEBUG --- status code --- 04:47:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:47:54 DEBUG --- http response --- 04:47:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:47:54 DEBUG ----- output ----- 04:47:54 DEBUG { 04:47:54 DEBUG "status": "success", 04:47:54 DEBUG "data": { 04:47:54 DEBUG "resultType": "vector", 04:47:54 DEBUG "result": [ 04:47:54 DEBUG { 04:47:54 DEBUG "metric": { 04:47:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:47:54 DEBUG }, 04:47:54 DEBUG "value": [ 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG [...] 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:47:54 DEBUG }, 04:47:54 DEBUG "value": [ 04:47:54 DEBUG 1713498474, 04:47:54 DEBUG "0" 04:47:54 DEBUG ] 04:47:54 DEBUG } 04:47:54 DEBUG ] 04:47:54 DEBUG } 04:47:54 DEBUG } 04:47:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498474" 04:47:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713498474" 04:47:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498474" 04:47:54 INFO [http_cmd]: http status code OK 04:47:54 INFO [http_cmd]: http status code OK 04:47:54 DEBUG --- status code --- 04:47:54 DEBUG --- status code --- 04:47:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:47:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:47:54 DEBUG --- http response --- 04:47:54 DEBUG --- http response --- 04:47:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:47:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:47:54 DEBUG ----- output ----- 04:47:54 DEBUG ----- output ----- 04:47:54 DEBUG { 04:47:54 DEBUG { 04:47:54 DEBUG "status": "success", 04:47:54 DEBUG "status": "success", 04:47:54 DEBUG "data": { 04:47:54 DEBUG "data": { 04:47:54 DEBUG "resultType": "vector", 04:47:54 DEBUG "resultType": "vector", 04:47:54 DEBUG "result": [ 04:47:54 DEBUG "result": [ 04:47:54 DEBUG { 04:47:54 DEBUG { 04:47:54 DEBUG "metric": { 04:47:54 DEBUG "metric": { 04:47:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:47:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:47:54 DEBUG }, 04:47:54 DEBUG }, 04:47:54 DEBUG "value": [ 04:47:54 DEBUG "value": [ 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG [...] 04:47:54 DEBUG [...] 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:47:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:47:54 DEBUG }, 04:47:54 DEBUG }, 04:47:54 DEBUG "value": [ 04:47:54 DEBUG "value": [ 04:47:54 DEBUG 1713498474, 04:47:54 DEBUG 1713498474, 04:47:54 DEBUG "340692" 04:47:54 DEBUG "55.171" 04:47:54 DEBUG ] 04:47:54 DEBUG ] 04:47:54 DEBUG } 04:47:54 DEBUG } 04:47:54 DEBUG ] 04:47:54 DEBUG ] 04:47:54 DEBUG } 04:47:54 DEBUG } 04:47:54 DEBUG } 04:47:54 DEBUG } 04:47:54 INFO [http_cmd]: http status code OK 04:47:54 DEBUG --- status code --- 04:47:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:47:54 DEBUG --- http response --- 04:47:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 04:47:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498474" 04:47:54 INFO [http_cmd]: http status code OK 04:47:54 DEBUG --- status code --- 04:47:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:47:54 DEBUG --- http response --- 04:47:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:47:54 DEBUG ----- output ----- 04:47:54 DEBUG { 04:47:54 DEBUG "status": "success", 04:47:54 DEBUG "data": { 04:47:54 DEBUG "resultType": "vector", 04:47:54 DEBUG "result": [ 04:47:54 DEBUG { 04:47:54 DEBUG "metric": { 04:47:54 DEBUG "instance": "10.142.0.107:9100" 04:47:54 DEBUG }, 04:47:54 DEBUG "value": [ 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG [...] 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG "instance": "10.142.0.43:9100" 04:47:54 DEBUG }, 04:47:54 DEBUG "value": [ 04:47:54 DEBUG 1713498474, 04:47:54 DEBUG "47750992.1" 04:47:54 DEBUG ] 04:47:54 DEBUG } 04:47:54 DEBUG ] 04:47:54 DEBUG } 04:47:54 DEBUG } 04:47:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498474" 04:47:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498474" 04:47:54 INFO [http_cmd]: http status code OK 04:47:54 DEBUG --- status code --- 04:47:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:47:54 DEBUG --- http response --- 04:47:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:47:54 DEBUG ----- output ----- 04:47:54 DEBUG { 04:47:54 DEBUG "status": "success", 04:47:54 DEBUG "data": { 04:47:54 DEBUG "resultType": "vector", 04:47:54 DEBUG "result": [ 04:47:54 DEBUG { 04:47:54 DEBUG "metric": { 04:47:54 DEBUG "pod": "ds-idrepo-0" 04:47:54 DEBUG }, 04:47:54 DEBUG "value": [ 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG [...] 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG "pod": "ds-idrepo-2" 04:47:54 DEBUG }, 04:47:54 DEBUG "value": [ 04:47:54 DEBUG 1713498474, 04:47:54 DEBUG "0" 04:47:54 DEBUG ] 04:47:54 DEBUG } 04:47:54 DEBUG ] 04:47:54 DEBUG } 04:47:54 DEBUG } 04:47:54 INFO [http_cmd]: http status code OK 04:47:54 DEBUG --- status code --- 04:47:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:47:54 DEBUG --- http response --- 04:47:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:47:54 DEBUG ----- output ----- 04:47:54 DEBUG { 04:47:54 DEBUG "status": "success", 04:47:54 DEBUG "data": { 04:47:54 DEBUG "resultType": "vector", 04:47:54 DEBUG "result": [ 04:47:54 DEBUG { 04:47:54 DEBUG "metric": { 04:47:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:47:54 DEBUG }, 04:47:54 DEBUG "value": [ 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG [...] 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:47:54 DEBUG }, 04:47:54 DEBUG "value": [ 04:47:54 DEBUG 1713498474, 04:47:54 DEBUG "0" 04:47:54 DEBUG ] 04:47:54 DEBUG } 04:47:54 DEBUG ] 04:47:54 DEBUG } 04:47:54 DEBUG } 04:47:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498474" 04:47:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498474" 04:47:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498474" 04:47:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498474" 04:47:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498474" 04:47:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498474" 04:47:54 INFO [http_cmd]: http status code OK 04:47:54 DEBUG --- status code --- 04:47:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:47:54 DEBUG --- http response --- 04:47:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:47:54 DEBUG ----- output ----- 04:47:54 DEBUG { 04:47:54 DEBUG "status": "success", 04:47:54 DEBUG "data": { 04:47:54 DEBUG "resultType": "vector", 04:47:54 DEBUG "result": [ 04:47:54 DEBUG { 04:47:54 DEBUG "metric": { 04:47:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:47:54 DEBUG }, 04:47:54 DEBUG "value": [ 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG [...] 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:47:54 DEBUG }, 04:47:54 DEBUG "value": [ 04:47:54 DEBUG 1713498474, 04:47:54 DEBUG "1992.6" 04:47:54 DEBUG ] 04:47:54 DEBUG } 04:47:54 DEBUG ] 04:47:54 DEBUG } 04:47:54 DEBUG } 04:47:54 INFO [http_cmd]: http status code OK 04:47:54 DEBUG --- status code --- 04:47:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:47:54 DEBUG --- http response --- 04:47:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 04:47:54 DEBUG ----- output ----- 04:47:54 DEBUG { 04:47:54 DEBUG "status": "success", 04:47:54 DEBUG "data": { 04:47:54 DEBUG "resultType": "vector", 04:47:54 DEBUG "result": [ 04:47:54 DEBUG { 04:47:54 DEBUG "metric": { 04:47:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:47:54 DEBUG }, 04:47:54 DEBUG "value": [ 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG [...] 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG "pod": "ds-cts-0" 04:47:54 DEBUG }, 04:47:54 DEBUG "value": [ 04:47:54 DEBUG 1713498474, 04:47:54 DEBUG "0" 04:47:54 DEBUG ] 04:47:54 DEBUG } 04:47:54 DEBUG ] 04:47:54 DEBUG } 04:47:54 DEBUG } 04:47:54 INFO [http_cmd]: http status code OK 04:47:54 DEBUG --- status code --- 04:47:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:47:54 DEBUG --- http response --- 04:47:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:47:54 DEBUG ----- output ----- 04:47:54 DEBUG { 04:47:54 DEBUG "status": "success", 04:47:54 DEBUG "data": { 04:47:54 DEBUG "resultType": "vector", 04:47:54 DEBUG "result": [ 04:47:54 DEBUG { 04:47:54 DEBUG "metric": { 04:47:54 DEBUG "instance": "10.142.0.107:9100" 04:47:54 DEBUG }, 04:47:54 DEBUG "value": [ 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG [...] 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG "instance": "10.142.0.43:9100" 04:47:54 DEBUG }, 04:47:54 DEBUG "value": [ 04:47:54 DEBUG 1713498474, 04:47:54 INFO [http_cmd]: http status code OK 04:47:54 DEBUG "47746943.03333333" 04:47:54 DEBUG --- status code --- 04:47:54 DEBUG ] 04:47:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:47:54 DEBUG } 04:47:54 DEBUG --- http response --- 04:47:54 DEBUG ] 04:47:54 DEBUG } 04:47:54 DEBUG } 04:47:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:47:54 DEBUG ----- output ----- 04:47:54 DEBUG { 04:47:54 DEBUG "status": "success", 04:47:54 DEBUG "data": { 04:47:54 DEBUG "resultType": "vector", 04:47:54 DEBUG "result": [ 04:47:54 DEBUG { 04:47:54 DEBUG "metric": { 04:47:54 DEBUG "pod": "ds-idrepo-0" 04:47:54 DEBUG }, 04:47:54 DEBUG "value": [ 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG [...] 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG "pod": "ds-idrepo-2" 04:47:54 DEBUG }, 04:47:54 DEBUG "value": [ 04:47:54 DEBUG 1713498474, 04:47:54 DEBUG "0" 04:47:54 DEBUG ] 04:47:54 DEBUG } 04:47:54 DEBUG ] 04:47:54 DEBUG } 04:47:54 DEBUG } 04:47:54 INFO [http_cmd]: http status code OK 04:47:54 DEBUG --- status code --- 04:47:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:47:54 DEBUG --- http response --- 04:47:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 04:47:54 INFO [http_cmd]: http status code OK 04:47:54 DEBUG ----- output ----- 04:47:54 DEBUG --- status code --- 04:47:54 DEBUG { 04:47:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:47:54 DEBUG "status": "success", 04:47:54 DEBUG --- http response --- 04:47:54 DEBUG "data": { 04:47:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:47:54 DEBUG "resultType": "vector", 04:47:54 DEBUG ----- output ----- 04:47:54 DEBUG "result": [ 04:47:54 DEBUG { 04:47:54 DEBUG { 04:47:54 DEBUG "status": "success", 04:47:54 DEBUG "metric": { 04:47:54 DEBUG "data": { 04:47:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:47:54 DEBUG "resultType": "vector", 04:47:54 DEBUG }, 04:47:54 DEBUG "result": [ 04:47:54 DEBUG "value": [ 04:47:54 DEBUG { 04:47:54 DEBUG 04:47:54 DEBUG "metric": { 04:47:54 DEBUG 04:47:54 DEBUG "instance": "10.142.0.107:9100" 04:47:54 DEBUG 04:47:54 DEBUG }, 04:47:54 DEBUG [...] 04:47:54 DEBUG "value": [ 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG 04:47:54 DEBUG "pod": "ds-cts-0" 04:47:54 DEBUG [...] 04:47:54 DEBUG }, 04:47:54 DEBUG 04:47:54 DEBUG "value": [ 04:47:54 DEBUG 04:47:54 DEBUG 1713498474, 04:47:54 DEBUG 04:47:54 DEBUG "0" 04:47:54 DEBUG "instance": "10.142.0.43:9100" 04:47:54 DEBUG ] 04:47:54 DEBUG }, 04:47:54 DEBUG } 04:47:54 DEBUG "value": [ 04:47:54 DEBUG ] 04:47:54 DEBUG 1713498474, 04:47:54 DEBUG } 04:47:54 DEBUG "393432998912" 04:47:54 DEBUG } 04:47:54 DEBUG ] 04:47:54 DEBUG } 04:47:54 DEBUG ] 04:47:54 DEBUG } 04:47:54 DEBUG } 04:47:54 INFO 04:47:54 INFO [loop_until]: kubectl --namespace=xlou top pods 04:47:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:47:54 INFO 04:47:54 INFO [loop_until]: kubectl --namespace=xlou top node 04:47:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:47:54 INFO [loop_until]: OK (rc = 0) 04:47:54 DEBUG --- stdout --- 04:47:54 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 9870m 8372Mi am-6fd554f8f7-hvvkw 9487m 8872Mi am-6fd554f8f7-pmj4b 9952m 9087Mi ds-cts-0 5644m 6042Mi ds-cts-1 4947m 10870Mi ds-cts-2 5119m 11356Mi ds-idrepo-0 4687m 3254Mi ds-idrepo-1 4691m 3207Mi ds-idrepo-2 4505m 3066Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 4m 1215Mi lodemon-6b89d774d5-rf7sk 8m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 6575m 1943Mi 04:47:54 DEBUG --- stderr --- 04:47:54 DEBUG 04:47:54 INFO [loop_until]: OK (rc = 0) 04:47:54 DEBUG --- stdout --- 04:47:54 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 79m 0% 6251Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 4672m 15% 6702Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9387m 31% 11582Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 85m 0% 3260Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 5597m 18% 8737Mi 7% gke-xlou-cdm-default-pool-1b0769c8-wpwl 154m 0% 5168Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 9823m 32% 11147Mi 9% gke-xlou-cdm-ds-0f601751-cf7b 10317m 34% 2722Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 11091m 37% 11281Mi 10% gke-xlou-cdm-ds-0f601751-j488 4865m 16% 6151Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 4958m 16% 14954Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 5034m 16% 6872Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 5189m 17% 13377Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 6654m 41% 5714Mi 9% 04:47:54 DEBUG --- stderr --- 04:47:54 DEBUG 04:48:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498534" 04:48:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498534" 04:48:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498534" 04:48:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713498534" 04:48:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498534" 04:48:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498534" 04:48:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713498534" 04:48:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498534" 04:48:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498534" 04:48:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498534" 04:48:54 ERROR [http_cmd]: ERROR 04:48:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713498534" 04:48:54 INFO [http_cmd]: http status code OK 04:48:54 ERROR --- status code --- 04:48:54 DEBUG --- status code --- 04:48:54 INFO [http_cmd]: http status code OK 04:48:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 04:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:48:54 INFO [http_cmd]: http status code OK 04:48:54 DEBUG --- status code --- 04:48:54 ERROR --- http response --- 04:48:54 DEBUG --- http response --- 04:48:54 DEBUG --- status code --- 04:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:48:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 04:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:48:54 DEBUG --- http response --- 04:48:54 INFO [http_cmd]: http status code OK 04:48:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 04:48:54 DEBUG --- http response --- 04:48:54 DEBUG ----- output ----- 04:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:48:54 DEBUG --- status code --- 04:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:48:54 DEBUG { 04:48:54 DEBUG ----- output ----- 04:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:48:54 DEBUG ----- output ----- 04:48:54 DEBUG "status": "success", 04:48:54 DEBUG { 04:48:54 DEBUG --- http response --- 04:48:54 DEBUG { 04:48:54 DEBUG "data": { 04:48:54 DEBUG "status": "success", 04:48:54 DEBUG "status": "success", 04:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:48:54 DEBUG "resultType": "vector", 04:48:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498534" 04:48:54 DEBUG "data": { 04:48:54 INFO [http_cmd]: http status code OK 04:48:54 INFO [http_cmd]: http status code OK 04:48:54 DEBUG "data": { 04:48:54 INFO [http_cmd]: http status code OK 04:48:54 DEBUG ----- output ----- 04:48:54 DEBUG "result": [ 04:48:54 DEBUG "resultType": "vector", 04:48:54 DEBUG --- status code --- 04:48:54 DEBUG --- status code --- 04:48:54 DEBUG "resultType": "vector", 04:48:54 DEBUG --- status code --- 04:48:54 DEBUG { 04:48:54 DEBUG { 04:48:54 DEBUG "result": [ 04:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:48:54 DEBUG "result": [ 04:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:48:54 DEBUG "status": "success", 04:48:54 DEBUG "metric": { 04:48:54 DEBUG { 04:48:54 DEBUG --- http response --- 04:48:54 DEBUG --- http response --- 04:48:54 DEBUG { 04:48:54 DEBUG --- http response --- 04:48:54 DEBUG "data": { 04:48:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:48:54 DEBUG "metric": { 04:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:48:54 DEBUG "metric": { 04:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:48:54 INFO [http_cmd]: http status code OK 04:48:54 DEBUG "resultType": "vector", 04:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:48:54 DEBUG }, 04:48:54 DEBUG "pod": "ds-cts-0" 04:48:54 DEBUG ----- output ----- 04:48:54 DEBUG "pod": "ds-idrepo-0" 04:48:54 DEBUG ----- output ----- 04:48:54 DEBUG --- status code --- 04:48:54 DEBUG "result": [ 04:48:54 DEBUG { 04:48:54 DEBUG "value": [ 04:48:54 DEBUG }, 04:48:54 DEBUG { 04:48:54 DEBUG }, 04:48:54 DEBUG { 04:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:48:54 DEBUG ----- output ----- 04:48:54 DEBUG "metric": { 04:48:54 INFO [http_cmd]: http status code OK 04:48:54 DEBUG 04:48:54 INFO [http_cmd]: http status code OK 04:48:54 DEBUG "value": [ 04:48:54 DEBUG "status": "success", 04:48:54 DEBUG "value": [ 04:48:54 DEBUG "status": "success", 04:48:54 DEBUG --- http response --- 04:48:54 DEBUG { 04:48:54 DEBUG "pod": "ds-cts-0" 04:48:54 DEBUG --- status code --- 04:48:54 DEBUG 04:48:54 DEBUG --- status code --- 04:48:54 DEBUG 04:48:54 DEBUG "data": { 04:48:54 DEBUG 04:48:54 DEBUG "data": { 04:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 04:48:54 DEBUG "status": "success", 04:48:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498534" 04:48:54 DEBUG }, 04:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:48:54 DEBUG 04:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:48:54 DEBUG 04:48:54 DEBUG "resultType": "vector", 04:48:54 DEBUG 04:48:54 DEBUG "resultType": "vector", 04:48:54 DEBUG ----- output ----- 04:48:54 DEBUG "data": { 04:48:54 DEBUG "value": [ 04:48:54 DEBUG --- http response --- 04:48:54 DEBUG [...] 04:48:54 DEBUG --- http response --- 04:48:54 DEBUG 04:48:54 DEBUG "result": [ 04:48:54 DEBUG 04:48:54 DEBUG "result": [ 04:48:54 DEBUG { 04:48:54 DEBUG "resultType": "vector", 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:48:54 DEBUG [...] 04:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:48:54 DEBUG { 04:48:54 DEBUG [...] 04:48:54 INFO [http_cmd]: http status code OK 04:48:54 DEBUG { 04:48:54 DEBUG "status": "success", 04:48:54 DEBUG "result": [ 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG ----- output ----- 04:48:54 DEBUG 04:48:54 DEBUG ----- output ----- 04:48:54 DEBUG "metric": { 04:48:54 DEBUG 04:48:54 DEBUG --- status code --- 04:48:54 DEBUG "metric": { 04:48:54 DEBUG "data": { 04:48:54 DEBUG { 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG { 04:48:54 DEBUG 04:48:54 DEBUG { 04:48:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:48:54 DEBUG 04:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:48:54 DEBUG "instance": "10.142.0.107:9100" 04:48:54 DEBUG "resultType": "vector", 04:48:54 DEBUG "metric": { 04:48:54 DEBUG [...] 04:48:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:48:54 DEBUG "status": "success", 04:48:54 DEBUG 04:48:54 DEBUG "status": "success", 04:48:54 DEBUG }, 04:48:54 DEBUG 04:48:54 DEBUG --- http response --- 04:48:54 DEBUG }, 04:48:54 DEBUG "result": [ 04:48:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG "data": { 04:48:54 DEBUG "resultType": "vector", 04:48:54 DEBUG "data": { 04:48:54 DEBUG "resultType": "vector", 04:48:54 DEBUG "pod": "ds-idrepo-2" 04:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 04:48:54 DEBUG "value": [ 04:48:54 DEBUG { 04:48:54 DEBUG }, 04:48:54 DEBUG 04:48:54 DEBUG }, 04:48:54 DEBUG "pod": "ds-cts-2" 04:48:54 DEBUG "result": [ 04:48:54 DEBUG "value": [ 04:48:54 DEBUG "result": [ 04:48:54 DEBUG }, 04:48:54 DEBUG ----- output ----- 04:48:54 DEBUG 04:48:54 DEBUG "metric": { 04:48:54 DEBUG "value": [ 04:48:54 DEBUG "pod": "ds-cts-2" 04:48:54 DEBUG "value": [ 04:48:54 DEBUG }, 04:48:54 DEBUG { 04:48:54 DEBUG 04:48:54 DEBUG { 04:48:54 DEBUG "value": [ 04:48:54 DEBUG { 04:48:54 DEBUG 04:48:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:48:54 DEBUG 04:48:54 DEBUG }, 04:48:54 DEBUG 1713498534, 04:48:54 DEBUG "value": [ 04:48:54 DEBUG "metric": { 04:48:54 DEBUG 04:48:54 DEBUG "metric": { 04:48:54 DEBUG 1713498534, 04:48:54 DEBUG "status": "success", 04:48:54 DEBUG 04:48:54 DEBUG }, 04:48:54 DEBUG 04:48:54 DEBUG "value": [ 04:48:54 DEBUG "1998.3333333333333" 04:48:54 DEBUG 1713498534, 04:48:54 DEBUG "pod": "ds-idrepo-0" 04:48:54 DEBUG 04:48:54 DEBUG "pod": "ds-cts-0" 04:48:54 DEBUG "0" 04:48:54 DEBUG "data": { 04:48:54 DEBUG [...] 04:48:54 DEBUG "value": [ 04:48:54 DEBUG 04:48:54 DEBUG 1713498534, 04:48:54 DEBUG ] 04:48:54 DEBUG "0" 04:48:54 DEBUG }, 04:48:54 DEBUG [...] 04:48:54 DEBUG }, 04:48:54 DEBUG ] 04:48:54 DEBUG "resultType": "vector", 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG [...] 04:48:54 DEBUG "0" 04:48:54 DEBUG } 04:48:54 DEBUG ] 04:48:54 DEBUG "value": [ 04:48:54 DEBUG 04:48:54 DEBUG "value": [ 04:48:54 DEBUG } 04:48:54 DEBUG "result": [ 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG ] 04:48:54 DEBUG ] 04:48:54 DEBUG } 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG ] 04:48:54 DEBUG { 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG } 04:48:54 DEBUG } 04:48:54 DEBUG } 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG } 04:48:54 DEBUG "metric": { 04:48:54 DEBUG "instance": "10.142.0.43:9100" 04:48:54 DEBUG [...] 04:48:54 DEBUG 04:48:54 DEBUG ] 04:48:54 DEBUG ] 04:48:54 INFO [http_cmd]: http status code OK 04:48:54 DEBUG 04:48:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:48:54 DEBUG 04:48:54 DEBUG } 04:48:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:48:54 DEBUG }, 04:48:54 DEBUG 04:48:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 04:48:54 DEBUG } 04:48:54 DEBUG } 04:48:54 DEBUG --- status code --- 04:48:54 DEBUG [...] 04:48:54 DEBUG }, 04:48:54 DEBUG [...] 04:48:54 DEBUG }, 04:48:54 DEBUG "value": [ 04:48:54 DEBUG 04:48:54 DEBUG }, 04:48:54 DEBUG } 04:48:54 DEBUG } 04:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:48:54 DEBUG 04:48:54 DEBUG "value": [ 04:48:54 DEBUG 04:48:54 DEBUG "value": [ 04:48:54 DEBUG 1713498534, 04:48:54 DEBUG 04:48:54 DEBUG "value": [ 04:48:54 DEBUG 1713498534, 04:48:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713498534" 04:48:54 DEBUG --- http response --- 04:48:54 DEBUG 04:48:54 DEBUG 1713498534, 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG "0.032000000000001215" 04:48:54 DEBUG "pod": "idm-b85f9d978-txpwv" 04:48:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498534" 04:48:54 DEBUG "6.518810913545871" 04:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:48:54 DEBUG 04:48:54 DEBUG "1998.4666666666667" 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG ] 04:48:54 DEBUG }, 04:48:54 DEBUG ] 04:48:54 DEBUG ----- output ----- 04:48:54 DEBUG "pod": "ds-cts-2" 04:48:54 DEBUG ] 04:48:54 DEBUG "pod": "ds-cts-2" 04:48:54 DEBUG 04:48:54 DEBUG } 04:48:54 DEBUG "value": [ 04:48:54 DEBUG } 04:48:54 DEBUG { 04:48:54 DEBUG }, 04:48:54 DEBUG } 04:48:54 DEBUG }, 04:48:54 DEBUG [...] 04:48:54 DEBUG ] 04:48:54 DEBUG 1713498534, 04:48:54 DEBUG ] 04:48:54 DEBUG "status": "success", 04:48:54 DEBUG "value": [ 04:48:54 DEBUG ] 04:48:54 DEBUG "value": [ 04:48:54 DEBUG 04:48:54 DEBUG } 04:48:54 DEBUG "0" 04:48:54 DEBUG } 04:48:54 DEBUG "data": { 04:48:54 DEBUG 1713498534, 04:48:54 DEBUG } 04:48:54 DEBUG 1713498534, 04:48:54 DEBUG 04:48:54 DEBUG } 04:48:54 DEBUG ] 04:48:54 DEBUG } 04:48:54 DEBUG "resultType": "vector", 04:48:54 DEBUG "0" 04:48:54 DEBUG } 04:48:54 DEBUG "0.000017104444444444444" 04:48:54 DEBUG 04:48:54 DEBUG } 04:48:54 DEBUG "result": [ 04:48:54 DEBUG ] 04:48:54 DEBUG ] 04:48:54 DEBUG "pod": "idm-b85f9d978-txpwv" 04:48:54 DEBUG ] 04:48:54 DEBUG { 04:48:54 DEBUG } 04:48:54 DEBUG } 04:48:54 DEBUG }, 04:48:54 DEBUG } 04:48:54 DEBUG "metric": { 04:48:54 DEBUG ] 04:48:54 DEBUG ] 04:48:54 DEBUG "value": [ 04:48:54 DEBUG } 04:48:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:48:54 DEBUG } 04:48:54 DEBUG } 04:48:54 DEBUG 1713498534, 04:48:54 DEBUG }, 04:48:54 DEBUG } 04:48:54 DEBUG } 04:48:54 DEBUG "0" 04:48:54 DEBUG "value": [ 04:48:54 DEBUG ] 04:48:54 DEBUG 04:48:54 DEBUG } 04:48:54 DEBUG 04:48:54 DEBUG ] 04:48:54 DEBUG 04:48:54 DEBUG } 04:48:54 DEBUG [...] 04:48:54 DEBUG } 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:48:54 DEBUG }, 04:48:54 DEBUG "value": [ 04:48:54 DEBUG 1713498534, 04:48:54 DEBUG "460494" 04:48:54 DEBUG ] 04:48:54 DEBUG } 04:48:54 DEBUG ] 04:48:54 DEBUG } 04:48:54 DEBUG } 04:48:54 INFO [http_cmd]: http status code OK 04:48:54 DEBUG --- status code --- 04:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:48:54 DEBUG --- http response --- 04:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:48:54 DEBUG ----- output ----- 04:48:54 DEBUG { 04:48:54 DEBUG "status": "success", 04:48:54 DEBUG "data": { 04:48:54 DEBUG "resultType": "vector", 04:48:54 DEBUG "result": [ 04:48:54 DEBUG { 04:48:54 DEBUG "metric": { 04:48:54 DEBUG "instance": "10.142.0.107:9100" 04:48:54 DEBUG }, 04:48:54 DEBUG "value": [ 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG [...] 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG "instance": "10.142.0.43:9100" 04:48:54 DEBUG }, 04:48:54 DEBUG "value": [ 04:48:54 DEBUG 1713498534, 04:48:54 DEBUG "0.04663333333332578" 04:48:54 DEBUG ] 04:48:54 DEBUG } 04:48:54 DEBUG ] 04:48:54 DEBUG } 04:48:54 DEBUG } 04:48:54 INFO [http_cmd]: http status code OK 04:48:54 DEBUG --- status code --- 04:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:48:54 DEBUG --- http response --- 04:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:48:54 DEBUG ----- output ----- 04:48:54 DEBUG { 04:48:54 DEBUG "status": "success", 04:48:54 DEBUG "data": { 04:48:54 DEBUG "resultType": "vector", 04:48:54 DEBUG "result": [ 04:48:54 DEBUG { 04:48:54 DEBUG "metric": { 04:48:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 04:48:54 DEBUG }, 04:48:54 DEBUG "value": [ 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG [...] 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 04:48:54 DEBUG }, 04:48:54 DEBUG "value": [ 04:48:54 DEBUG 1713498534, 04:48:54 DEBUG "2137767936" 04:48:54 DEBUG ] 04:48:54 DEBUG } 04:48:54 DEBUG ] 04:48:54 DEBUG } 04:48:54 DEBUG } 04:48:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498534" 04:48:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498534" 04:48:54 INFO [http_cmd]: http status code OK 04:48:54 DEBUG --- status code --- 04:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:48:54 DEBUG --- http response --- 04:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:48:54 DEBUG ----- output ----- 04:48:54 DEBUG { 04:48:54 DEBUG "status": "success", 04:48:54 DEBUG "data": { 04:48:54 DEBUG "resultType": "vector", 04:48:54 DEBUG "result": [ 04:48:54 DEBUG { 04:48:54 DEBUG "metric": { 04:48:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:48:54 DEBUG }, 04:48:54 DEBUG "value": [ 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG [...] 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:48:54 DEBUG }, 04:48:54 DEBUG "value": [ 04:48:54 DEBUG 1713498534, 04:48:54 DEBUG "0" 04:48:54 DEBUG ] 04:48:54 DEBUG } 04:48:54 DEBUG ] 04:48:54 DEBUG } 04:48:54 DEBUG } 04:48:54 INFO [http_cmd]: http status code OK 04:48:54 DEBUG --- status code --- 04:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:48:54 DEBUG --- http response --- 04:48:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 04:48:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713498534" 04:48:54 INFO [http_cmd]: http status code OK 04:48:54 DEBUG --- status code --- 04:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:48:54 DEBUG --- http response --- 04:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:48:54 DEBUG ----- output ----- 04:48:54 DEBUG { 04:48:54 DEBUG "status": "success", 04:48:54 DEBUG "data": { 04:48:54 DEBUG "resultType": "vector", 04:48:54 DEBUG "result": [ 04:48:54 DEBUG { 04:48:54 DEBUG "metric": { 04:48:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:48:54 DEBUG }, 04:48:54 DEBUG "value": [ 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG [...] 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:48:54 DEBUG }, 04:48:54 DEBUG "value": [ 04:48:54 DEBUG 1713498534, 04:48:54 DEBUG "55.171" 04:48:54 DEBUG ] 04:48:54 DEBUG } 04:48:54 DEBUG ] 04:48:54 DEBUG } 04:48:54 DEBUG } 04:48:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498534" 04:48:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498534" 04:48:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498534" 04:48:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498534" 04:48:54 INFO [http_cmd]: http status code OK 04:48:54 DEBUG --- status code --- 04:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:48:54 DEBUG --- http response --- 04:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:48:54 DEBUG ----- output ----- 04:48:54 DEBUG { 04:48:54 DEBUG "status": "success", 04:48:54 DEBUG "data": { 04:48:54 DEBUG "resultType": "vector", 04:48:54 DEBUG "result": [ 04:48:54 DEBUG { 04:48:54 DEBUG "metric": { 04:48:54 DEBUG "instance": "10.142.0.107:9100" 04:48:54 DEBUG }, 04:48:54 DEBUG "value": [ 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG [...] 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG "instance": "10.142.0.43:9100" 04:48:54 DEBUG }, 04:48:54 DEBUG "value": [ 04:48:54 DEBUG 1713498534, 04:48:54 DEBUG "47893165.03333333" 04:48:54 DEBUG ] 04:48:54 DEBUG } 04:48:54 DEBUG ] 04:48:54 DEBUG } 04:48:54 DEBUG } 04:48:54 INFO [http_cmd]: http status code OK 04:48:54 DEBUG --- status code --- 04:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:48:54 DEBUG --- http response --- 04:48:54 INFO [http_cmd]: http status code OK 04:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:48:54 DEBUG --- status code --- 04:48:54 DEBUG ----- output ----- 04:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:48:54 DEBUG { 04:48:54 DEBUG --- http response --- 04:48:54 DEBUG "status": "success", 04:48:54 DEBUG "data": { 04:48:54 DEBUG "resultType": "vector", 04:48:54 DEBUG "result": [ 04:48:54 DEBUG { 04:48:54 DEBUG "metric": { 04:48:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:48:54 DEBUG }, 04:48:54 DEBUG "value": [ 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG [...] 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:48:54 DEBUG }, 04:48:54 DEBUG "value": [ 04:48:54 DEBUG 1713498534, 04:48:54 DEBUG "0" 04:48:54 DEBUG ] 04:48:54 DEBUG } 04:48:54 DEBUG ] 04:48:54 DEBUG } 04:48:54 DEBUG } 04:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 04:48:54 DEBUG ----- output ----- 04:48:54 DEBUG { 04:48:54 DEBUG "status": "success", 04:48:54 DEBUG "data": { 04:48:54 DEBUG "resultType": "vector", 04:48:54 DEBUG "result": [ 04:48:54 DEBUG { 04:48:54 DEBUG "metric": { 04:48:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:48:54 DEBUG }, 04:48:54 DEBUG "value": [ 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG [...] 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG "pod": "idm-b85f9d978-txpwv" 04:48:54 DEBUG }, 04:48:54 DEBUG "value": [ 04:48:54 DEBUG 1713498534, 04:48:54 DEBUG "0.0033105131654831152" 04:48:54 DEBUG ] 04:48:54 DEBUG } 04:48:54 DEBUG ] 04:48:54 DEBUG } 04:48:54 DEBUG } 04:48:54 INFO [http_cmd]: http status code OK 04:48:54 DEBUG --- status code --- 04:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:48:54 DEBUG --- http response --- 04:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:48:54 DEBUG ----- output ----- 04:48:54 DEBUG { 04:48:54 DEBUG "status": "success", 04:48:54 DEBUG "data": { 04:48:54 DEBUG "resultType": "vector", 04:48:54 DEBUG "result": [ 04:48:54 DEBUG { 04:48:54 DEBUG "metric": { 04:48:54 DEBUG "pod": "ds-idrepo-0" 04:48:54 DEBUG }, 04:48:54 DEBUG "value": [ 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG [...] 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG "pod": "ds-idrepo-2" 04:48:54 DEBUG }, 04:48:54 DEBUG "value": [ 04:48:54 DEBUG 1713498534, 04:48:54 DEBUG "0" 04:48:54 DEBUG ] 04:48:54 DEBUG } 04:48:54 DEBUG ] 04:48:54 DEBUG } 04:48:54 DEBUG } 04:48:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498534" 04:48:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498534" 04:48:54 INFO [http_cmd]: http status code OK 04:48:54 INFO [http_cmd]: http status code OK 04:48:54 DEBUG --- status code --- 04:48:54 DEBUG --- status code --- 04:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:48:54 DEBUG --- http response --- 04:48:54 DEBUG --- http response --- 04:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:48:54 DEBUG ----- output ----- 04:48:54 DEBUG ----- output ----- 04:48:54 DEBUG { 04:48:54 DEBUG { 04:48:54 DEBUG "status": "success", 04:48:54 DEBUG "status": "success", 04:48:54 DEBUG "data": { 04:48:54 DEBUG "data": { 04:48:54 DEBUG "resultType": "vector", 04:48:54 DEBUG "resultType": "vector", 04:48:54 DEBUG "result": [ 04:48:54 DEBUG "result": [ 04:48:54 DEBUG { 04:48:54 DEBUG { 04:48:54 DEBUG "metric": { 04:48:54 DEBUG "metric": { 04:48:54 DEBUG "instance": "10.142.0.107:9100" 04:48:54 DEBUG "instance": "10.142.0.107:9100" 04:48:54 DEBUG }, 04:48:54 DEBUG }, 04:48:54 DEBUG "value": [ 04:48:54 DEBUG "value": [ 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG [...] 04:48:54 DEBUG [...] 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG "instance": "10.142.0.43:9100" 04:48:54 DEBUG "instance": "10.142.0.43:9100" 04:48:54 DEBUG }, 04:48:54 DEBUG }, 04:48:54 DEBUG "value": [ 04:48:54 DEBUG "value": [ 04:48:54 DEBUG 1713498534, 04:48:54 DEBUG 1713498534, 04:48:54 DEBUG "395606930432" 04:48:54 DEBUG "47889486.96666667" 04:48:54 DEBUG ] 04:48:54 DEBUG ] 04:48:54 DEBUG } 04:48:54 DEBUG } 04:48:54 DEBUG ] 04:48:54 DEBUG ] 04:48:54 DEBUG } 04:48:54 DEBUG } 04:48:54 DEBUG } 04:48:54 DEBUG } 04:48:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498534" 04:48:54 INFO [http_cmd]: http status code OK 04:48:54 DEBUG --- status code --- 04:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:48:54 DEBUG --- http response --- 04:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 04:48:54 DEBUG ----- output ----- 04:48:54 DEBUG { 04:48:54 DEBUG "status": "success", 04:48:54 DEBUG "data": { 04:48:54 DEBUG "resultType": "vector", 04:48:54 DEBUG "result": [ 04:48:54 DEBUG { 04:48:54 DEBUG "metric": { 04:48:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:48:54 DEBUG }, 04:48:54 DEBUG "value": [ 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG [...] 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG "pod": "idm-b85f9d978-txpwv" 04:48:54 DEBUG }, 04:48:54 DEBUG "value": [ 04:48:54 DEBUG 1713498534, 04:48:54 DEBUG "47467782.7407265" 04:48:54 DEBUG ] 04:48:54 DEBUG } 04:48:54 DEBUG ] 04:48:54 DEBUG } 04:48:54 DEBUG } 04:48:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498534" 04:48:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498534" 04:48:54 INFO [http_cmd]: http status code OK 04:48:54 DEBUG --- status code --- 04:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:48:54 DEBUG --- http response --- 04:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:48:54 DEBUG ----- output ----- 04:48:54 DEBUG { 04:48:54 DEBUG "status": "success", 04:48:54 DEBUG "data": { 04:48:54 DEBUG "resultType": "vector", 04:48:54 DEBUG "result": [ 04:48:54 DEBUG { 04:48:54 DEBUG "metric": { 04:48:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:48:54 DEBUG }, 04:48:54 DEBUG "value": [ 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG [...] 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:48:54 DEBUG }, 04:48:54 DEBUG "value": [ 04:48:54 DEBUG 1713498534, 04:48:54 DEBUG "1998.3999999999999" 04:48:54 DEBUG ] 04:48:54 DEBUG } 04:48:54 DEBUG ] 04:48:54 DEBUG } 04:48:54 DEBUG } 04:48:54 INFO [http_cmd]: http status code OK 04:48:54 DEBUG --- status code --- 04:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:48:54 DEBUG --- http response --- 04:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:48:54 DEBUG ----- output ----- 04:48:54 DEBUG { 04:48:54 DEBUG "status": "success", 04:48:54 DEBUG "data": { 04:48:54 DEBUG "resultType": "vector", 04:48:54 DEBUG "result": [ 04:48:54 DEBUG { 04:48:54 DEBUG "metric": { 04:48:54 DEBUG "pod": "ds-idrepo-0" 04:48:54 DEBUG }, 04:48:54 DEBUG "value": [ 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG [...] 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG 04:48:54 DEBUG "pod": "ds-idrepo-2" 04:48:54 DEBUG }, 04:48:54 DEBUG "value": [ 04:48:54 DEBUG 1713498534, 04:48:54 DEBUG "0" 04:48:54 DEBUG ] 04:48:54 DEBUG } 04:48:54 DEBUG ] 04:48:54 DEBUG } 04:48:54 DEBUG } 04:48:54 INFO 04:48:54 INFO [loop_until]: kubectl --namespace=xlou top pods 04:48:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:48:54 INFO [loop_until]: OK (rc = 0) 04:48:54 DEBUG --- stdout --- 04:48:54 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 10072m 9405Mi am-6fd554f8f7-hvvkw 9565m 8872Mi am-6fd554f8f7-pmj4b 9830m 8461Mi ds-cts-0 5596m 7623Mi ds-cts-1 5034m 10875Mi ds-cts-2 5195m 11362Mi ds-idrepo-0 4622m 3254Mi ds-idrepo-1 4665m 3207Mi ds-idrepo-2 4551m 3066Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 4m 1215Mi lodemon-6b89d774d5-rf7sk 11m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 6242m 1945Mi 04:48:54 DEBUG --- stderr --- 04:48:54 DEBUG 04:48:54 INFO 04:48:54 INFO [loop_until]: kubectl --namespace=xlou top node 04:48:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:48:54 INFO [loop_until]: OK (rc = 0) 04:48:54 DEBUG --- stdout --- 04:48:54 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 75m 0% 6263Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 4636m 15% 6701Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9380m 31% 11585Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 3266Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 5638m 18% 10324Mi 9% gke-xlou-cdm-default-pool-1b0769c8-wpwl 157m 0% 5196Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 9997m 33% 12171Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 10343m 34% 2721Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 10985m 36% 10670Mi 9% gke-xlou-cdm-ds-0f601751-j488 4900m 16% 6155Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 4868m 16% 14954Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 5004m 16% 6879Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 5259m 17% 13385Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 6662m 41% 5716Mi 9% 04:48:54 DEBUG --- stderr --- 04:48:54 DEBUG 04:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713498594" 04:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498594" 04:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713498594" 04:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713498594" 04:49:54 INFO [http_cmd]: http status code OK 04:49:54 DEBUG --- status code --- 04:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:49:54 DEBUG --- http response --- 04:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:49:54 DEBUG ----- output ----- 04:49:54 DEBUG { 04:49:54 DEBUG "status": "success", 04:49:54 DEBUG "data": { 04:49:54 DEBUG "resultType": "vector", 04:49:54 DEBUG "result": [ 04:49:54 DEBUG { 04:49:54 DEBUG "metric": { 04:49:54 DEBUG "pod": "ds-idrepo-0" 04:49:54 DEBUG }, 04:49:54 DEBUG "value": [ 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG [...] 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG "pod": "ds-cts-2" 04:49:54 DEBUG }, 04:49:54 DEBUG "value": [ 04:49:54 DEBUG 1713498594, 04:49:54 DEBUG "0" 04:49:54 DEBUG ] 04:49:54 DEBUG } 04:49:54 DEBUG ] 04:49:54 DEBUG } 04:49:54 DEBUG } 04:49:54 INFO [http_cmd]: http status code OK 04:49:54 DEBUG --- status code --- 04:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:49:54 DEBUG --- http response --- 04:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:49:54 DEBUG ----- output ----- 04:49:54 DEBUG { 04:49:54 DEBUG "status": "success", 04:49:54 DEBUG "data": { 04:49:54 DEBUG "resultType": "vector", 04:49:54 DEBUG "result": [ 04:49:54 DEBUG { 04:49:54 DEBUG "metric": { 04:49:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:49:54 DEBUG }, 04:49:54 DEBUG "value": [ 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG [...] 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:49:54 DEBUG }, 04:49:54 DEBUG "value": [ 04:49:54 DEBUG 1713498594, 04:49:54 DEBUG "580599" 04:49:54 DEBUG ] 04:49:54 DEBUG } 04:49:54 DEBUG ] 04:49:54 DEBUG } 04:49:54 DEBUG } 04:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498594" 04:49:54 ERROR [http_cmd]: ERROR 04:49:54 ERROR --- status code --- 04:49:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 04:49:54 ERROR --- http response --- 04:49:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 04:49:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 04:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498594" 04:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498594" 04:49:54 INFO [http_cmd]: http status code OK 04:49:54 DEBUG --- status code --- 04:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:49:54 DEBUG --- http response --- 04:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498594" 04:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:49:54 DEBUG ----- output ----- 04:49:54 DEBUG { 04:49:54 DEBUG "status": "success", 04:49:54 DEBUG "data": { 04:49:54 DEBUG "resultType": "vector", 04:49:54 DEBUG "result": [ 04:49:54 DEBUG { 04:49:54 DEBUG "metric": { 04:49:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 04:49:54 DEBUG }, 04:49:54 DEBUG "value": [ 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG [...] 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 04:49:54 DEBUG }, 04:49:54 DEBUG "value": [ 04:49:54 DEBUG 1713498594, 04:49:54 DEBUG "2141679616" 04:49:54 DEBUG ] 04:49:54 DEBUG } 04:49:54 DEBUG ] 04:49:54 DEBUG } 04:49:54 DEBUG } 04:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498594" 04:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498594" 04:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713498594" 04:49:54 INFO [http_cmd]: http status code OK 04:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498594" 04:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498594" 04:49:54 INFO [http_cmd]: http status code OK 04:49:54 DEBUG --- status code --- 04:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498594" 04:49:54 INFO [http_cmd]: http status code OK 04:49:54 DEBUG --- status code --- 04:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498594" 04:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498594" 04:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:49:54 DEBUG --- status code --- 04:49:54 INFO [http_cmd]: http status code OK 04:49:54 INFO [http_cmd]: http status code OK 04:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713498594" 04:49:54 DEBUG --- http response --- 04:49:54 INFO [http_cmd]: http status code OK 04:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:49:54 INFO [http_cmd]: http status code OK 04:49:54 DEBUG --- status code --- 04:49:54 DEBUG --- status code --- 04:49:54 DEBUG --- http response --- 04:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:49:54 DEBUG --- status code --- 04:49:54 INFO [http_cmd]: http status code OK 04:49:54 DEBUG --- http response --- 04:49:54 DEBUG --- status code --- 04:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:49:54 DEBUG ----- output ----- 04:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:49:54 DEBUG --- status code --- 04:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:49:54 INFO [http_cmd]: http status code OK 04:49:54 INFO [http_cmd]: http status code OK 04:49:54 DEBUG --- http response --- 04:49:54 DEBUG --- http response --- 04:49:54 DEBUG ----- output ----- 04:49:54 DEBUG { 04:49:54 DEBUG --- http response --- 04:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:49:54 DEBUG ----- output ----- 04:49:54 DEBUG --- http response --- 04:49:54 DEBUG --- status code --- 04:49:54 INFO [http_cmd]: http status code OK 04:49:54 DEBUG --- status code --- 04:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498594" 04:49:54 INFO [http_cmd]: http status code OK 04:49:54 DEBUG { 04:49:54 DEBUG "status": "success", 04:49:54 DEBUG --- http response --- 04:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:49:54 DEBUG { 04:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:49:54 DEBUG --- status code --- 04:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:49:54 DEBUG ----- output ----- 04:49:54 DEBUG ----- output ----- 04:49:54 DEBUG --- status code --- 04:49:54 DEBUG "status": "success", 04:49:54 DEBUG "data": { 04:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 04:49:54 DEBUG ----- output ----- 04:49:54 DEBUG "status": "success", 04:49:54 DEBUG --- http response --- 04:49:54 DEBUG ----- output ----- 04:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498594" 04:49:54 DEBUG --- http response --- 04:49:54 DEBUG { 04:49:54 DEBUG { 04:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:49:54 DEBUG "data": { 04:49:54 DEBUG "resultType": "vector", 04:49:54 INFO [http_cmd]: http status code OK 04:49:54 DEBUG ----- output ----- 04:49:54 DEBUG { 04:49:54 DEBUG "data": { 04:49:54 DEBUG { 04:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:49:54 DEBUG --- http response --- 04:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 04:49:54 DEBUG "status": "success", 04:49:54 DEBUG "status": "success", 04:49:54 DEBUG --- http response --- 04:49:54 DEBUG "resultType": "vector", 04:49:54 DEBUG "result": [ 04:49:54 DEBUG --- status code --- 04:49:54 DEBUG { 04:49:54 DEBUG "status": "success", 04:49:54 DEBUG "resultType": "vector", 04:49:54 DEBUG "status": "success", 04:49:54 DEBUG ----- output ----- 04:49:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 04:49:54 DEBUG ----- output ----- 04:49:54 DEBUG "data": { 04:49:54 DEBUG "data": { 04:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:49:54 DEBUG "result": [ 04:49:54 DEBUG { 04:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:49:54 DEBUG "status": "success", 04:49:54 DEBUG "data": { 04:49:54 DEBUG "result": [ 04:49:54 DEBUG "data": { 04:49:54 DEBUG { 04:49:54 DEBUG { 04:49:54 DEBUG "resultType": "vector", 04:49:54 DEBUG "resultType": "vector", 04:49:54 DEBUG ----- output ----- 04:49:54 DEBUG { 04:49:54 DEBUG "metric": { 04:49:54 DEBUG --- http response --- 04:49:54 DEBUG "data": { 04:49:54 DEBUG "resultType": "vector", 04:49:54 DEBUG { 04:49:54 DEBUG "metric": { 04:49:54 DEBUG "status": "success", 04:49:54 DEBUG "status": "success", 04:49:54 DEBUG "result": [ 04:49:54 DEBUG "result": [ 04:49:54 DEBUG { 04:49:54 DEBUG "status": "success", 04:49:54 DEBUG "data": { 04:49:54 DEBUG "resultType": "vector", 04:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:49:54 DEBUG "result": [ 04:49:54 DEBUG "pod": "ds-idrepo-0" 04:49:54 DEBUG "resultType": "vector", 04:49:54 DEBUG "data": { 04:49:54 DEBUG "data": { 04:49:54 DEBUG { 04:49:54 INFO [http_cmd]: http status code OK 04:49:54 DEBUG { 04:49:54 DEBUG "metric": { 04:49:54 DEBUG "pod": "ds-cts-0" 04:49:54 DEBUG "resultType": "vector", 04:49:54 DEBUG "result": [ 04:49:54 DEBUG ----- output ----- 04:49:54 DEBUG { 04:49:54 DEBUG }, 04:49:54 DEBUG "result": [ 04:49:54 DEBUG "resultType": "vector", 04:49:54 DEBUG "resultType": "vector", 04:49:54 DEBUG "metric": { 04:49:54 DEBUG --- status code --- 04:49:54 DEBUG "metric": { 04:49:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:49:54 DEBUG }, 04:49:54 DEBUG "result": [ 04:49:54 DEBUG { 04:49:54 DEBUG { 04:49:54 DEBUG "metric": { 04:49:54 DEBUG "value": [ 04:49:54 DEBUG { 04:49:54 DEBUG "result": [ 04:49:54 DEBUG "result": [ 04:49:54 DEBUG { 04:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:49:54 DEBUG "pod": "ds-cts-0" 04:49:54 DEBUG }, 04:49:54 DEBUG "value": [ 04:49:54 DEBUG { 04:49:54 DEBUG "metric": { 04:49:54 DEBUG "status": "success", 04:49:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:49:54 DEBUG 04:49:54 DEBUG "metric": { 04:49:54 DEBUG { 04:49:54 DEBUG "pod": "ds-cts-0" 04:49:54 DEBUG "metric": { 04:49:54 DEBUG --- http response --- 04:49:54 DEBUG }, 04:49:54 DEBUG "value": [ 04:49:54 DEBUG 04:49:54 DEBUG "metric": { 04:49:54 DEBUG "instance": "10.142.0.107:9100" 04:49:54 DEBUG "data": { 04:49:54 DEBUG }, 04:49:54 DEBUG "value": [ 04:49:54 DEBUG "instance": "10.142.0.107:9100" 04:49:54 DEBUG "metric": { 04:49:54 DEBUG }, 04:49:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:49:54 DEBUG "value": [ 04:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 04:49:54 DEBUG }, 04:49:54 DEBUG "resultType": "vector", 04:49:54 DEBUG "result": [ 04:49:54 DEBUG 04:49:54 DEBUG }, 04:49:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:49:54 DEBUG "value": [ 04:49:54 DEBUG }, 04:49:54 DEBUG 04:49:54 DEBUG ----- output ----- 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG }, 04:49:54 DEBUG "value": [ 04:49:54 DEBUG 04:49:54 DEBUG { 04:49:54 DEBUG 04:49:54 DEBUG "value": [ 04:49:54 DEBUG }, 04:49:54 DEBUG 04:49:54 DEBUG "value": [ 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG [...] 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG "value": [ 04:49:54 DEBUG "metric": { 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG "value": [ 04:49:54 DEBUG 04:49:54 DEBUG { 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG [...] 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:49:54 DEBUG [...] 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG "status": "success", 04:49:54 DEBUG 04:49:54 DEBUG [...] 04:49:54 INFO [http_cmd]: http status code OK 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG [...] 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG }, 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG [...] 04:49:54 DEBUG "data": { 04:49:54 DEBUG [...] 04:49:54 DEBUG 04:49:54 DEBUG --- status code --- 04:49:54 DEBUG 04:49:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:49:54 DEBUG 04:49:54 DEBUG [...] 04:49:54 DEBUG 04:49:54 DEBUG "value": [ 04:49:54 DEBUG 04:49:54 DEBUG [...] 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG "resultType": "vector", 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:49:54 DEBUG "pod": "ds-cts-2" 04:49:54 DEBUG }, 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG [...] 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG [...] 04:49:54 DEBUG 04:49:54 DEBUG "result": [ 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG --- http response --- 04:49:54 DEBUG }, 04:49:54 DEBUG "value": [ 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG { 04:49:54 DEBUG 04:49:54 DEBUG "pod": "ds-cts-2" 04:49:54 DEBUG "value": [ 04:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:49:54 DEBUG 1713498594, 04:49:54 DEBUG "pod": "ds-idrepo-2" 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG }, 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG "metric": { 04:49:54 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 04:49:54 DEBUG }, 04:49:54 DEBUG 1713498594, 04:49:54 DEBUG ----- output ----- 04:49:54 DEBUG "2007.4666666666667" 04:49:54 DEBUG }, 04:49:54 DEBUG "instance": "10.142.0.43:9100" 04:49:54 DEBUG 04:49:54 DEBUG [...] 04:49:54 DEBUG "value": [ 04:49:54 DEBUG "instance": "10.142.0.43:9100" 04:49:54 DEBUG "pod": "ds-cts-2" 04:49:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:49:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:49:54 DEBUG }, 04:49:54 DEBUG "value": [ 04:49:54 DEBUG "0.000017104444444444444" 04:49:54 DEBUG { 04:49:54 DEBUG ] 04:49:54 DEBUG "value": [ 04:49:54 DEBUG }, 04:49:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 04:49:54 DEBUG 04:49:54 DEBUG 1713498594, 04:49:54 DEBUG }, 04:49:54 DEBUG }, 04:49:54 DEBUG }, 04:49:54 DEBUG }, 04:49:54 DEBUG "value": [ 04:49:54 DEBUG 1713498594, 04:49:54 DEBUG ] 04:49:54 DEBUG "status": "success", 04:49:54 DEBUG } 04:49:54 DEBUG 1713498594, 04:49:54 DEBUG "value": [ 04:49:54 DEBUG }, 04:49:54 DEBUG 04:49:54 DEBUG "0" 04:49:54 DEBUG "value": [ 04:49:54 DEBUG "value": [ 04:49:54 DEBUG "value": [ 04:49:54 DEBUG "value": [ 04:49:54 DEBUG 1713498594, 04:49:54 DEBUG "0" 04:49:54 DEBUG } 04:49:54 DEBUG "data": { 04:49:54 DEBUG ] 04:49:54 DEBUG "0" 04:49:54 DEBUG 1713498594, 04:49:54 DEBUG "value": [ 04:49:54 DEBUG 04:49:54 DEBUG ] 04:49:54 DEBUG 1713498594, 04:49:54 DEBUG 1713498594, 04:49:54 DEBUG 1713498594, 04:49:54 DEBUG 04:49:54 DEBUG "0" 04:49:54 DEBUG ] 04:49:54 DEBUG ] 04:49:54 DEBUG "resultType": "vector", 04:49:54 DEBUG } 04:49:54 DEBUG ] 04:49:54 DEBUG "0.044900000000003125" 04:49:54 DEBUG 1713498594, 04:49:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:49:54 DEBUG } 04:49:54 DEBUG "0.03266666666666633" 04:49:54 DEBUG "0" 04:49:54 DEBUG "2007.1" 04:49:54 DEBUG ] 04:49:54 DEBUG ] 04:49:54 DEBUG } 04:49:54 DEBUG } 04:49:54 DEBUG "result": [ 04:49:54 DEBUG } 04:49:54 DEBUG } 04:49:54 DEBUG ] 04:49:54 DEBUG "6.534969883704159" 04:49:54 DEBUG }, 04:49:54 DEBUG "value": [ 04:49:54 DEBUG ] 04:49:54 DEBUG ] 04:49:54 DEBUG 04:49:54 DEBUG } 04:49:54 DEBUG } 04:49:54 DEBUG ] 04:49:54 DEBUG } 04:49:54 DEBUG { 04:49:54 DEBUG ] 04:49:54 DEBUG } 04:49:54 DEBUG ] 04:49:54 DEBUG ] 04:49:54 DEBUG 1713498594, 04:49:54 DEBUG } 04:49:54 DEBUG } 04:49:54 DEBUG ] 04:49:54 DEBUG ] 04:49:54 DEBUG ] 04:49:54 DEBUG } 04:49:54 DEBUG "metric": { 04:49:54 DEBUG } 04:49:54 DEBUG ] 04:49:54 DEBUG } 04:49:54 DEBUG } 04:49:54 DEBUG "55.171" 04:49:54 DEBUG ] 04:49:54 DEBUG 04:49:54 DEBUG } 04:49:54 DEBUG } 04:49:54 DEBUG } 04:49:54 DEBUG } 04:49:54 DEBUG "pod": "ds-idrepo-0" 04:49:54 DEBUG } 04:49:54 DEBUG } 04:49:54 DEBUG ] 04:49:54 DEBUG } 04:49:54 DEBUG ] 04:49:54 DEBUG } 04:49:54 DEBUG [...] 04:49:54 DEBUG } 04:49:54 DEBUG } 04:49:54 DEBUG } 04:49:54 DEBUG }, 04:49:54 DEBUG } 04:49:54 DEBUG } 04:49:54 DEBUG } 04:49:54 DEBUG } 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG "value": [ 04:49:54 DEBUG ] 04:49:54 DEBUG } 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG } 04:49:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:49:54 DEBUG 04:49:54 DEBUG } 04:49:54 DEBUG }, 04:49:54 DEBUG 04:49:54 DEBUG "value": [ 04:49:54 DEBUG [...] 04:49:54 DEBUG 1713498594, 04:49:54 DEBUG 04:49:54 DEBUG "0" 04:49:54 DEBUG 04:49:54 DEBUG ] 04:49:54 DEBUG 04:49:54 DEBUG } 04:49:54 DEBUG "pod": "ds-idrepo-2" 04:49:54 DEBUG ] 04:49:54 DEBUG }, 04:49:54 DEBUG } 04:49:54 DEBUG "value": [ 04:49:54 DEBUG } 04:49:54 DEBUG 1713498594, 04:49:54 DEBUG "0" 04:49:54 DEBUG ] 04:49:54 DEBUG } 04:49:54 DEBUG ] 04:49:54 DEBUG } 04:49:54 DEBUG } 04:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498594" 04:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498594" 04:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498594" 04:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498594" 04:49:54 INFO [http_cmd]: http status code OK 04:49:54 DEBUG --- status code --- 04:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:49:54 DEBUG --- http response --- 04:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:49:54 DEBUG ----- output ----- 04:49:54 DEBUG { 04:49:54 DEBUG "status": "success", 04:49:54 DEBUG "data": { 04:49:54 DEBUG "resultType": "vector", 04:49:54 DEBUG "result": [ 04:49:54 DEBUG { 04:49:54 DEBUG "metric": { 04:49:54 DEBUG "instance": "10.142.0.107:9100" 04:49:54 DEBUG }, 04:49:54 DEBUG "value": [ 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG [...] 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG "instance": "10.142.0.43:9100" 04:49:54 DEBUG }, 04:49:54 DEBUG "value": [ 04:49:54 DEBUG 1713498594, 04:49:54 DEBUG "48020777.766666666" 04:49:54 DEBUG ] 04:49:54 DEBUG } 04:49:54 DEBUG ] 04:49:54 DEBUG } 04:49:54 DEBUG } 04:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498594" 04:49:54 INFO [http_cmd]: http status code OK 04:49:54 INFO [http_cmd]: http status code OK 04:49:54 DEBUG --- status code --- 04:49:54 DEBUG --- status code --- 04:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:49:54 DEBUG --- http response --- 04:49:54 DEBUG --- http response --- 04:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 04:49:54 DEBUG ----- output ----- 04:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498594" 04:49:54 DEBUG ----- output ----- 04:49:54 DEBUG { 04:49:54 DEBUG { 04:49:54 DEBUG "status": "success", 04:49:54 DEBUG "status": "success", 04:49:54 INFO [http_cmd]: http status code OK 04:49:54 DEBUG "data": { 04:49:54 DEBUG "data": { 04:49:54 DEBUG --- status code --- 04:49:54 DEBUG "resultType": "vector", 04:49:54 DEBUG "resultType": "vector", 04:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:49:54 DEBUG "result": [ 04:49:54 DEBUG "result": [ 04:49:54 DEBUG --- http response --- 04:49:54 DEBUG { 04:49:54 DEBUG { 04:49:54 DEBUG "metric": { 04:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 04:49:54 DEBUG "metric": { 04:49:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:49:54 DEBUG ----- output ----- 04:49:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:49:54 DEBUG }, 04:49:54 DEBUG { 04:49:54 DEBUG }, 04:49:54 DEBUG "value": [ 04:49:54 DEBUG "status": "success", 04:49:54 DEBUG "value": [ 04:49:54 DEBUG 04:49:54 DEBUG "data": { 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG "resultType": "vector", 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG "result": [ 04:49:54 DEBUG 04:49:54 DEBUG [...] 04:49:54 DEBUG { 04:49:54 DEBUG [...] 04:49:54 DEBUG 04:49:54 DEBUG "metric": { 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG }, 04:49:54 DEBUG 04:49:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:49:54 DEBUG "value": [ 04:49:54 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 04:49:54 DEBUG }, 04:49:54 DEBUG 04:49:54 DEBUG }, 04:49:54 DEBUG "value": [ 04:49:54 DEBUG 04:49:54 DEBUG "value": [ 04:49:54 DEBUG 1713498594, 04:49:54 DEBUG 04:49:54 DEBUG 1713498594, 04:49:54 DEBUG "0" 04:49:54 DEBUG [...] 04:49:54 DEBUG "12.998575721380227" 04:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498594" 04:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498594" 04:49:54 DEBUG ] 04:49:54 DEBUG 04:49:54 DEBUG ] 04:49:54 INFO [http_cmd]: http status code OK 04:49:54 DEBUG } 04:49:54 DEBUG 04:49:54 DEBUG } 04:49:54 DEBUG --- status code --- 04:49:54 DEBUG ] 04:49:54 DEBUG 04:49:54 DEBUG ] 04:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:49:54 DEBUG } 04:49:54 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 04:49:54 DEBUG } 04:49:54 DEBUG --- http response --- 04:49:54 DEBUG } 04:49:54 DEBUG }, 04:49:54 DEBUG } 04:49:54 INFO [http_cmd]: http status code OK 04:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:49:54 DEBUG "value": [ 04:49:54 DEBUG --- status code --- 04:49:54 DEBUG ----- output ----- 04:49:54 DEBUG 1713498594, 04:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:49:54 DEBUG { 04:49:54 DEBUG "95085.61554241904" 04:49:54 DEBUG --- http response --- 04:49:54 DEBUG "status": "success", 04:49:54 DEBUG ] 04:49:54 DEBUG "data": { 04:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:49:54 DEBUG } 04:49:54 DEBUG "resultType": "vector", 04:49:54 DEBUG ----- output ----- 04:49:54 DEBUG ] 04:49:54 DEBUG "result": [ 04:49:54 DEBUG { 04:49:54 DEBUG } 04:49:54 DEBUG { 04:49:54 DEBUG "status": "success", 04:49:54 DEBUG } 04:49:54 DEBUG "metric": { 04:49:54 DEBUG "data": { 04:49:54 DEBUG "pod": "ds-idrepo-0" 04:49:54 DEBUG "resultType": "vector", 04:49:54 DEBUG }, 04:49:54 DEBUG "result": [ 04:49:54 DEBUG "value": [ 04:49:54 DEBUG { 04:49:54 DEBUG 04:49:54 DEBUG "metric": { 04:49:54 DEBUG 04:49:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:49:54 DEBUG 04:49:54 DEBUG }, 04:49:54 DEBUG [...] 04:49:54 DEBUG "value": [ 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG "pod": "ds-idrepo-2" 04:49:54 DEBUG [...] 04:49:54 DEBUG }, 04:49:54 DEBUG 04:49:54 DEBUG "value": [ 04:49:54 DEBUG 04:49:54 DEBUG 1713498594, 04:49:54 DEBUG 04:49:54 DEBUG "0" 04:49:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:49:54 DEBUG ] 04:49:54 DEBUG }, 04:49:54 DEBUG } 04:49:54 DEBUG "value": [ 04:49:54 DEBUG ] 04:49:54 DEBUG 1713498594, 04:49:54 DEBUG } 04:49:54 DEBUG "2007.0666666666666" 04:49:54 DEBUG } 04:49:54 DEBUG ] 04:49:54 DEBUG } 04:49:54 DEBUG ] 04:49:54 DEBUG } 04:49:54 DEBUG } 04:49:54 INFO [http_cmd]: http status code OK 04:49:54 INFO [http_cmd]: http status code OK 04:49:54 DEBUG --- status code --- 04:49:54 DEBUG --- status code --- 04:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:49:54 DEBUG --- http response --- 04:49:54 DEBUG --- http response --- 04:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:49:54 DEBUG ----- output ----- 04:49:54 DEBUG ----- output ----- 04:49:54 DEBUG { 04:49:54 DEBUG { 04:49:54 DEBUG "status": "success", 04:49:54 DEBUG "status": "success", 04:49:54 DEBUG "data": { 04:49:54 DEBUG "data": { 04:49:54 DEBUG "resultType": "vector", 04:49:54 DEBUG "resultType": "vector", 04:49:54 DEBUG "result": [ 04:49:54 DEBUG "result": [ 04:49:54 DEBUG { 04:49:54 DEBUG { 04:49:54 DEBUG "metric": { 04:49:54 DEBUG "metric": { 04:49:54 DEBUG "instance": "10.142.0.107:9100" 04:49:54 DEBUG "instance": "10.142.0.107:9100" 04:49:54 DEBUG }, 04:49:54 DEBUG }, 04:49:54 DEBUG "value": [ 04:49:54 DEBUG "value": [ 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG [...] 04:49:54 DEBUG [...] 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG 04:49:54 DEBUG "instance": "10.142.0.43:9100" 04:49:54 DEBUG "instance": "10.142.0.43:9100" 04:49:54 DEBUG }, 04:49:54 DEBUG }, 04:49:54 DEBUG "value": [ 04:49:54 DEBUG "value": [ 04:49:54 DEBUG 1713498594, 04:49:54 DEBUG 1713498594, 04:49:54 DEBUG "397816790016" 04:49:54 DEBUG "48016227" 04:49:54 DEBUG ] 04:49:54 DEBUG ] 04:49:54 DEBUG } 04:49:54 DEBUG } 04:49:54 DEBUG ] 04:49:54 DEBUG ] 04:49:54 DEBUG } 04:49:54 DEBUG } 04:49:54 DEBUG } 04:49:54 DEBUG } 04:49:54 INFO 04:49:54 INFO [loop_until]: kubectl --namespace=xlou top pods 04:49:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:49:54 INFO 04:49:54 INFO [loop_until]: kubectl --namespace=xlou top node 04:49:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:49:54 INFO [loop_until]: OK (rc = 0) 04:49:54 DEBUG --- stdout --- 04:49:54 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 9820m 8942Mi am-6fd554f8f7-hvvkw 9446m 8874Mi am-6fd554f8f7-pmj4b 10036m 8460Mi ds-cts-0 5703m 9051Mi ds-cts-1 5126m 10881Mi ds-cts-2 5344m 11368Mi ds-idrepo-0 4769m 3254Mi ds-idrepo-1 4629m 3207Mi ds-idrepo-2 4528m 3065Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 4m 1216Mi lodemon-6b89d774d5-rf7sk 9m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 6484m 1947Mi 04:49:54 DEBUG --- stderr --- 04:49:54 DEBUG 04:49:54 INFO [loop_until]: OK (rc = 0) 04:49:54 DEBUG --- stdout --- 04:49:54 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 74m 0% 6250Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 4643m 15% 6695Mi 5% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9327m 31% 11587Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 86m 0% 3264Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 5778m 19% 12016Mi 10% gke-xlou-cdm-default-pool-1b0769c8-wpwl 154m 0% 5190Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 9503m 31% 11720Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 10356m 34% 2724Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 10933m 36% 10676Mi 9% gke-xlou-cdm-ds-0f601751-j488 4879m 16% 6156Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 5119m 17% 14948Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 5056m 16% 6873Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 5318m 17% 13366Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 6668m 41% 5718Mi 9% 04:49:54 DEBUG --- stderr --- 04:49:54 DEBUG 04:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498654" 04:50:54 INFO [http_cmd]: http status code OK 04:50:54 DEBUG --- status code --- 04:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:50:54 DEBUG --- http response --- 04:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:50:54 DEBUG ----- output ----- 04:50:54 DEBUG { 04:50:54 DEBUG "status": "success", 04:50:54 DEBUG "data": { 04:50:54 DEBUG "resultType": "vector", 04:50:54 DEBUG "result": [ 04:50:54 DEBUG { 04:50:54 DEBUG "metric": { 04:50:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:50:54 DEBUG }, 04:50:54 DEBUG "value": [ 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG [...] 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:50:54 DEBUG }, 04:50:54 DEBUG "value": [ 04:50:54 DEBUG 1713498654, 04:50:54 DEBUG "700568.5" 04:50:54 DEBUG ] 04:50:54 DEBUG } 04:50:54 DEBUG ] 04:50:54 DEBUG } 04:50:54 DEBUG } 04:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713498654" 04:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713498654" 04:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713498654" 04:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498654" 04:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713498654" 04:50:54 INFO [http_cmd]: http status code OK 04:50:54 DEBUG --- status code --- 04:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:50:54 DEBUG --- http response --- 04:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:50:54 DEBUG ----- output ----- 04:50:54 DEBUG { 04:50:54 DEBUG "status": "success", 04:50:54 DEBUG "data": { 04:50:54 DEBUG "resultType": "vector", 04:50:54 DEBUG "result": [ 04:50:54 DEBUG { 04:50:54 DEBUG "metric": { 04:50:54 DEBUG "pod": "ds-idrepo-0" 04:50:54 DEBUG }, 04:50:54 DEBUG "value": [ 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG [...] 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG "pod": "ds-cts-2" 04:50:54 DEBUG }, 04:50:54 DEBUG "value": [ 04:50:54 DEBUG 1713498654, 04:50:54 DEBUG "0" 04:50:54 DEBUG ] 04:50:54 DEBUG } 04:50:54 DEBUG ] 04:50:54 DEBUG } 04:50:54 DEBUG } 04:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498654" 04:50:54 ERROR [http_cmd]: ERROR 04:50:54 ERROR --- status code --- 04:50:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 04:50:54 ERROR --- http response --- 04:50:54 INFO [http_cmd]: http status code OK 04:50:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 04:50:54 DEBUG --- status code --- 04:50:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 04:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:50:54 DEBUG --- http response --- 04:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498654" 04:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:50:54 DEBUG ----- output ----- 04:50:54 DEBUG { 04:50:54 DEBUG "status": "success", 04:50:54 DEBUG "data": { 04:50:54 DEBUG "resultType": "vector", 04:50:54 DEBUG "result": [ 04:50:54 DEBUG { 04:50:54 DEBUG "metric": { 04:50:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 04:50:54 DEBUG }, 04:50:54 DEBUG "value": [ 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG [...] 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 04:50:54 DEBUG }, 04:50:54 DEBUG "value": [ 04:50:54 DEBUG 1713498654, 04:50:54 DEBUG "2143707136" 04:50:54 DEBUG ] 04:50:54 DEBUG } 04:50:54 DEBUG ] 04:50:54 DEBUG } 04:50:54 DEBUG } 04:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498654" 04:50:54 INFO [http_cmd]: http status code OK 04:50:54 DEBUG --- status code --- 04:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:50:54 DEBUG --- http response --- 04:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 04:50:54 DEBUG ----- output ----- 04:50:54 INFO [http_cmd]: http status code OK 04:50:54 DEBUG { 04:50:54 DEBUG --- status code --- 04:50:54 DEBUG "status": "success", 04:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498654" 04:50:54 DEBUG "data": { 04:50:54 INFO [http_cmd]: http status code OK 04:50:54 DEBUG --- http response --- 04:50:54 DEBUG "resultType": "vector", 04:50:54 DEBUG --- status code --- 04:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:50:54 DEBUG "result": [ 04:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:50:54 DEBUG ----- output ----- 04:50:54 DEBUG { 04:50:54 DEBUG --- http response --- 04:50:54 DEBUG { 04:50:54 DEBUG "metric": { 04:50:54 DEBUG "status": "success", 04:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:50:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:50:54 DEBUG "data": { 04:50:54 DEBUG ----- output ----- 04:50:54 DEBUG }, 04:50:54 DEBUG "resultType": "vector", 04:50:54 DEBUG { 04:50:54 DEBUG "value": [ 04:50:54 DEBUG "result": [ 04:50:54 DEBUG "status": "success", 04:50:54 DEBUG 04:50:54 DEBUG { 04:50:54 DEBUG "data": { 04:50:54 DEBUG 04:50:54 DEBUG "metric": { 04:50:54 DEBUG "resultType": "vector", 04:50:54 DEBUG 04:50:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:50:54 DEBUG "result": [ 04:50:54 DEBUG [...] 04:50:54 DEBUG }, 04:50:54 DEBUG { 04:50:54 DEBUG 04:50:54 DEBUG "value": [ 04:50:54 DEBUG "metric": { 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG }, 04:50:54 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 04:50:54 DEBUG 04:50:54 DEBUG "value": [ 04:50:54 DEBUG }, 04:50:54 DEBUG [...] 04:50:54 DEBUG 04:50:54 DEBUG "value": [ 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG 1713498654, 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG "0" 04:50:54 DEBUG 04:50:54 DEBUG [...] 04:50:54 DEBUG ] 04:50:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:50:54 DEBUG 04:50:54 DEBUG } 04:50:54 DEBUG }, 04:50:54 DEBUG 04:50:54 DEBUG ] 04:50:54 DEBUG "value": [ 04:50:54 DEBUG 04:50:54 DEBUG } 04:50:54 DEBUG 1713498654, 04:50:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:50:54 DEBUG } 04:50:54 DEBUG "1998.8666666666666" 04:50:54 DEBUG }, 04:50:54 DEBUG ] 04:50:54 DEBUG "value": [ 04:50:54 DEBUG } 04:50:54 DEBUG 1713498654, 04:50:54 DEBUG ] 04:50:54 DEBUG "55.171" 04:50:54 DEBUG } 04:50:54 DEBUG ] 04:50:54 DEBUG } 04:50:54 DEBUG } 04:50:54 DEBUG ] 04:50:54 DEBUG } 04:50:54 DEBUG } 04:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498654" 04:50:54 INFO [http_cmd]: http status code OK 04:50:54 INFO [http_cmd]: http status code OK 04:50:54 DEBUG --- status code --- 04:50:54 DEBUG --- status code --- 04:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:50:54 DEBUG --- http response --- 04:50:54 DEBUG --- http response --- 04:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:50:54 DEBUG ----- output ----- 04:50:54 DEBUG { 04:50:54 DEBUG "status": "success", 04:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:50:54 DEBUG "data": { 04:50:54 DEBUG ----- output ----- 04:50:54 INFO [http_cmd]: http status code OK 04:50:54 DEBUG "resultType": "vector", 04:50:54 DEBUG { 04:50:54 DEBUG --- status code --- 04:50:54 DEBUG "result": [ 04:50:54 DEBUG "status": "success", 04:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:50:54 DEBUG { 04:50:54 DEBUG "data": { 04:50:54 DEBUG --- http response --- 04:50:54 DEBUG "metric": { 04:50:54 DEBUG "resultType": "vector", 04:50:54 DEBUG "pod": "ds-cts-0" 04:50:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 04:50:54 DEBUG "result": [ 04:50:54 DEBUG }, 04:50:54 DEBUG { 04:50:54 DEBUG "value": [ 04:50:54 DEBUG "metric": { 04:50:54 DEBUG 04:50:54 DEBUG "pod": "ds-cts-0" 04:50:54 DEBUG 04:50:54 DEBUG }, 04:50:54 DEBUG 04:50:54 DEBUG "value": [ 04:50:54 DEBUG [...] 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG [...] 04:50:54 DEBUG "pod": "ds-cts-2" 04:50:54 DEBUG 04:50:54 DEBUG }, 04:50:54 DEBUG 04:50:54 DEBUG "value": [ 04:50:54 DEBUG 04:50:54 DEBUG 1713498654, 04:50:54 DEBUG "pod": "ds-cts-2" 04:50:54 DEBUG "0" 04:50:54 DEBUG }, 04:50:54 DEBUG ] 04:50:54 DEBUG "value": [ 04:50:54 DEBUG } 04:50:54 DEBUG 1713498654, 04:50:54 DEBUG ] 04:50:54 DEBUG "0.000017104444444444444" 04:50:54 DEBUG } 04:50:54 DEBUG ] 04:50:54 DEBUG } 04:50:54 DEBUG } 04:50:54 DEBUG ] 04:50:54 DEBUG } 04:50:54 DEBUG } 04:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498654" 04:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498654" 04:50:54 INFO [http_cmd]: http status code OK 04:50:54 DEBUG --- status code --- 04:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:50:54 DEBUG --- http response --- 04:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:50:54 DEBUG ----- output ----- 04:50:54 DEBUG { 04:50:54 DEBUG "status": "success", 04:50:54 DEBUG "data": { 04:50:54 DEBUG "resultType": "vector", 04:50:54 DEBUG "result": [ 04:50:54 DEBUG { 04:50:54 DEBUG "metric": { 04:50:54 DEBUG "pod": "ds-idrepo-0" 04:50:54 DEBUG }, 04:50:54 DEBUG "value": [ 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG [...] 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG "pod": "ds-idrepo-2" 04:50:54 DEBUG }, 04:50:54 DEBUG "value": [ 04:50:54 DEBUG 1713498654, 04:50:54 DEBUG "0" 04:50:54 DEBUG ] 04:50:54 DEBUG } 04:50:54 DEBUG ] 04:50:54 DEBUG } 04:50:54 DEBUG } 04:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498654" 04:50:54 INFO [http_cmd]: http status code OK 04:50:54 INFO [http_cmd]: http status code OK 04:50:54 DEBUG --- status code --- 04:50:54 DEBUG --- status code --- 04:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:50:54 DEBUG --- http response --- 04:50:54 DEBUG --- http response --- 04:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:50:54 DEBUG ----- output ----- 04:50:54 DEBUG { 04:50:54 DEBUG "status": "success", 04:50:54 DEBUG "data": { 04:50:54 DEBUG "resultType": "vector", 04:50:54 DEBUG "result": [ 04:50:54 DEBUG { 04:50:54 DEBUG "metric": { 04:50:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:50:54 DEBUG }, 04:50:54 DEBUG "value": [ 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG [...] 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:50:54 DEBUG }, 04:50:54 DEBUG "value": [ 04:50:54 DEBUG 1713498654, 04:50:54 DEBUG "0" 04:50:54 DEBUG ] 04:50:54 DEBUG } 04:50:54 DEBUG ] 04:50:54 DEBUG } 04:50:54 DEBUG } 04:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:50:54 DEBUG ----- output ----- 04:50:54 DEBUG { 04:50:54 DEBUG "status": "success", 04:50:54 DEBUG "data": { 04:50:54 DEBUG "resultType": "vector", 04:50:54 DEBUG "result": [ 04:50:54 DEBUG { 04:50:54 DEBUG "metric": { 04:50:54 DEBUG "pod": "ds-cts-0" 04:50:54 DEBUG }, 04:50:54 DEBUG "value": [ 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG [...] 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG "pod": "ds-cts-2" 04:50:54 DEBUG }, 04:50:54 DEBUG "value": [ 04:50:54 DEBUG 1713498654, 04:50:54 DEBUG "0" 04:50:54 DEBUG ] 04:50:54 DEBUG } 04:50:54 DEBUG ] 04:50:54 DEBUG } 04:50:54 DEBUG } 04:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498654" 04:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498654" 04:50:54 INFO [http_cmd]: http status code OK 04:50:54 DEBUG --- status code --- 04:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:50:54 DEBUG --- http response --- 04:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498654" 04:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:50:54 DEBUG ----- output ----- 04:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498654" 04:50:54 DEBUG { 04:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498654" 04:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713498654" 04:50:54 DEBUG "status": "success", 04:50:54 INFO [http_cmd]: http status code OK 04:50:54 DEBUG "data": { 04:50:54 DEBUG --- status code --- 04:50:54 DEBUG "resultType": "vector", 04:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:50:54 DEBUG "result": [ 04:50:54 DEBUG --- http response --- 04:50:54 DEBUG { 04:50:54 DEBUG "metric": { 04:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 04:50:54 DEBUG "instance": "10.142.0.107:9100" 04:50:54 DEBUG ----- output ----- 04:50:54 DEBUG }, 04:50:54 DEBUG { 04:50:54 DEBUG "value": [ 04:50:54 INFO [http_cmd]: http status code OK 04:50:54 DEBUG "status": "success", 04:50:54 DEBUG 04:50:54 DEBUG --- status code --- 04:50:54 DEBUG "data": { 04:50:54 DEBUG 04:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:50:54 DEBUG "resultType": "vector", 04:50:54 DEBUG 04:50:54 DEBUG --- http response --- 04:50:54 DEBUG "result": [ 04:50:54 DEBUG [...] 04:50:54 DEBUG { 04:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:50:54 DEBUG 04:50:54 DEBUG "metric": { 04:50:54 DEBUG ----- output ----- 04:50:54 INFO [http_cmd]: http status code OK 04:50:54 DEBUG 04:50:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:50:54 DEBUG { 04:50:54 DEBUG --- status code --- 04:50:54 DEBUG 04:50:54 DEBUG }, 04:50:54 DEBUG "status": "success", 04:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:50:54 DEBUG "instance": "10.142.0.43:9100" 04:50:54 DEBUG "value": [ 04:50:54 DEBUG "data": { 04:50:54 DEBUG --- http response --- 04:50:54 DEBUG }, 04:50:54 DEBUG 04:50:54 DEBUG "resultType": "vector", 04:50:54 DEBUG "value": [ 04:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 04:50:54 DEBUG 04:50:54 DEBUG "result": [ 04:50:54 DEBUG 1713498654, 04:50:54 DEBUG ----- output ----- 04:50:54 DEBUG 04:50:54 DEBUG { 04:50:54 DEBUG "0.03200000000000264" 04:50:54 DEBUG { 04:50:54 DEBUG [...] 04:50:54 DEBUG "metric": { 04:50:54 DEBUG ] 04:50:54 DEBUG "status": "success", 04:50:54 DEBUG 04:50:54 DEBUG "pod": "ds-idrepo-0" 04:50:54 DEBUG } 04:50:54 DEBUG "data": { 04:50:54 DEBUG 04:50:54 DEBUG }, 04:50:54 DEBUG ] 04:50:54 DEBUG "resultType": "vector", 04:50:54 DEBUG 04:50:54 DEBUG "value": [ 04:50:54 DEBUG } 04:50:54 DEBUG "result": [ 04:50:54 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 04:50:54 DEBUG 04:50:54 DEBUG } 04:50:54 DEBUG { 04:50:54 DEBUG }, 04:50:54 DEBUG 04:50:54 DEBUG "metric": { 04:50:54 DEBUG "value": [ 04:50:54 DEBUG 04:50:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:50:54 DEBUG 1713498654, 04:50:54 DEBUG [...] 04:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498654" 04:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498654" 04:50:54 INFO [http_cmd]: http status code OK 04:50:54 DEBUG }, 04:50:54 INFO [http_cmd]: http status code OK 04:50:54 DEBUG "0" 04:50:54 DEBUG 04:50:54 DEBUG --- status code --- 04:50:54 DEBUG "value": [ 04:50:54 INFO [http_cmd]: http status code OK 04:50:54 DEBUG --- status code --- 04:50:54 DEBUG ] 04:50:54 DEBUG 04:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:50:54 DEBUG 04:50:54 DEBUG --- status code --- 04:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:50:54 DEBUG } 04:50:54 DEBUG 04:50:54 DEBUG --- http response --- 04:50:54 DEBUG 04:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:50:54 DEBUG --- http response --- 04:50:54 DEBUG ] 04:50:54 DEBUG "pod": "ds-idrepo-2" 04:50:54 DEBUG 04:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:50:54 DEBUG --- http response --- 04:50:54 DEBUG } 04:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:50:54 DEBUG }, 04:50:54 DEBUG [...] 04:50:54 DEBUG ----- output ----- 04:50:54 DEBUG } 04:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:50:54 DEBUG ----- output ----- 04:50:54 DEBUG "value": [ 04:50:54 DEBUG 04:50:54 DEBUG { 04:50:54 DEBUG ----- output ----- 04:50:54 DEBUG { 04:50:54 DEBUG 1713498654, 04:50:54 DEBUG 04:50:54 DEBUG "status": "success", 04:50:54 DEBUG { 04:50:54 DEBUG "status": "success", 04:50:54 DEBUG "0" 04:50:54 DEBUG 04:50:54 DEBUG "data": { 04:50:54 DEBUG "status": "success", 04:50:54 DEBUG "data": { 04:50:54 DEBUG ] 04:50:54 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 04:50:54 DEBUG "resultType": "vector", 04:50:54 DEBUG "data": { 04:50:54 DEBUG "resultType": "vector", 04:50:54 DEBUG } 04:50:54 DEBUG }, 04:50:54 DEBUG "result": [ 04:50:54 DEBUG "resultType": "vector", 04:50:54 DEBUG "result": [ 04:50:54 DEBUG ] 04:50:54 DEBUG "value": [ 04:50:54 DEBUG { 04:50:54 DEBUG "result": [ 04:50:54 DEBUG { 04:50:54 DEBUG } 04:50:54 DEBUG 1713498654, 04:50:54 DEBUG "metric": { 04:50:54 DEBUG { 04:50:54 DEBUG "metric": { 04:50:54 DEBUG } 04:50:54 DEBUG "0" 04:50:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:50:54 DEBUG "metric": { 04:50:54 DEBUG "instance": "10.142.0.107:9100" 04:50:54 DEBUG ] 04:50:54 DEBUG }, 04:50:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 04:50:54 DEBUG }, 04:50:54 DEBUG } 04:50:54 DEBUG "value": [ 04:50:54 DEBUG }, 04:50:54 DEBUG "value": [ 04:50:54 DEBUG ] 04:50:54 DEBUG 04:50:54 DEBUG "value": [ 04:50:54 DEBUG 04:50:54 DEBUG } 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG } 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG [...] 04:50:54 DEBUG 04:50:54 DEBUG [...] 04:50:54 DEBUG 04:50:54 DEBUG [...] 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:50:54 DEBUG 04:50:54 DEBUG "instance": "10.142.0.43:9100" 04:50:54 DEBUG }, 04:50:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 04:50:54 DEBUG }, 04:50:54 DEBUG "value": [ 04:50:54 DEBUG }, 04:50:54 DEBUG "value": [ 04:50:54 DEBUG 1713498654, 04:50:54 DEBUG "value": [ 04:50:54 DEBUG 1713498654, 04:50:54 DEBUG "1998.7666666666667" 04:50:54 DEBUG 1713498654, 04:50:54 DEBUG "0.04360000000000545" 04:50:54 DEBUG ] 04:50:54 DEBUG "6.5281346378957545" 04:50:54 DEBUG ] 04:50:54 DEBUG } 04:50:54 DEBUG ] 04:50:54 DEBUG } 04:50:54 DEBUG ] 04:50:54 DEBUG } 04:50:54 DEBUG ] 04:50:54 DEBUG } 04:50:54 DEBUG ] 04:50:54 DEBUG } 04:50:54 DEBUG } 04:50:54 DEBUG } 04:50:54 DEBUG } 04:50:54 DEBUG } 04:50:54 INFO [http_cmd]: http status code OK 04:50:54 INFO [http_cmd]: http status code OK 04:50:54 DEBUG --- status code --- 04:50:54 DEBUG --- status code --- 04:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:50:54 DEBUG --- http response --- 04:50:54 DEBUG --- http response --- 04:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:50:54 DEBUG ----- output ----- 04:50:54 DEBUG { 04:50:54 DEBUG "status": "success", 04:50:54 DEBUG "data": { 04:50:54 DEBUG "resultType": "vector", 04:50:54 DEBUG "result": [ 04:50:54 DEBUG { 04:50:54 DEBUG "metric": { 04:50:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:50:54 DEBUG }, 04:50:54 DEBUG "value": [ 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG [...] 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:50:54 DEBUG }, 04:50:54 DEBUG "value": [ 04:50:54 DEBUG 1713498654, 04:50:54 DEBUG "1998.8666666666666" 04:50:54 DEBUG ] 04:50:54 DEBUG } 04:50:54 DEBUG ] 04:50:54 DEBUG } 04:50:54 DEBUG } 04:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:50:54 DEBUG ----- output ----- 04:50:54 DEBUG { 04:50:54 DEBUG "status": "success", 04:50:54 DEBUG "data": { 04:50:54 DEBUG "resultType": "vector", 04:50:54 DEBUG "result": [ 04:50:54 DEBUG { 04:50:54 DEBUG "metric": { 04:50:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:50:54 DEBUG }, 04:50:54 DEBUG "value": [ 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG [...] 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:50:54 DEBUG }, 04:50:54 DEBUG "value": [ 04:50:54 DEBUG 1713498654, 04:50:54 DEBUG "0" 04:50:54 DEBUG ] 04:50:54 DEBUG } 04:50:54 DEBUG ] 04:50:54 DEBUG } 04:50:54 DEBUG } 04:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498654" 04:50:54 INFO [http_cmd]: http status code OK 04:50:54 DEBUG --- status code --- 04:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:50:54 DEBUG --- http response --- 04:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:50:54 DEBUG ----- output ----- 04:50:54 DEBUG { 04:50:54 DEBUG "status": "success", 04:50:54 DEBUG "data": { 04:50:54 DEBUG "resultType": "vector", 04:50:54 DEBUG "result": [ 04:50:54 DEBUG { 04:50:54 DEBUG "metric": { 04:50:54 DEBUG "instance": "10.142.0.107:9100" 04:50:54 DEBUG }, 04:50:54 DEBUG "value": [ 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG [...] 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG "instance": "10.142.0.43:9100" 04:50:54 DEBUG }, 04:50:54 DEBUG "value": [ 04:50:54 DEBUG 1713498654, 04:50:54 DEBUG "47873199.16666667" 04:50:54 DEBUG ] 04:50:54 DEBUG } 04:50:54 DEBUG ] 04:50:54 DEBUG } 04:50:54 DEBUG } 04:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498654" 04:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498654" 04:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498654" 04:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498654" 04:50:54 INFO [http_cmd]: http status code OK 04:50:54 INFO [http_cmd]: http status code OK 04:50:54 DEBUG --- status code --- 04:50:54 DEBUG --- status code --- 04:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:50:54 DEBUG --- http response --- 04:50:54 DEBUG --- http response --- 04:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:50:54 DEBUG ----- output ----- 04:50:54 DEBUG { 04:50:54 DEBUG "status": "success", 04:50:54 DEBUG "data": { 04:50:54 DEBUG "resultType": "vector", 04:50:54 DEBUG "result": [ 04:50:54 DEBUG { 04:50:54 DEBUG "metric": { 04:50:54 DEBUG "instance": "10.142.0.107:9100" 04:50:54 DEBUG }, 04:50:54 DEBUG "value": [ 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG [...] 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG "instance": "10.142.0.43:9100" 04:50:54 DEBUG }, 04:50:54 DEBUG "value": [ 04:50:54 DEBUG 1713498654, 04:50:54 DEBUG "400062436352" 04:50:54 DEBUG ] 04:50:54 DEBUG } 04:50:54 DEBUG ] 04:50:54 DEBUG } 04:50:54 DEBUG } 04:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 04:50:54 DEBUG ----- output ----- 04:50:54 DEBUG { 04:50:54 DEBUG "status": "success", 04:50:54 DEBUG "data": { 04:50:54 DEBUG "resultType": "vector", 04:50:54 DEBUG "result": [ 04:50:54 DEBUG { 04:50:54 DEBUG "metric": { 04:50:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:50:54 DEBUG }, 04:50:54 DEBUG "value": [ 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG [...] 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 04:50:54 DEBUG }, 04:50:54 DEBUG "value": [ 04:50:54 DEBUG 1713498654, 04:50:54 DEBUG "0.00002037653986209423" 04:50:54 DEBUG ] 04:50:54 DEBUG } 04:50:54 DEBUG ] 04:50:54 DEBUG } 04:50:54 DEBUG } 04:50:54 INFO [http_cmd]: http status code OK 04:50:54 DEBUG --- status code --- 04:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:50:54 DEBUG --- http response --- 04:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:50:54 DEBUG ----- output ----- 04:50:54 DEBUG { 04:50:54 DEBUG "status": "success", 04:50:54 DEBUG "data": { 04:50:54 DEBUG "resultType": "vector", 04:50:54 DEBUG "result": [ 04:50:54 DEBUG { 04:50:54 DEBUG "metric": { 04:50:54 DEBUG "pod": "ds-idrepo-0" 04:50:54 DEBUG }, 04:50:54 DEBUG "value": [ 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG [...] 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG "pod": "ds-idrepo-2" 04:50:54 DEBUG }, 04:50:54 DEBUG "value": [ 04:50:54 DEBUG 1713498654, 04:50:54 DEBUG "0" 04:50:54 DEBUG ] 04:50:54 DEBUG } 04:50:54 DEBUG ] 04:50:54 DEBUG } 04:50:54 DEBUG } 04:50:54 INFO [http_cmd]: http status code OK 04:50:54 DEBUG --- status code --- 04:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:50:54 DEBUG --- http response --- 04:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:50:54 DEBUG ----- output ----- 04:50:54 DEBUG { 04:50:54 DEBUG "status": "success", 04:50:54 DEBUG "data": { 04:50:54 DEBUG "resultType": "vector", 04:50:54 DEBUG "result": [ 04:50:54 DEBUG { 04:50:54 DEBUG "metric": { 04:50:54 DEBUG "instance": "10.142.0.107:9100" 04:50:54 DEBUG }, 04:50:54 DEBUG "value": [ 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG [...] 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG 04:50:54 DEBUG "instance": "10.142.0.43:9100" 04:50:54 DEBUG }, 04:50:54 DEBUG "value": [ 04:50:54 DEBUG 1713498654, 04:50:54 DEBUG "47869604.46666667" 04:50:54 DEBUG ] 04:50:54 DEBUG } 04:50:54 DEBUG ] 04:50:54 DEBUG } 04:50:54 DEBUG } 04:50:54 INFO 04:50:54 INFO [loop_until]: kubectl --namespace=xlou top pods 04:50:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:50:54 INFO 04:50:54 INFO [loop_until]: kubectl --namespace=xlou top node 04:50:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:50:54 INFO [loop_until]: OK (rc = 0) 04:50:54 DEBUG --- stdout --- 04:50:54 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 9714m 8941Mi am-6fd554f8f7-hvvkw 9530m 8872Mi am-6fd554f8f7-pmj4b 9789m 8461Mi ds-cts-0 5637m 10925Mi ds-cts-1 5242m 10887Mi ds-cts-2 5360m 11373Mi ds-idrepo-0 4695m 3253Mi ds-idrepo-1 4765m 3207Mi ds-idrepo-2 4553m 3066Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 4m 1215Mi lodemon-6b89d774d5-rf7sk 9m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 6425m 1950Mi 04:50:54 DEBUG --- stderr --- 04:50:54 DEBUG 04:50:54 INFO [loop_until]: OK (rc = 0) 04:50:54 DEBUG --- stdout --- 04:50:54 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 79m 0% 6246Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 4485m 15% 6693Mi 5% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9495m 31% 11591Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 82m 0% 3264Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 5718m 19% 13542Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 163m 0% 5177Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 9689m 32% 11716Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 10279m 34% 2719Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 10874m 36% 10666Mi 9% gke-xlou-cdm-ds-0f601751-j488 4759m 15% 6166Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 5268m 17% 14948Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 5046m 16% 6870Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 5443m 18% 13389Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 6459m 40% 5719Mi 9% 04:50:54 DEBUG --- stderr --- 04:50:54 DEBUG 04:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498714" 04:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498714" 04:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498714" 04:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713498714" 04:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498714" 04:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498714" 04:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713498714" 04:51:54 INFO [http_cmd]: http status code OK 04:51:54 INFO [http_cmd]: http status code OK 04:51:54 DEBUG --- status code --- 04:51:54 DEBUG --- status code --- 04:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:51:54 DEBUG --- http response --- 04:51:54 DEBUG --- http response --- 04:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:51:54 DEBUG ----- output ----- 04:51:54 INFO [http_cmd]: http status code OK 04:51:54 INFO [http_cmd]: http status code OK 04:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713498714" 04:51:54 DEBUG ----- output ----- 04:51:54 INFO [http_cmd]: http status code OK 04:51:54 DEBUG { 04:51:54 DEBUG --- status code --- 04:51:54 DEBUG --- status code --- 04:51:54 DEBUG { 04:51:54 DEBUG --- status code --- 04:51:54 DEBUG "status": "success", 04:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:51:54 DEBUG "status": "success", 04:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:51:54 DEBUG "data": { 04:51:54 DEBUG --- http response --- 04:51:54 DEBUG --- http response --- 04:51:54 DEBUG "data": { 04:51:54 DEBUG --- http response --- 04:51:54 DEBUG "resultType": "vector", 04:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:51:54 DEBUG "resultType": "vector", 04:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:51:54 DEBUG "result": [ 04:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:51:54 DEBUG ----- output ----- 04:51:54 DEBUG "result": [ 04:51:54 DEBUG ----- output ----- 04:51:54 DEBUG { 04:51:54 DEBUG ----- output ----- 04:51:54 DEBUG { 04:51:54 DEBUG { 04:51:54 DEBUG { 04:51:54 DEBUG "metric": { 04:51:54 DEBUG { 04:51:54 DEBUG "status": "success", 04:51:54 DEBUG "metric": { 04:51:54 DEBUG "status": "success", 04:51:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:51:54 DEBUG "status": "success", 04:51:54 DEBUG "data": { 04:51:54 DEBUG "pod": "ds-idrepo-0" 04:51:54 DEBUG "data": { 04:51:54 DEBUG }, 04:51:54 DEBUG "data": { 04:51:54 DEBUG "resultType": "vector", 04:51:54 DEBUG }, 04:51:54 DEBUG "resultType": "vector", 04:51:54 DEBUG "value": [ 04:51:54 DEBUG "resultType": "vector", 04:51:54 DEBUG "result": [ 04:51:54 DEBUG "value": [ 04:51:54 DEBUG "result": [ 04:51:54 DEBUG 04:51:54 DEBUG "result": [ 04:51:54 DEBUG { 04:51:54 DEBUG 04:51:54 DEBUG { 04:51:54 DEBUG 04:51:54 DEBUG { 04:51:54 DEBUG "metric": { 04:51:54 DEBUG 04:51:54 DEBUG "metric": { 04:51:54 DEBUG 04:51:54 DEBUG "metric": { 04:51:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:51:54 DEBUG 04:51:54 DEBUG "pod": "ds-cts-0" 04:51:54 DEBUG [...] 04:51:54 DEBUG "pod": "ds-cts-0" 04:51:54 DEBUG }, 04:51:54 DEBUG [...] 04:51:54 DEBUG }, 04:51:54 DEBUG 04:51:54 DEBUG }, 04:51:54 DEBUG "value": [ 04:51:54 DEBUG 04:51:54 DEBUG "value": [ 04:51:54 DEBUG 04:51:54 DEBUG "value": [ 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG "pod": "ds-idrepo-2" 04:51:54 DEBUG 04:51:54 DEBUG }, 04:51:54 DEBUG 04:51:54 DEBUG [...] 04:51:54 DEBUG }, 04:51:54 DEBUG [...] 04:51:54 DEBUG "value": [ 04:51:54 DEBUG [...] 04:51:54 DEBUG 04:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713498714" 04:51:54 DEBUG "value": [ 04:51:54 DEBUG 04:51:54 DEBUG 1713498714, 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 1713498714, 04:51:54 DEBUG 04:51:54 DEBUG "55.171" 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG "0" 04:51:54 DEBUG 04:51:54 DEBUG ] 04:51:54 DEBUG 04:51:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:51:54 DEBUG ] 04:51:54 DEBUG "pod": "ds-cts-2" 04:51:54 DEBUG } 04:51:54 DEBUG "pod": "ds-cts-2" 04:51:54 DEBUG }, 04:51:54 DEBUG } 04:51:54 DEBUG }, 04:51:54 DEBUG ] 04:51:54 DEBUG }, 04:51:54 DEBUG "value": [ 04:51:54 DEBUG ] 04:51:54 DEBUG "value": [ 04:51:54 DEBUG } 04:51:54 DEBUG "value": [ 04:51:54 DEBUG 1713498714, 04:51:54 DEBUG } 04:51:54 DEBUG 1713498714, 04:51:54 DEBUG } 04:51:54 DEBUG 1713498714, 04:51:54 DEBUG "820591" 04:51:54 DEBUG } 04:51:54 DEBUG "0" 04:51:54 DEBUG "0" 04:51:54 DEBUG ] 04:51:54 DEBUG ] 04:51:54 DEBUG ] 04:51:54 DEBUG } 04:51:54 DEBUG } 04:51:54 DEBUG } 04:51:54 DEBUG ] 04:51:54 DEBUG ] 04:51:54 DEBUG ] 04:51:54 DEBUG } 04:51:54 DEBUG } 04:51:54 DEBUG } 04:51:54 DEBUG } 04:51:54 DEBUG } 04:51:54 DEBUG } 04:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498714" 04:51:54 INFO [http_cmd]: http status code OK 04:51:54 INFO [http_cmd]: http status code OK 04:51:54 DEBUG --- status code --- 04:51:54 ERROR [http_cmd]: ERROR 04:51:54 DEBUG --- status code --- 04:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:51:54 ERROR --- status code --- 04:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498714" 04:51:54 DEBUG --- http response --- 04:51:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 04:51:54 DEBUG --- http response --- 04:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:51:54 ERROR --- http response --- 04:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498714" 04:51:54 DEBUG ----- output ----- 04:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:51:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 04:51:54 DEBUG { 04:51:54 DEBUG "status": "success", 04:51:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 04:51:54 DEBUG ----- output ----- 04:51:54 DEBUG "data": { 04:51:54 DEBUG { 04:51:54 DEBUG "resultType": "vector", 04:51:54 DEBUG "status": "success", 04:51:54 DEBUG "result": [ 04:51:54 DEBUG "data": { 04:51:54 DEBUG { 04:51:54 DEBUG "resultType": "vector", 04:51:54 DEBUG "metric": { 04:51:54 DEBUG "result": [ 04:51:54 DEBUG "pod": "ds-idrepo-0" 04:51:54 DEBUG { 04:51:54 DEBUG }, 04:51:54 DEBUG "metric": { 04:51:54 DEBUG "value": [ 04:51:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:51:54 DEBUG 04:51:54 DEBUG }, 04:51:54 DEBUG 04:51:54 DEBUG "value": [ 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG [...] 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG [...] 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG "pod": "ds-cts-2" 04:51:54 DEBUG 04:51:54 DEBUG }, 04:51:54 DEBUG 04:51:54 DEBUG "value": [ 04:51:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:51:54 DEBUG 1713498714, 04:51:54 DEBUG }, 04:51:54 DEBUG "0" 04:51:54 DEBUG "value": [ 04:51:54 DEBUG ] 04:51:54 DEBUG 1713498714, 04:51:54 DEBUG } 04:51:54 DEBUG "2004.0666666666666" 04:51:54 DEBUG ] 04:51:54 DEBUG ] 04:51:54 DEBUG } 04:51:54 DEBUG } 04:51:54 DEBUG } 04:51:54 DEBUG ] 04:51:54 INFO [http_cmd]: http status code OK 04:51:54 DEBUG } 04:51:54 DEBUG --- status code --- 04:51:54 DEBUG } 04:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:51:54 DEBUG --- http response --- 04:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:51:54 DEBUG ----- output ----- 04:51:54 DEBUG { 04:51:54 DEBUG "status": "success", 04:51:54 DEBUG "data": { 04:51:54 DEBUG "resultType": "vector", 04:51:54 DEBUG "result": [ 04:51:54 DEBUG { 04:51:54 DEBUG "metric": { 04:51:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 04:51:54 DEBUG }, 04:51:54 DEBUG "value": [ 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG [...] 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 04:51:54 DEBUG }, 04:51:54 DEBUG "value": [ 04:51:54 DEBUG 1713498714, 04:51:54 DEBUG "2146963456" 04:51:54 DEBUG ] 04:51:54 DEBUG } 04:51:54 DEBUG ] 04:51:54 DEBUG } 04:51:54 DEBUG } 04:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498714" 04:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498714" 04:51:54 INFO [http_cmd]: http status code OK 04:51:54 DEBUG --- status code --- 04:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:51:54 DEBUG --- http response --- 04:51:54 INFO [http_cmd]: http status code OK 04:51:54 INFO [http_cmd]: http status code OK 04:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:51:54 DEBUG --- status code --- 04:51:54 DEBUG --- status code --- 04:51:54 DEBUG ----- output ----- 04:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:51:54 DEBUG { 04:51:54 DEBUG --- http response --- 04:51:54 DEBUG --- http response --- 04:51:54 DEBUG "status": "success", 04:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:51:54 DEBUG "data": { 04:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:51:54 DEBUG ----- output ----- 04:51:54 DEBUG "resultType": "vector", 04:51:54 DEBUG ----- output ----- 04:51:54 DEBUG { 04:51:54 DEBUG "result": [ 04:51:54 DEBUG { 04:51:54 DEBUG "status": "success", 04:51:54 DEBUG { 04:51:54 DEBUG "status": "success", 04:51:54 DEBUG "data": { 04:51:54 DEBUG "metric": { 04:51:54 DEBUG "data": { 04:51:54 DEBUG "resultType": "vector", 04:51:54 DEBUG "instance": "10.142.0.107:9100" 04:51:54 DEBUG "resultType": "vector", 04:51:54 DEBUG "result": [ 04:51:54 DEBUG }, 04:51:54 DEBUG "result": [ 04:51:54 DEBUG { 04:51:54 DEBUG "value": [ 04:51:54 DEBUG { 04:51:54 DEBUG "metric": { 04:51:54 DEBUG 04:51:54 DEBUG "metric": { 04:51:54 DEBUG "pod": "ds-idrepo-0" 04:51:54 DEBUG 04:51:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:51:54 DEBUG }, 04:51:54 DEBUG 04:51:54 DEBUG }, 04:51:54 DEBUG "value": [ 04:51:54 DEBUG [...] 04:51:54 DEBUG "value": [ 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG [...] 04:51:54 DEBUG "instance": "10.142.0.43:9100" 04:51:54 DEBUG [...] 04:51:54 DEBUG 04:51:54 DEBUG }, 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG "value": [ 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 1713498714, 04:51:54 DEBUG 04:51:54 DEBUG "pod": "ds-idrepo-2" 04:51:54 DEBUG "0.03266666666666491" 04:51:54 DEBUG "pod": "ds-idrepo-1" 04:51:54 DEBUG }, 04:51:54 DEBUG ] 04:51:54 DEBUG }, 04:51:54 DEBUG "value": [ 04:51:54 DEBUG } 04:51:54 DEBUG "value": [ 04:51:54 DEBUG 1713498714, 04:51:54 DEBUG ] 04:51:54 DEBUG 1713498714, 04:51:54 DEBUG "0" 04:51:54 DEBUG } 04:51:54 DEBUG "4206.727065986868" 04:51:54 DEBUG ] 04:51:54 DEBUG } 04:51:54 DEBUG ] 04:51:54 DEBUG } 04:51:54 DEBUG } 04:51:54 DEBUG ] 04:51:54 DEBUG ] 04:51:54 DEBUG } 04:51:54 DEBUG } 04:51:54 DEBUG } 04:51:54 DEBUG } 04:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498714" 04:51:54 INFO [http_cmd]: http status code OK 04:51:54 DEBUG --- status code --- 04:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:51:54 DEBUG --- http response --- 04:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:51:54 DEBUG ----- output ----- 04:51:54 DEBUG { 04:51:54 DEBUG "status": "success", 04:51:54 DEBUG "data": { 04:51:54 DEBUG "resultType": "vector", 04:51:54 DEBUG "result": [ 04:51:54 DEBUG { 04:51:54 DEBUG "metric": { 04:51:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:51:54 DEBUG }, 04:51:54 DEBUG "value": [ 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG [...] 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:51:54 DEBUG }, 04:51:54 DEBUG "value": [ 04:51:54 DEBUG 1713498714, 04:51:54 DEBUG "2003.8666666666666" 04:51:54 DEBUG ] 04:51:54 DEBUG } 04:51:54 DEBUG ] 04:51:54 DEBUG } 04:51:54 DEBUG } 04:51:54 INFO [http_cmd]: http status code OK 04:51:54 DEBUG --- status code --- 04:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:51:54 DEBUG --- http response --- 04:51:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 04:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498714" 04:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498714" 04:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498714" 04:51:54 INFO [http_cmd]: http status code OK 04:51:54 DEBUG --- status code --- 04:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:51:54 DEBUG --- http response --- 04:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:51:54 DEBUG ----- output ----- 04:51:54 DEBUG { 04:51:54 DEBUG "status": "success", 04:51:54 DEBUG "data": { 04:51:54 DEBUG "resultType": "vector", 04:51:54 DEBUG "result": [ 04:51:54 DEBUG { 04:51:54 DEBUG "metric": { 04:51:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:51:54 DEBUG }, 04:51:54 DEBUG "value": [ 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG [...] 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:51:54 DEBUG }, 04:51:54 DEBUG "value": [ 04:51:54 DEBUG 1713498714, 04:51:54 DEBUG "0" 04:51:54 DEBUG ] 04:51:54 DEBUG } 04:51:54 DEBUG ] 04:51:54 DEBUG } 04:51:54 DEBUG } 04:51:54 INFO [http_cmd]: http status code OK 04:51:54 DEBUG --- status code --- 04:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:51:54 DEBUG --- http response --- 04:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 04:51:54 DEBUG ----- output ----- 04:51:54 DEBUG { 04:51:54 DEBUG "status": "success", 04:51:54 DEBUG "data": { 04:51:54 DEBUG "resultType": "vector", 04:51:54 DEBUG "result": [ 04:51:54 DEBUG { 04:51:54 DEBUG "metric": { 04:51:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:51:54 DEBUG }, 04:51:54 DEBUG "value": [ 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG [...] 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG "pod": "ds-cts-0" 04:51:54 DEBUG }, 04:51:54 DEBUG "value": [ 04:51:54 DEBUG 1713498714, 04:51:54 DEBUG "0" 04:51:54 DEBUG ] 04:51:54 DEBUG } 04:51:54 DEBUG ] 04:51:54 DEBUG } 04:51:54 DEBUG } 04:51:54 INFO [http_cmd]: http status code OK 04:51:54 INFO [http_cmd]: http status code OK 04:51:54 DEBUG --- status code --- 04:51:54 DEBUG --- status code --- 04:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:51:54 DEBUG --- http response --- 04:51:54 DEBUG --- http response --- 04:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:51:54 DEBUG ----- output ----- 04:51:54 DEBUG { 04:51:54 DEBUG "status": "success", 04:51:54 DEBUG "data": { 04:51:54 DEBUG "resultType": "vector", 04:51:54 DEBUG "result": [ 04:51:54 DEBUG { 04:51:54 DEBUG "metric": { 04:51:54 DEBUG "pod": "ds-cts-0" 04:51:54 DEBUG }, 04:51:54 DEBUG "value": [ 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG [...] 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG "pod": "ds-cts-2" 04:51:54 DEBUG }, 04:51:54 DEBUG "value": [ 04:51:54 DEBUG 1713498714, 04:51:54 DEBUG "0" 04:51:54 DEBUG ] 04:51:54 DEBUG } 04:51:54 DEBUG ] 04:51:54 DEBUG } 04:51:54 DEBUG } 04:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:51:54 DEBUG ----- output ----- 04:51:54 DEBUG { 04:51:54 DEBUG "status": "success", 04:51:54 DEBUG "data": { 04:51:54 DEBUG "resultType": "vector", 04:51:54 DEBUG "result": [ 04:51:54 DEBUG { 04:51:54 DEBUG "metric": { 04:51:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:51:54 DEBUG }, 04:51:54 DEBUG "value": [ 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG [...] 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:51:54 DEBUG }, 04:51:54 DEBUG "value": [ 04:51:54 DEBUG 1713498714, 04:51:54 DEBUG "0" 04:51:54 DEBUG ] 04:51:54 DEBUG } 04:51:54 DEBUG ] 04:51:54 DEBUG } 04:51:54 DEBUG } 04:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498714" 04:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498714" 04:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713498714" 04:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498714" 04:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498714" 04:51:54 INFO [http_cmd]: http status code OK 04:51:54 DEBUG --- status code --- 04:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:51:54 DEBUG --- http response --- 04:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:51:54 DEBUG ----- output ----- 04:51:54 DEBUG { 04:51:54 DEBUG "status": "success", 04:51:54 DEBUG "data": { 04:51:54 DEBUG "resultType": "vector", 04:51:54 DEBUG "result": [ 04:51:54 DEBUG { 04:51:54 DEBUG "metric": { 04:51:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:51:54 DEBUG }, 04:51:54 DEBUG "value": [ 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG [...] 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG "pod": "ds-idrepo-1" 04:51:54 DEBUG }, 04:51:54 DEBUG "value": [ 04:51:54 DEBUG 1713498714, 04:51:54 DEBUG "0" 04:51:54 DEBUG ] 04:51:54 DEBUG } 04:51:54 DEBUG ] 04:51:54 DEBUG } 04:51:54 DEBUG } 04:51:54 INFO [http_cmd]: http status code OK 04:51:54 DEBUG --- status code --- 04:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:51:54 DEBUG --- http response --- 04:51:54 INFO [http_cmd]: http status code OK 04:51:54 INFO [http_cmd]: http status code OK 04:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:51:54 DEBUG --- status code --- 04:51:54 INFO [http_cmd]: http status code OK 04:51:54 DEBUG --- status code --- 04:51:54 DEBUG ----- output ----- 04:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:51:54 DEBUG --- status code --- 04:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:51:54 DEBUG { 04:51:54 DEBUG --- http response --- 04:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:51:54 DEBUG --- http response --- 04:51:54 DEBUG "status": "success", 04:51:54 DEBUG --- http response --- 04:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:51:54 DEBUG "data": { 04:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:51:54 DEBUG ----- output ----- 04:51:54 DEBUG "resultType": "vector", 04:51:54 DEBUG ----- output ----- 04:51:54 DEBUG ----- output ----- 04:51:54 DEBUG { 04:51:54 DEBUG "result": [ 04:51:54 DEBUG { 04:51:54 DEBUG { 04:51:54 DEBUG "status": "success", 04:51:54 DEBUG { 04:51:54 DEBUG "status": "success", 04:51:54 DEBUG "status": "success", 04:51:54 DEBUG "data": { 04:51:54 DEBUG "metric": { 04:51:54 DEBUG "data": { 04:51:54 DEBUG "data": { 04:51:54 DEBUG "resultType": "vector", 04:51:54 DEBUG "instance": "10.142.0.107:9100" 04:51:54 DEBUG "resultType": "vector", 04:51:54 DEBUG "resultType": "vector", 04:51:54 DEBUG "result": [ 04:51:54 DEBUG }, 04:51:54 DEBUG "result": [ 04:51:54 DEBUG "result": [ 04:51:54 DEBUG { 04:51:54 DEBUG "value": [ 04:51:54 DEBUG { 04:51:54 DEBUG { 04:51:54 DEBUG "metric": { 04:51:54 DEBUG 04:51:54 DEBUG "metric": { 04:51:54 DEBUG "metric": { 04:51:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:51:54 DEBUG 04:51:54 DEBUG "instance": "10.142.0.107:9100" 04:51:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 04:51:54 DEBUG }, 04:51:54 DEBUG 04:51:54 DEBUG }, 04:51:54 DEBUG }, 04:51:54 DEBUG "value": [ 04:51:54 DEBUG [...] 04:51:54 DEBUG "value": [ 04:51:54 DEBUG "value": [ 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG [...] 04:51:54 DEBUG "instance": "10.142.0.43:9100" 04:51:54 DEBUG [...] 04:51:54 DEBUG [...] 04:51:54 DEBUG 04:51:54 DEBUG }, 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG "value": [ 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 1713498714, 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:51:54 DEBUG "0.04320000000000637" 04:51:54 DEBUG "instance": "10.142.0.43:9100" 04:51:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 04:51:54 DEBUG }, 04:51:54 DEBUG ] 04:51:54 DEBUG }, 04:51:54 DEBUG }, 04:51:54 DEBUG "value": [ 04:51:54 DEBUG } 04:51:54 DEBUG "value": [ 04:51:54 DEBUG "value": [ 04:51:54 DEBUG 1713498714, 04:51:54 DEBUG ] 04:51:54 DEBUG 1713498714, 04:51:54 DEBUG 1713498714, 04:51:54 DEBUG "2003.9" 04:51:54 DEBUG } 04:51:54 DEBUG "402247222272" 04:51:54 DEBUG "6.534546871384033" 04:51:54 DEBUG ] 04:51:54 DEBUG } 04:51:54 DEBUG ] 04:51:54 DEBUG ] 04:51:54 DEBUG } 04:51:54 DEBUG } 04:51:54 DEBUG } 04:51:54 DEBUG ] 04:51:54 DEBUG ] 04:51:54 DEBUG ] 04:51:54 DEBUG } 04:51:54 DEBUG } 04:51:54 DEBUG } 04:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498714" 04:51:54 DEBUG } 04:51:54 DEBUG } 04:51:54 DEBUG } 04:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498714" 04:51:54 INFO [http_cmd]: http status code OK 04:51:54 DEBUG --- status code --- 04:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:51:54 DEBUG --- http response --- 04:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:51:54 DEBUG ----- output ----- 04:51:54 DEBUG { 04:51:54 DEBUG "status": "success", 04:51:54 DEBUG "data": { 04:51:54 DEBUG "resultType": "vector", 04:51:54 DEBUG "result": [ 04:51:54 DEBUG { 04:51:54 DEBUG "metric": { 04:51:54 DEBUG "instance": "10.142.0.107:9100" 04:51:54 DEBUG }, 04:51:54 DEBUG "value": [ 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG [...] 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG "instance": "10.142.0.43:9100" 04:51:54 DEBUG }, 04:51:54 DEBUG "value": [ 04:51:54 DEBUG 1713498714, 04:51:54 DEBUG "47963262.8" 04:51:54 DEBUG ] 04:51:54 DEBUG } 04:51:54 DEBUG ] 04:51:54 DEBUG } 04:51:54 DEBUG } 04:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498714" 04:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498714" 04:51:54 INFO [http_cmd]: http status code OK 04:51:54 DEBUG --- status code --- 04:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:51:54 DEBUG --- http response --- 04:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:51:54 DEBUG ----- output ----- 04:51:54 DEBUG { 04:51:54 DEBUG "status": "success", 04:51:54 DEBUG "data": { 04:51:54 DEBUG "resultType": "vector", 04:51:54 DEBUG "result": [ 04:51:54 DEBUG { 04:51:54 DEBUG "metric": { 04:51:54 DEBUG "instance": "10.142.0.107:9100" 04:51:54 DEBUG }, 04:51:54 DEBUG "value": [ 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG [...] 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG "instance": "10.142.0.43:9100" 04:51:54 DEBUG }, 04:51:54 DEBUG "value": [ 04:51:54 DEBUG 1713498714, 04:51:54 DEBUG "47960808.03333333" 04:51:54 DEBUG ] 04:51:54 DEBUG } 04:51:54 DEBUG ] 04:51:54 DEBUG } 04:51:54 DEBUG } 04:51:54 INFO [http_cmd]: http status code OK 04:51:54 DEBUG --- status code --- 04:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:51:54 DEBUG --- http response --- 04:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:51:54 DEBUG ----- output ----- 04:51:54 DEBUG { 04:51:54 DEBUG "status": "success", 04:51:54 INFO [http_cmd]: http status code OK 04:51:54 DEBUG "data": { 04:51:54 DEBUG --- status code --- 04:51:54 DEBUG "resultType": "vector", 04:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:51:54 DEBUG "result": [ 04:51:54 DEBUG --- http response --- 04:51:54 DEBUG { 04:51:54 DEBUG "metric": { 04:51:54 DEBUG "pod": "ds-idrepo-0" 04:51:54 DEBUG }, 04:51:54 DEBUG "value": [ 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG [...] 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG "pod": "ds-idrepo-2" 04:51:54 DEBUG }, 04:51:54 DEBUG "value": [ 04:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 04:51:54 DEBUG 1713498714, 04:51:54 DEBUG ----- output ----- 04:51:54 DEBUG "0" 04:51:54 DEBUG { 04:51:54 DEBUG ] 04:51:54 DEBUG "status": "success", 04:51:54 DEBUG } 04:51:54 DEBUG "data": { 04:51:54 DEBUG ] 04:51:54 DEBUG "resultType": "vector", 04:51:54 DEBUG } 04:51:54 DEBUG "result": [ 04:51:54 DEBUG } 04:51:54 DEBUG { 04:51:54 DEBUG "metric": { 04:51:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:51:54 DEBUG }, 04:51:54 DEBUG "value": [ 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG [...] 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG 04:51:54 DEBUG "pod": "ds-cts-0" 04:51:54 DEBUG }, 04:51:54 DEBUG "value": [ 04:51:54 DEBUG 1713498714, 04:51:54 DEBUG "0" 04:51:54 DEBUG ] 04:51:54 DEBUG } 04:51:54 DEBUG ] 04:51:54 DEBUG } 04:51:54 DEBUG } 04:51:54 INFO 04:51:54 INFO [loop_until]: kubectl --namespace=xlou top pods 04:51:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:51:54 INFO 04:51:54 INFO [loop_until]: kubectl --namespace=xlou top node 04:51:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:51:54 INFO [loop_until]: OK (rc = 0) 04:51:54 DEBUG --- stdout --- 04:51:54 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 9704m 8942Mi am-6fd554f8f7-hvvkw 9309m 8873Mi am-6fd554f8f7-pmj4b 10111m 8460Mi ds-cts-0 5700m 12162Mi ds-cts-1 5260m 11806Mi ds-cts-2 5397m 12127Mi ds-idrepo-0 4571m 3253Mi ds-idrepo-1 4561m 3206Mi ds-idrepo-2 4481m 3065Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 4m 1215Mi lodemon-6b89d774d5-rf7sk 8m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 6320m 1952Mi 04:51:54 DEBUG --- stderr --- 04:51:54 DEBUG 04:51:54 INFO [loop_until]: OK (rc = 0) 04:51:54 DEBUG --- stdout --- 04:51:54 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 79m 0% 6261Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 4626m 15% 6704Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9357m 31% 11585Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 78m 0% 3274Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 5734m 19% 14783Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 160m 0% 5177Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 9709m 32% 11714Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 10321m 34% 2734Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 10852m 36% 10662Mi 9% gke-xlou-cdm-ds-0f601751-j488 4844m 16% 6155Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 5452m 18% 16151Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 5116m 17% 6872Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 5522m 18% 14158Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 6665m 41% 5721Mi 9% 04:51:54 DEBUG --- stderr --- 04:51:54 DEBUG 04:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713498774" 04:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498774" 04:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498774" 04:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498774" 04:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498774" 04:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498774" 04:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713498774" 04:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713498774" 04:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498774" 04:52:54 INFO [http_cmd]: http status code OK 04:52:54 DEBUG --- status code --- 04:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:52:54 DEBUG --- http response --- 04:52:54 INFO [http_cmd]: http status code OK 04:52:54 INFO [http_cmd]: http status code OK 04:52:54 ERROR [http_cmd]: ERROR 04:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:52:54 INFO [http_cmd]: http status code OK 04:52:54 DEBUG --- status code --- 04:52:54 INFO [http_cmd]: http status code OK 04:52:54 INFO [http_cmd]: http status code OK 04:52:54 INFO [http_cmd]: http status code OK 04:52:54 DEBUG --- status code --- 04:52:54 ERROR --- status code --- 04:52:54 DEBUG ----- output ----- 04:52:54 DEBUG --- status code --- 04:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:52:54 DEBUG --- status code --- 04:52:54 DEBUG --- status code --- 04:52:54 DEBUG --- status code --- 04:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:52:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 04:52:54 DEBUG { 04:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:52:54 DEBUG --- http response --- 04:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:52:54 DEBUG --- http response --- 04:52:54 ERROR --- http response --- 04:52:54 DEBUG "status": "success", 04:52:54 DEBUG --- http response --- 04:52:54 DEBUG --- http response --- 04:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:52:54 DEBUG --- http response --- 04:52:54 DEBUG --- http response --- 04:52:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 04:52:54 DEBUG "data": { 04:52:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 04:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:52:54 DEBUG ----- output ----- 04:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:52:54 DEBUG "resultType": "vector", 04:52:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 04:52:54 DEBUG ----- output ----- 04:52:54 DEBUG ----- output ----- 04:52:54 DEBUG { 04:52:54 DEBUG ----- output ----- 04:52:54 DEBUG ----- output ----- 04:52:54 DEBUG "result": [ 04:52:54 DEBUG { 04:52:54 DEBUG { 04:52:54 DEBUG "status": "success", 04:52:54 DEBUG { 04:52:54 DEBUG { 04:52:54 DEBUG { 04:52:54 DEBUG "status": "success", 04:52:54 DEBUG "status": "success", 04:52:54 DEBUG "data": { 04:52:54 DEBUG "status": "success", 04:52:54 DEBUG "status": "success", 04:52:54 DEBUG "metric": { 04:52:54 DEBUG "data": { 04:52:54 DEBUG "data": { 04:52:54 DEBUG "resultType": "vector", 04:52:54 DEBUG "data": { 04:52:54 DEBUG "data": { 04:52:54 DEBUG "pod": "ds-cts-0" 04:52:54 DEBUG "resultType": "vector", 04:52:54 DEBUG "resultType": "vector", 04:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713498774" 04:52:54 DEBUG "result": [ 04:52:54 DEBUG "resultType": "vector", 04:52:54 DEBUG "resultType": "vector", 04:52:54 DEBUG }, 04:52:54 DEBUG "result": [ 04:52:54 DEBUG "result": [ 04:52:54 DEBUG { 04:52:54 DEBUG "result": [ 04:52:54 DEBUG "result": [ 04:52:54 DEBUG "value": [ 04:52:54 DEBUG { 04:52:54 DEBUG { 04:52:54 DEBUG "metric": { 04:52:54 DEBUG { 04:52:54 DEBUG { 04:52:54 DEBUG 04:52:54 DEBUG "metric": { 04:52:54 DEBUG "metric": { 04:52:54 DEBUG "pod": "ds-idrepo-0" 04:52:54 DEBUG "metric": { 04:52:54 DEBUG "metric": { 04:52:54 DEBUG 04:52:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:52:54 DEBUG "pod": "ds-idrepo-0" 04:52:54 DEBUG }, 04:52:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 04:52:54 INFO [http_cmd]: http status code OK 04:52:54 DEBUG "instance": "10.142.0.107:9100" 04:52:54 DEBUG 04:52:54 DEBUG }, 04:52:54 DEBUG }, 04:52:54 DEBUG "value": [ 04:52:54 DEBUG }, 04:52:54 DEBUG --- status code --- 04:52:54 DEBUG }, 04:52:54 DEBUG [...] 04:52:54 DEBUG "value": [ 04:52:54 DEBUG "value": [ 04:52:54 DEBUG 04:52:54 DEBUG "value": [ 04:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:52:54 DEBUG "value": [ 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG --- http response --- 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498774" 04:52:54 DEBUG 04:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498774" 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG [...] 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG ----- output ----- 04:52:54 DEBUG "pod": "ds-cts-2" 04:52:54 DEBUG [...] 04:52:54 DEBUG [...] 04:52:54 DEBUG 04:52:54 DEBUG [...] 04:52:54 DEBUG [...] 04:52:54 DEBUG { 04:52:54 DEBUG }, 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG "status": "success", 04:52:54 DEBUG "value": [ 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG "data": { 04:52:54 DEBUG "resultType": "vector", 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG "pod": "ds-idrepo-2" 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG 1713498774, 04:52:54 DEBUG "result": [ 04:52:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:52:54 DEBUG "pod": "ds-cts-2" 04:52:54 DEBUG }, 04:52:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 04:52:54 DEBUG "instance": "10.142.0.43:9100" 04:52:54 DEBUG "0" 04:52:54 DEBUG { 04:52:54 DEBUG }, 04:52:54 DEBUG }, 04:52:54 DEBUG "value": [ 04:52:54 DEBUG }, 04:52:54 DEBUG }, 04:52:54 DEBUG ] 04:52:54 DEBUG "metric": { 04:52:54 DEBUG "value": [ 04:52:54 DEBUG "value": [ 04:52:54 DEBUG 1713498774, 04:52:54 DEBUG "value": [ 04:52:54 DEBUG "value": [ 04:52:54 DEBUG } 04:52:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:52:54 DEBUG 1713498774, 04:52:54 DEBUG 1713498774, 04:52:54 DEBUG "0" 04:52:54 DEBUG ] 04:52:54 DEBUG } 04:52:54 DEBUG ] 04:52:54 DEBUG }, 04:52:54 DEBUG "2008.7666666666667" 04:52:54 DEBUG "0" 04:52:54 DEBUG 1713498774, 04:52:54 DEBUG 1713498774, 04:52:54 DEBUG ] 04:52:54 DEBUG } 04:52:54 DEBUG "value": [ 04:52:54 DEBUG ] 04:52:54 DEBUG ] 04:52:54 DEBUG "2149806080" 04:52:54 DEBUG "0.028999999999999675" 04:52:54 DEBUG } 04:52:54 DEBUG } 04:52:54 DEBUG 04:52:54 DEBUG } 04:52:54 DEBUG } 04:52:54 DEBUG ] 04:52:54 DEBUG ] 04:52:54 DEBUG } 04:52:54 DEBUG 04:52:54 DEBUG ] 04:52:54 DEBUG ] 04:52:54 DEBUG } 04:52:54 DEBUG } 04:52:54 DEBUG 04:52:54 DEBUG } 04:52:54 DEBUG } 04:52:54 DEBUG ] 04:52:54 DEBUG ] 04:52:54 DEBUG [...] 04:52:54 DEBUG } 04:52:54 DEBUG } 04:52:54 DEBUG } 04:52:54 DEBUG } 04:52:54 DEBUG 04:52:54 DEBUG } 04:52:54 DEBUG } 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:52:54 DEBUG }, 04:52:54 DEBUG "value": [ 04:52:54 DEBUG 1713498774, 04:52:54 DEBUG "2008.8" 04:52:54 DEBUG ] 04:52:54 DEBUG } 04:52:54 DEBUG ] 04:52:54 DEBUG } 04:52:54 DEBUG } 04:52:54 INFO [http_cmd]: http status code OK 04:52:54 DEBUG --- status code --- 04:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:52:54 DEBUG --- http response --- 04:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:52:54 DEBUG ----- output ----- 04:52:54 DEBUG { 04:52:54 DEBUG "status": "success", 04:52:54 DEBUG "data": { 04:52:54 DEBUG "resultType": "vector", 04:52:54 DEBUG "result": [ 04:52:54 DEBUG { 04:52:54 DEBUG "metric": { 04:52:54 DEBUG "pod": "ds-cts-0" 04:52:54 DEBUG }, 04:52:54 DEBUG "value": [ 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG [...] 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG "pod": "ds-cts-2" 04:52:54 DEBUG }, 04:52:54 DEBUG "value": [ 04:52:54 DEBUG 1713498774, 04:52:54 DEBUG "0" 04:52:54 DEBUG ] 04:52:54 DEBUG } 04:52:54 DEBUG ] 04:52:54 DEBUG } 04:52:54 DEBUG } 04:52:54 INFO [http_cmd]: http status code OK 04:52:54 INFO [http_cmd]: http status code OK 04:52:54 DEBUG --- status code --- 04:52:54 DEBUG --- status code --- 04:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:52:54 DEBUG --- http response --- 04:52:54 DEBUG --- http response --- 04:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:52:54 DEBUG ----- output ----- 04:52:54 DEBUG { 04:52:54 DEBUG "status": "success", 04:52:54 DEBUG "data": { 04:52:54 DEBUG "resultType": "vector", 04:52:54 DEBUG "result": [ 04:52:54 DEBUG { 04:52:54 DEBUG "metric": { 04:52:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:52:54 DEBUG }, 04:52:54 DEBUG "value": [ 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG [...] 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:52:54 DEBUG }, 04:52:54 DEBUG "value": [ 04:52:54 DEBUG 1713498774, 04:52:54 DEBUG "940913" 04:52:54 DEBUG ] 04:52:54 DEBUG } 04:52:54 DEBUG ] 04:52:54 DEBUG } 04:52:54 DEBUG } 04:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:52:54 DEBUG ----- output ----- 04:52:54 DEBUG { 04:52:54 DEBUG "status": "success", 04:52:54 DEBUG "data": { 04:52:54 DEBUG "resultType": "vector", 04:52:54 DEBUG "result": [ 04:52:54 DEBUG { 04:52:54 DEBUG "metric": { 04:52:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:52:54 DEBUG }, 04:52:54 DEBUG "value": [ 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG [...] 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:52:54 DEBUG }, 04:52:54 DEBUG "value": [ 04:52:54 DEBUG 1713498774, 04:52:54 DEBUG "55.171" 04:52:54 DEBUG ] 04:52:54 DEBUG } 04:52:54 DEBUG ] 04:52:54 DEBUG } 04:52:54 DEBUG } 04:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498774" 04:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498774" 04:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498774" 04:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498774" 04:52:54 INFO [http_cmd]: http status code OK 04:52:54 DEBUG --- status code --- 04:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:52:54 DEBUG --- http response --- 04:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498774" 04:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 04:52:54 DEBUG ----- output ----- 04:52:54 DEBUG { 04:52:54 DEBUG "status": "success", 04:52:54 DEBUG "data": { 04:52:54 DEBUG "resultType": "vector", 04:52:54 DEBUG "result": [ 04:52:54 DEBUG { 04:52:54 DEBUG "metric": { 04:52:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:52:54 DEBUG }, 04:52:54 DEBUG "value": [ 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG [...] 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 04:52:54 DEBUG }, 04:52:54 DEBUG "value": [ 04:52:54 DEBUG 1713498774, 04:52:54 DEBUG "0" 04:52:54 DEBUG ] 04:52:54 DEBUG } 04:52:54 DEBUG ] 04:52:54 DEBUG } 04:52:54 DEBUG } 04:52:54 INFO [http_cmd]: http status code OK 04:52:54 DEBUG --- status code --- 04:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:52:54 DEBUG --- http response --- 04:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:52:54 DEBUG ----- output ----- 04:52:54 DEBUG { 04:52:54 DEBUG "status": "success", 04:52:54 DEBUG "data": { 04:52:54 DEBUG "resultType": "vector", 04:52:54 DEBUG "result": [ 04:52:54 DEBUG { 04:52:54 DEBUG "metric": { 04:52:54 DEBUG "pod": "ds-idrepo-0" 04:52:54 DEBUG }, 04:52:54 DEBUG "value": [ 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG [...] 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG "pod": "ds-idrepo-2" 04:52:54 DEBUG }, 04:52:54 DEBUG "value": [ 04:52:54 DEBUG 1713498774, 04:52:54 DEBUG "0" 04:52:54 DEBUG ] 04:52:54 DEBUG } 04:52:54 DEBUG ] 04:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498774" 04:52:54 DEBUG } 04:52:54 DEBUG } 04:52:54 INFO [http_cmd]: http status code OK 04:52:54 DEBUG --- status code --- 04:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:52:54 DEBUG --- http response --- 04:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:52:54 DEBUG ----- output ----- 04:52:54 DEBUG { 04:52:54 DEBUG "status": "success", 04:52:54 DEBUG "data": { 04:52:54 DEBUG "resultType": "vector", 04:52:54 DEBUG "result": [ 04:52:54 DEBUG { 04:52:54 DEBUG "metric": { 04:52:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:52:54 DEBUG }, 04:52:54 DEBUG "value": [ 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG [...] 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:52:54 DEBUG }, 04:52:54 DEBUG "value": [ 04:52:54 DEBUG 1713498774, 04:52:54 DEBUG "0" 04:52:54 DEBUG ] 04:52:54 DEBUG } 04:52:54 DEBUG ] 04:52:54 DEBUG } 04:52:54 DEBUG } 04:52:54 INFO [http_cmd]: http status code OK 04:52:54 DEBUG --- status code --- 04:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:52:54 DEBUG --- http response --- 04:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:52:54 DEBUG ----- output ----- 04:52:54 DEBUG { 04:52:54 DEBUG "status": "success", 04:52:54 DEBUG "data": { 04:52:54 DEBUG "resultType": "vector", 04:52:54 DEBUG "result": [ 04:52:54 DEBUG { 04:52:54 DEBUG "metric": { 04:52:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:52:54 DEBUG }, 04:52:54 DEBUG "value": [ 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG [...] 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:52:54 DEBUG }, 04:52:54 DEBUG "value": [ 04:52:54 DEBUG 1713498774, 04:52:54 DEBUG "0" 04:52:54 DEBUG ] 04:52:54 DEBUG } 04:52:54 DEBUG ] 04:52:54 DEBUG } 04:52:54 DEBUG } 04:52:54 INFO [http_cmd]: http status code OK 04:52:54 INFO [http_cmd]: http status code OK 04:52:54 DEBUG --- status code --- 04:52:54 DEBUG --- status code --- 04:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:52:54 DEBUG --- http response --- 04:52:54 DEBUG --- http response --- 04:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:52:54 DEBUG ----- output ----- 04:52:54 DEBUG { 04:52:54 DEBUG "status": "success", 04:52:54 DEBUG "data": { 04:52:54 DEBUG "resultType": "vector", 04:52:54 DEBUG "result": [ 04:52:54 DEBUG { 04:52:54 DEBUG "metric": { 04:52:54 DEBUG "pod": "ds-cts-0" 04:52:54 DEBUG }, 04:52:54 DEBUG "value": [ 04:52:54 DEBUG 04:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 04:52:54 DEBUG 04:52:54 DEBUG ----- output ----- 04:52:54 DEBUG 04:52:54 DEBUG { 04:52:54 DEBUG [...] 04:52:54 DEBUG "status": "success", 04:52:54 DEBUG 04:52:54 DEBUG "data": { 04:52:54 DEBUG 04:52:54 DEBUG "resultType": "vector", 04:52:54 DEBUG 04:52:54 DEBUG "result": [ 04:52:54 DEBUG "pod": "ds-cts-2" 04:52:54 DEBUG { 04:52:54 DEBUG }, 04:52:54 DEBUG "metric": { 04:52:54 DEBUG "value": [ 04:52:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:52:54 DEBUG 1713498774, 04:52:54 DEBUG }, 04:52:54 DEBUG "0" 04:52:54 DEBUG "value": [ 04:52:54 DEBUG ] 04:52:54 DEBUG 04:52:54 DEBUG } 04:52:54 DEBUG 04:52:54 DEBUG ] 04:52:54 DEBUG 04:52:54 DEBUG } 04:52:54 DEBUG [...] 04:52:54 DEBUG } 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 04:52:54 DEBUG }, 04:52:54 DEBUG "value": [ 04:52:54 DEBUG 1713498774, 04:52:54 DEBUG "0" 04:52:54 DEBUG ] 04:52:54 DEBUG } 04:52:54 DEBUG ] 04:52:54 DEBUG } 04:52:54 DEBUG } 04:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498774" 04:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498774" 04:52:54 INFO [http_cmd]: http status code OK 04:52:54 DEBUG --- status code --- 04:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:52:54 DEBUG --- http response --- 04:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:52:54 DEBUG ----- output ----- 04:52:54 DEBUG { 04:52:54 DEBUG "status": "success", 04:52:54 DEBUG "data": { 04:52:54 DEBUG "resultType": "vector", 04:52:54 DEBUG "result": [ 04:52:54 DEBUG { 04:52:54 DEBUG "metric": { 04:52:54 DEBUG "instance": "10.142.0.107:9100" 04:52:54 DEBUG }, 04:52:54 DEBUG "value": [ 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG [...] 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG "instance": "10.142.0.43:9100" 04:52:54 DEBUG }, 04:52:54 DEBUG "value": [ 04:52:54 DEBUG 1713498774, 04:52:54 DEBUG "48091649.36666667" 04:52:54 DEBUG ] 04:52:54 DEBUG } 04:52:54 DEBUG ] 04:52:54 DEBUG } 04:52:54 DEBUG } 04:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498774" 04:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498774" 04:52:54 INFO [http_cmd]: http status code OK 04:52:54 DEBUG --- status code --- 04:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:52:54 DEBUG --- http response --- 04:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:52:54 DEBUG ----- output ----- 04:52:54 DEBUG { 04:52:54 DEBUG "status": "success", 04:52:54 DEBUG "data": { 04:52:54 DEBUG "resultType": "vector", 04:52:54 DEBUG "result": [ 04:52:54 DEBUG { 04:52:54 DEBUG "metric": { 04:52:54 DEBUG "pod": "ds-idrepo-0" 04:52:54 DEBUG }, 04:52:54 DEBUG "value": [ 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG [...] 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG "pod": "ds-idrepo-2" 04:52:54 DEBUG }, 04:52:54 DEBUG "value": [ 04:52:54 DEBUG 1713498774, 04:52:54 DEBUG "0" 04:52:54 DEBUG ] 04:52:54 DEBUG } 04:52:54 DEBUG ] 04:52:54 DEBUG } 04:52:54 DEBUG } 04:52:54 INFO [http_cmd]: http status code OK 04:52:54 DEBUG --- status code --- 04:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:52:54 DEBUG --- http response --- 04:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498774" 04:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 04:52:54 DEBUG ----- output ----- 04:52:54 DEBUG { 04:52:54 DEBUG "status": "success", 04:52:54 DEBUG "data": { 04:52:54 DEBUG "resultType": "vector", 04:52:54 DEBUG "result": [ 04:52:54 DEBUG { 04:52:54 DEBUG "metric": { 04:52:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:52:54 DEBUG }, 04:52:54 DEBUG "value": [ 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG [...] 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 04:52:54 DEBUG }, 04:52:54 DEBUG "value": [ 04:52:54 DEBUG 1713498774, 04:52:54 DEBUG "0" 04:52:54 DEBUG ] 04:52:54 DEBUG } 04:52:54 DEBUG ] 04:52:54 DEBUG } 04:52:54 DEBUG } 04:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498774" 04:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713498774" 04:52:54 INFO [http_cmd]: http status code OK 04:52:54 DEBUG --- status code --- 04:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:52:54 DEBUG --- http response --- 04:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:52:54 DEBUG ----- output ----- 04:52:54 DEBUG { 04:52:54 DEBUG "status": "success", 04:52:54 DEBUG "data": { 04:52:54 DEBUG "resultType": "vector", 04:52:54 DEBUG "result": [ 04:52:54 DEBUG { 04:52:54 DEBUG "metric": { 04:52:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:52:54 DEBUG }, 04:52:54 DEBUG "value": [ 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG [...] 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:52:54 DEBUG }, 04:52:54 DEBUG "value": [ 04:52:54 DEBUG 1713498774, 04:52:54 DEBUG "2008.7" 04:52:54 DEBUG ] 04:52:54 DEBUG } 04:52:54 DEBUG ] 04:52:54 DEBUG } 04:52:54 DEBUG } 04:52:54 INFO [http_cmd]: http status code OK 04:52:54 INFO [http_cmd]: http status code OK 04:52:54 DEBUG --- status code --- 04:52:54 DEBUG --- status code --- 04:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:52:54 DEBUG --- http response --- 04:52:54 DEBUG --- http response --- 04:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:52:54 DEBUG ----- output ----- 04:52:54 DEBUG ----- output ----- 04:52:54 DEBUG { 04:52:54 DEBUG { 04:52:54 DEBUG "status": "success", 04:52:54 DEBUG "status": "success", 04:52:54 DEBUG "data": { 04:52:54 DEBUG "data": { 04:52:54 DEBUG "resultType": "vector", 04:52:54 DEBUG "resultType": "vector", 04:52:54 DEBUG "result": [ 04:52:54 DEBUG "result": [ 04:52:54 DEBUG { 04:52:54 DEBUG { 04:52:54 DEBUG "metric": { 04:52:54 DEBUG "metric": { 04:52:54 DEBUG "instance": "10.142.0.107:9100" 04:52:54 DEBUG "instance": "10.142.0.107:9100" 04:52:54 DEBUG }, 04:52:54 DEBUG }, 04:52:54 DEBUG "value": [ 04:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498774" 04:52:54 DEBUG "value": [ 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG [...] 04:52:54 DEBUG [...] 04:52:54 INFO [http_cmd]: http status code OK 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG --- status code --- 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG --- http response --- 04:52:54 DEBUG "instance": "10.142.0.43:9100" 04:52:54 DEBUG "instance": "10.142.0.43:9100" 04:52:54 DEBUG }, 04:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:52:54 DEBUG }, 04:52:54 DEBUG "value": [ 04:52:54 DEBUG ----- output ----- 04:52:54 DEBUG "value": [ 04:52:54 DEBUG 1713498774, 04:52:54 DEBUG { 04:52:54 DEBUG 1713498774, 04:52:54 DEBUG "0.04306666666666861" 04:52:54 DEBUG "status": "success", 04:52:54 DEBUG "404531395584" 04:52:54 DEBUG ] 04:52:54 DEBUG "data": { 04:52:54 DEBUG ] 04:52:54 DEBUG } 04:52:54 DEBUG "resultType": "vector", 04:52:54 DEBUG } 04:52:54 DEBUG ] 04:52:54 DEBUG "result": [ 04:52:54 DEBUG ] 04:52:54 DEBUG } 04:52:54 DEBUG { 04:52:54 DEBUG } 04:52:54 DEBUG } 04:52:54 DEBUG "metric": { 04:52:54 DEBUG } 04:52:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 04:52:54 DEBUG }, 04:52:54 DEBUG "value": [ 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG [...] 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 04:52:54 DEBUG }, 04:52:54 DEBUG "value": [ 04:52:54 DEBUG 1713498774, 04:52:54 DEBUG "6.523921411734049" 04:52:54 DEBUG ] 04:52:54 DEBUG } 04:52:54 DEBUG ] 04:52:54 DEBUG } 04:52:54 DEBUG } 04:52:54 INFO [http_cmd]: http status code OK 04:52:54 DEBUG --- status code --- 04:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:52:54 DEBUG --- http response --- 04:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:52:54 DEBUG ----- output ----- 04:52:54 DEBUG { 04:52:54 DEBUG "status": "success", 04:52:54 DEBUG "data": { 04:52:54 DEBUG "resultType": "vector", 04:52:54 DEBUG "result": [ 04:52:54 DEBUG { 04:52:54 DEBUG "metric": { 04:52:54 DEBUG "instance": "10.142.0.107:9100" 04:52:54 DEBUG }, 04:52:54 DEBUG "value": [ 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG [...] 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG "instance": "10.142.0.43:9100" 04:52:54 DEBUG }, 04:52:54 DEBUG "value": [ 04:52:54 DEBUG 1713498774, 04:52:54 DEBUG "48095068.766666666" 04:52:54 DEBUG ] 04:52:54 DEBUG } 04:52:54 DEBUG ] 04:52:54 DEBUG } 04:52:54 DEBUG } 04:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498774" 04:52:54 INFO [http_cmd]: http status code OK 04:52:54 DEBUG --- status code --- 04:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:52:54 DEBUG --- http response --- 04:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 04:52:54 DEBUG ----- output ----- 04:52:54 DEBUG { 04:52:54 DEBUG "status": "success", 04:52:54 DEBUG "data": { 04:52:54 DEBUG "resultType": "vector", 04:52:54 DEBUG "result": [ 04:52:54 DEBUG { 04:52:54 DEBUG "metric": { 04:52:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:52:54 DEBUG }, 04:52:54 DEBUG "value": [ 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG [...] 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG 04:52:54 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 04:52:54 DEBUG }, 04:52:54 DEBUG "value": [ 04:52:54 DEBUG 1713498774, 04:52:54 DEBUG "0.000028691899726589127" 04:52:54 DEBUG ] 04:52:54 DEBUG } 04:52:54 DEBUG ] 04:52:54 DEBUG } 04:52:54 DEBUG } 04:52:54 INFO 04:52:54 INFO [loop_until]: kubectl --namespace=xlou top pods 04:52:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:52:54 INFO [loop_until]: OK (rc = 0) 04:52:54 DEBUG --- stdout --- 04:52:54 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 9762m 8942Mi am-6fd554f8f7-hvvkw 9244m 8872Mi am-6fd554f8f7-pmj4b 10465m 9085Mi ds-cts-0 5966m 12169Mi ds-cts-1 5503m 12091Mi ds-cts-2 5669m 12130Mi ds-idrepo-0 4667m 3254Mi ds-idrepo-1 4423m 3207Mi ds-idrepo-2 4558m 3065Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 3m 1215Mi lodemon-6b89d774d5-rf7sk 10m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 6329m 1956Mi 04:52:54 DEBUG --- stderr --- 04:52:54 DEBUG 04:52:54 INFO 04:52:54 INFO [loop_until]: kubectl --namespace=xlou top node 04:52:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:52:54 INFO [loop_until]: OK (rc = 0) 04:52:54 DEBUG --- stdout --- 04:52:54 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 79m 0% 6258Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 4646m 15% 6698Mi 5% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9396m 31% 11574Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 3246Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 5809m 19% 14793Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 161m 0% 5185Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 9709m 32% 11718Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 9970m 33% 2725Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 11222m 37% 11270Mi 10% gke-xlou-cdm-ds-0f601751-j488 4869m 16% 6150Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 5505m 18% 16176Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 5118m 17% 6862Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 5569m 18% 14153Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 6662m 41% 5721Mi 9% 04:52:54 DEBUG --- stderr --- 04:52:54 DEBUG 04:53:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498834" 04:53:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498834" 04:53:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713498834" 04:53:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498834" 04:53:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498834" 04:53:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498834" 04:53:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713498834" 04:53:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498834" 04:53:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498834" 04:53:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498834" 04:53:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498834" 04:53:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713498834" 04:53:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498834" 04:53:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713498834" 04:53:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498834" 04:53:54 ERROR [http_cmd]: ERROR 04:53:54 INFO [http_cmd]: http status code OK 04:53:54 INFO [http_cmd]: http status code OK 04:53:54 INFO [http_cmd]: http status code OK 04:53:54 INFO [http_cmd]: http status code OK 04:53:54 INFO [http_cmd]: http status code OK 04:53:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498834" 04:53:54 ERROR --- status code --- 04:53:54 DEBUG --- status code --- 04:53:54 DEBUG --- status code --- 04:53:54 DEBUG --- status code --- 04:53:54 DEBUG --- status code --- 04:53:54 DEBUG --- status code --- 04:53:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 04:53:54 INFO [http_cmd]: http status code OK 04:53:54 INFO [http_cmd]: http status code OK 04:53:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:53:54 INFO [http_cmd]: http status code OK 04:53:54 INFO [http_cmd]: http status code OK 04:53:54 INFO [http_cmd]: http status code OK 04:53:54 INFO [http_cmd]: http status code OK 04:53:54 INFO [http_cmd]: http status code OK 04:53:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:53:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:53:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:53:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:53:54 ERROR --- http response --- 04:53:54 DEBUG --- status code --- 04:53:54 DEBUG --- status code --- 04:53:54 DEBUG --- http response --- 04:53:54 DEBUG --- status code --- 04:53:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:53:54 DEBUG --- status code --- 04:53:54 DEBUG --- status code --- 04:53:54 INFO [http_cmd]: http status code OK 04:53:54 DEBUG --- status code --- 04:53:54 DEBUG --- http response --- 04:53:54 DEBUG --- http response --- 04:53:54 DEBUG --- http response --- 04:53:54 DEBUG --- http response --- 04:53:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:53:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 04:53:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:53:54 DEBUG --- status code --- 04:53:54 DEBUG --- http response --- 04:53:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 04:53:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:53:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:53:54 DEBUG --- status code --- 04:53:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:53:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:53:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:53:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 04:53:54 DEBUG --- http response --- 04:53:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:53:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 04:53:54 DEBUG --- http response --- 04:53:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:53:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:53:54 INFO [http_cmd]: http status code OK 04:53:54 DEBUG --- http response --- 04:53:54 DEBUG --- http response --- 04:53:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:53:54 DEBUG --- http response --- 04:53:54 DEBUG ----- output ----- 04:53:54 DEBUG ----- output ----- 04:53:54 DEBUG { 04:53:54 INFO [http_cmd]: http status code OK 04:53:54 DEBUG ----- output ----- 04:53:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:53:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:53:54 DEBUG --- http response --- 04:53:54 DEBUG ----- output ----- 04:53:54 DEBUG --- status code --- 04:53:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:53:54 DEBUG --- http response --- 04:53:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:53:54 DEBUG { 04:53:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:53:54 DEBUG ----- output ----- 04:53:54 DEBUG "status": "success", 04:53:54 DEBUG --- status code --- 04:53:54 DEBUG { 04:53:54 DEBUG ----- output ----- 04:53:54 DEBUG ----- output ----- 04:53:54 DEBUG { 04:53:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:53:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:53:54 DEBUG ----- output ----- 04:53:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:53:54 DEBUG ----- output ----- 04:53:54 DEBUG "status": "success", 04:53:54 DEBUG ----- output ----- 04:53:54 DEBUG { 04:53:54 DEBUG "data": { 04:53:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:53:54 DEBUG "status": "success", 04:53:54 DEBUG { 04:53:54 DEBUG { 04:53:54 DEBUG "status": "success", 04:53:54 DEBUG ----- output ----- 04:53:54 DEBUG --- http response --- 04:53:54 DEBUG { 04:53:54 DEBUG ----- output ----- 04:53:54 DEBUG { 04:53:54 DEBUG "data": { 04:53:54 DEBUG { 04:53:54 DEBUG "status": "success", 04:53:54 DEBUG "resultType": "vector", 04:53:54 DEBUG --- http response --- 04:53:54 DEBUG "data": { 04:53:54 DEBUG "status": "success", 04:53:54 DEBUG "status": "success", 04:53:54 DEBUG "data": { 04:53:54 DEBUG { 04:53:54 DEBUG "status": "success", 04:53:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:53:54 DEBUG { 04:53:54 DEBUG "status": "success", 04:53:54 DEBUG "resultType": "vector", 04:53:54 DEBUG "status": "success", 04:53:54 DEBUG "data": { 04:53:54 DEBUG "result": [ 04:53:54 DEBUG "resultType": "vector", 04:53:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:53:54 DEBUG "data": { 04:53:54 DEBUG "data": { 04:53:54 DEBUG "resultType": "vector", 04:53:54 DEBUG "status": "success", 04:53:54 DEBUG "data": { 04:53:54 DEBUG ----- output ----- 04:53:54 DEBUG "status": "success", 04:53:54 DEBUG "data": { 04:53:54 DEBUG "result": [ 04:53:54 DEBUG "data": { 04:53:54 DEBUG "resultType": "vector", 04:53:54 DEBUG { 04:53:54 DEBUG "result": [ 04:53:54 DEBUG ----- output ----- 04:53:54 DEBUG "resultType": "vector", 04:53:54 DEBUG "resultType": "vector", 04:53:54 DEBUG "result": [ 04:53:54 DEBUG "data": { 04:53:54 DEBUG "resultType": "vector", 04:53:54 DEBUG { 04:53:54 DEBUG "data": { 04:53:54 DEBUG "resultType": "vector", 04:53:54 DEBUG { 04:53:54 DEBUG "resultType": "vector", 04:53:54 DEBUG "result": [ 04:53:54 DEBUG "metric": { 04:53:54 DEBUG { 04:53:54 DEBUG { 04:53:54 DEBUG "result": [ 04:53:54 DEBUG "result": [ 04:53:54 DEBUG { 04:53:54 DEBUG "resultType": "vector", 04:53:54 DEBUG "result": [ 04:53:54 DEBUG "status": "success", 04:53:54 DEBUG "resultType": "vector", 04:53:54 DEBUG "result": [ 04:53:54 DEBUG "metric": { 04:53:54 DEBUG "result": [ 04:53:54 DEBUG { 04:53:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:53:54 DEBUG "metric": { 04:53:54 DEBUG "status": "success", 04:53:54 DEBUG { 04:53:54 DEBUG { 04:53:54 DEBUG "metric": { 04:53:54 DEBUG "result": [ 04:53:54 DEBUG { 04:53:54 DEBUG "data": { 04:53:54 DEBUG "result": [ 04:53:54 DEBUG { 04:53:54 DEBUG "pod": "ds-cts-0" 04:53:54 DEBUG { 04:53:54 DEBUG "metric": { 04:53:54 DEBUG }, 04:53:54 DEBUG "pod": "ds-cts-0" 04:53:54 DEBUG "data": { 04:53:54 DEBUG "metric": { 04:53:54 DEBUG "metric": { 04:53:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 04:53:54 DEBUG { 04:53:54 DEBUG "metric": { 04:53:54 DEBUG "resultType": "vector", 04:53:54 DEBUG { 04:53:54 DEBUG "metric": { 04:53:54 DEBUG }, 04:53:54 DEBUG "metric": { 04:53:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:53:54 DEBUG "value": [ 04:53:54 DEBUG }, 04:53:54 DEBUG "resultType": "vector", 04:53:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:53:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:53:54 DEBUG }, 04:53:54 DEBUG "metric": { 04:53:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:53:54 DEBUG "result": [ 04:53:54 DEBUG "metric": { 04:53:54 DEBUG "pod": "ds-idrepo-0" 04:53:54 DEBUG "value": [ 04:53:54 DEBUG "pod": "ds-idrepo-0" 04:53:54 DEBUG }, 04:53:54 DEBUG 04:53:54 DEBUG "value": [ 04:53:54 DEBUG "result": [ 04:53:54 DEBUG }, 04:53:54 DEBUG }, 04:53:54 DEBUG "value": [ 04:53:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:53:54 DEBUG }, 04:53:54 DEBUG { 04:53:54 DEBUG "pod": "ds-cts-0" 04:53:54 DEBUG }, 04:53:54 DEBUG 04:53:54 DEBUG }, 04:53:54 DEBUG "value": [ 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG { 04:53:54 DEBUG "value": [ 04:53:54 DEBUG "value": [ 04:53:54 DEBUG 04:53:54 DEBUG }, 04:53:54 DEBUG "value": [ 04:53:54 DEBUG "metric": { 04:53:54 DEBUG }, 04:53:54 DEBUG "value": [ 04:53:54 DEBUG 04:53:54 DEBUG "value": [ 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG "metric": { 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498834" 04:53:54 DEBUG "value": [ 04:53:54 DEBUG 04:53:54 DEBUG "instance": "10.142.0.107:9100" 04:53:54 DEBUG "value": [ 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG [...] 04:53:54 DEBUG 04:53:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG }, 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG [...] 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG [...] 04:53:54 DEBUG }, 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG [...] 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG "value": [ 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG [...] 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG "value": [ 04:53:54 DEBUG [...] 04:53:54 DEBUG [...] 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG [...] 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG [...] 04:53:54 DEBUG 04:53:54 DEBUG [...] 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG [...] 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG [...] 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG "pod": "ds-cts-2" 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG }, 04:53:54 DEBUG "pod": "ds-cts-2" 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG [...] 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG }, 04:53:54 DEBUG 04:53:54 DEBUG "pod": "ds-cts-0" 04:53:54 DEBUG "value": [ 04:53:54 DEBUG }, 04:53:54 DEBUG [...] 04:53:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:53:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:53:54 DEBUG }, 04:53:54 DEBUG 04:53:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG "pod": "ds-cts-2" 04:53:54 DEBUG "value": [ 04:53:54 DEBUG "pod": "ds-idrepo-2" 04:53:54 DEBUG }, 04:53:54 DEBUG 1713498834, 04:53:54 DEBUG "value": [ 04:53:54 DEBUG 04:53:54 DEBUG }, 04:53:54 DEBUG }, 04:53:54 DEBUG "value": [ 04:53:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:53:54 DEBUG }, 04:53:54 DEBUG 04:53:54 DEBUG "pod": "ds-cts-2" 04:53:54 DEBUG }, 04:53:54 DEBUG 1713498834, 04:53:54 DEBUG }, 04:53:54 DEBUG "value": [ 04:53:54 DEBUG "0" 04:53:54 DEBUG 1713498834, 04:53:54 DEBUG 04:53:54 DEBUG "value": [ 04:53:54 DEBUG "value": [ 04:53:54 DEBUG 1713498834, 04:53:54 DEBUG }, 04:53:54 DEBUG "value": [ 04:53:54 DEBUG 04:53:54 DEBUG }, 04:53:54 DEBUG "value": [ 04:53:54 DEBUG 1713498834, 04:53:54 DEBUG "value": [ 04:53:54 DEBUG 1713498834, 04:53:54 DEBUG "36416274.025833845" 04:53:54 DEBUG ] 04:53:54 DEBUG 04:53:54 DEBUG 1713498834, 04:53:54 DEBUG 1713498834, 04:53:54 DEBUG "2149928960" 04:53:54 DEBUG "value": [ 04:53:54 DEBUG 1713498834, 04:53:54 DEBUG "instance": "10.142.0.43:9100" 04:53:54 DEBUG "value": [ 04:53:54 DEBUG "0" 04:53:54 DEBUG "0" 04:53:54 DEBUG 1713498834, 04:53:54 INFO [http_cmd]: http status code OK 04:53:54 DEBUG ] 04:53:54 DEBUG "0" 04:53:54 DEBUG } 04:53:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:53:54 DEBUG "1993.3666666666666" 04:53:54 DEBUG "55.171" 04:53:54 DEBUG ] 04:53:54 DEBUG 1713498834, 04:53:54 DEBUG "1993.4333333333334" 04:53:54 DEBUG }, 04:53:54 DEBUG 1713498834, 04:53:54 DEBUG ] 04:53:54 DEBUG } 04:53:54 DEBUG ] 04:53:54 DEBUG --- status code --- 04:53:54 DEBUG } 04:53:54 DEBUG ] 04:53:54 DEBUG ] 04:53:54 DEBUG }, 04:53:54 DEBUG ] 04:53:54 DEBUG ] 04:53:54 DEBUG } 04:53:54 DEBUG "0" 04:53:54 DEBUG ] 04:53:54 DEBUG "value": [ 04:53:54 DEBUG "0.000017104444444444444" 04:53:54 DEBUG ] 04:53:54 DEBUG "0" 04:53:54 DEBUG } 04:53:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:53:54 DEBUG ] 04:53:54 DEBUG } 04:53:54 DEBUG } 04:53:54 DEBUG "value": [ 04:53:54 DEBUG } 04:53:54 DEBUG } 04:53:54 DEBUG ] 04:53:54 DEBUG ] 04:53:54 DEBUG } 04:53:54 DEBUG 1713498834, 04:53:54 DEBUG ] 04:53:54 DEBUG } 04:53:54 DEBUG ] 04:53:54 DEBUG ] 04:53:54 DEBUG --- http response --- 04:53:54 DEBUG } 04:53:54 DEBUG } 04:53:54 DEBUG } 04:53:54 DEBUG 1713498834, 04:53:54 DEBUG "1061013.5" 04:53:54 DEBUG ] 04:53:54 DEBUG } 04:53:54 DEBUG } 04:53:54 DEBUG ] 04:53:54 DEBUG "0.03600000000000089" 04:53:54 DEBUG } 04:53:54 DEBUG } 04:53:54 DEBUG } 04:53:54 DEBUG } 04:53:54 DEBUG ] 04:53:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:53:54 DEBUG ] 04:53:54 DEBUG ] 04:53:54 DEBUG } 04:53:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498834" 04:53:54 DEBUG } 04:53:54 DEBUG ] 04:53:54 DEBUG } 04:53:54 DEBUG ] 04:53:54 DEBUG ] 04:53:54 DEBUG ] 04:53:54 DEBUG } 04:53:54 DEBUG } 04:53:54 DEBUG ----- output ----- 04:53:54 DEBUG } 04:53:54 DEBUG } 04:53:54 DEBUG } 04:53:54 DEBUG } 04:53:54 DEBUG } 04:53:54 DEBUG } 04:53:54 DEBUG } 04:53:54 DEBUG } 04:53:54 DEBUG } 04:53:54 DEBUG { 04:53:54 DEBUG } 04:53:54 DEBUG ] 04:53:54 DEBUG } 04:53:54 DEBUG ] 04:53:54 DEBUG } 04:53:54 DEBUG "status": "success", 04:53:54 DEBUG } 04:53:54 DEBUG } 04:53:54 DEBUG } 04:53:54 DEBUG "data": { 04:53:54 DEBUG } 04:53:54 DEBUG } 04:53:54 DEBUG "resultType": "vector", 04:53:54 DEBUG "result": [ 04:53:54 DEBUG { 04:53:54 DEBUG "metric": { 04:53:54 DEBUG "pod": "ds-idrepo-0" 04:53:54 DEBUG }, 04:53:54 DEBUG "value": [ 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG [...] 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG "pod": "ds-idrepo-2" 04:53:54 DEBUG }, 04:53:54 DEBUG "value": [ 04:53:54 DEBUG 1713498834, 04:53:54 DEBUG "0" 04:53:54 DEBUG ] 04:53:54 DEBUG } 04:53:54 DEBUG ] 04:53:54 DEBUG } 04:53:54 DEBUG } 04:53:54 INFO [http_cmd]: http status code OK 04:53:54 DEBUG --- status code --- 04:53:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:53:54 DEBUG --- http response --- 04:53:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 04:53:54 DEBUG ----- output ----- 04:53:54 DEBUG { 04:53:54 DEBUG "status": "success", 04:53:54 DEBUG "data": { 04:53:54 DEBUG "resultType": "vector", 04:53:54 DEBUG "result": [ 04:53:54 DEBUG { 04:53:54 DEBUG "metric": { 04:53:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:53:54 DEBUG }, 04:53:54 DEBUG "value": [ 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG [...] 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG "pod": "ds-cts-0" 04:53:54 DEBUG }, 04:53:54 DEBUG "value": [ 04:53:54 DEBUG 1713498834, 04:53:54 DEBUG "125416.32785233692" 04:53:54 DEBUG ] 04:53:54 DEBUG } 04:53:54 DEBUG ] 04:53:54 DEBUG } 04:53:54 DEBUG } 04:53:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498834" 04:53:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498834" 04:53:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498834" 04:53:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498834" 04:53:54 INFO [http_cmd]: http status code OK 04:53:54 DEBUG --- status code --- 04:53:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:53:54 DEBUG --- http response --- 04:53:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:53:54 DEBUG ----- output ----- 04:53:54 DEBUG { 04:53:54 DEBUG "status": "success", 04:53:54 DEBUG "data": { 04:53:54 DEBUG "resultType": "vector", 04:53:54 DEBUG "result": [ 04:53:54 DEBUG { 04:53:54 DEBUG "metric": { 04:53:54 DEBUG "instance": "10.142.0.107:9100" 04:53:54 DEBUG }, 04:53:54 DEBUG "value": [ 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG [...] 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG "instance": "10.142.0.43:9100" 04:53:54 DEBUG }, 04:53:54 DEBUG "value": [ 04:53:54 DEBUG 1713498834, 04:53:54 DEBUG "47792326.766666666" 04:53:54 DEBUG ] 04:53:54 DEBUG } 04:53:54 DEBUG ] 04:53:54 DEBUG } 04:53:54 DEBUG } 04:53:54 INFO [http_cmd]: http status code OK 04:53:54 INFO [http_cmd]: http status code OK 04:53:54 DEBUG --- status code --- 04:53:54 DEBUG --- status code --- 04:53:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:53:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:53:54 DEBUG --- http response --- 04:53:54 DEBUG --- http response --- 04:53:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:53:54 DEBUG ----- output ----- 04:53:54 DEBUG { 04:53:54 DEBUG "status": "success", 04:53:54 DEBUG "data": { 04:53:54 DEBUG "resultType": "vector", 04:53:54 DEBUG "result": [ 04:53:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:53:54 DEBUG { 04:53:54 DEBUG ----- output ----- 04:53:54 DEBUG "metric": { 04:53:54 DEBUG { 04:53:54 DEBUG "pod": "ds-idrepo-0" 04:53:54 DEBUG "status": "success", 04:53:54 DEBUG }, 04:53:54 DEBUG "data": { 04:53:54 DEBUG "value": [ 04:53:54 DEBUG "resultType": "vector", 04:53:54 DEBUG 04:53:54 DEBUG "result": [ 04:53:54 DEBUG 04:53:54 DEBUG { 04:53:54 DEBUG 04:53:54 DEBUG "metric": { 04:53:54 DEBUG [...] 04:53:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:53:54 DEBUG 04:53:54 DEBUG }, 04:53:54 DEBUG 04:53:54 DEBUG "value": [ 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG "pod": "ds-idrepo-2" 04:53:54 DEBUG 04:53:54 DEBUG }, 04:53:54 DEBUG 04:53:54 DEBUG "value": [ 04:53:54 DEBUG [...] 04:53:54 DEBUG 1713498834, 04:53:54 DEBUG 04:53:54 DEBUG "0" 04:53:54 DEBUG 04:53:54 DEBUG ] 04:53:54 DEBUG 04:53:54 DEBUG } 04:53:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:53:54 DEBUG ] 04:53:54 DEBUG }, 04:53:54 DEBUG } 04:53:54 DEBUG "value": [ 04:53:54 DEBUG } 04:53:54 DEBUG 1713498834, 04:53:54 DEBUG "1993.4666666666667" 04:53:54 DEBUG ] 04:53:54 DEBUG } 04:53:54 DEBUG ] 04:53:54 DEBUG } 04:53:54 DEBUG } 04:53:54 INFO [http_cmd]: http status code OK 04:53:54 DEBUG --- status code --- 04:53:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:53:54 DEBUG --- http response --- 04:53:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 04:53:54 DEBUG ----- output ----- 04:53:54 DEBUG { 04:53:54 DEBUG "status": "success", 04:53:54 DEBUG "data": { 04:53:54 DEBUG "resultType": "vector", 04:53:54 DEBUG "result": [ 04:53:54 DEBUG { 04:53:54 DEBUG "metric": { 04:53:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:53:54 DEBUG }, 04:53:54 DEBUG "value": [ 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG [...] 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG "pod": "ds-cts-0" 04:53:54 DEBUG }, 04:53:54 DEBUG "value": [ 04:53:54 DEBUG 1713498834, 04:53:54 DEBUG "2652.7542906487247" 04:53:54 DEBUG ] 04:53:54 DEBUG } 04:53:54 DEBUG ] 04:53:54 DEBUG } 04:53:54 DEBUG } 04:53:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498834" 04:53:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498834" 04:53:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713498834" 04:53:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498834" 04:53:54 INFO [http_cmd]: http status code OK 04:53:54 INFO [http_cmd]: http status code OK 04:53:54 DEBUG --- status code --- 04:53:54 DEBUG --- status code --- 04:53:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:53:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:53:54 DEBUG --- http response --- 04:53:54 DEBUG --- http response --- 04:53:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:53:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:53:54 DEBUG ----- output ----- 04:53:54 DEBUG ----- output ----- 04:53:54 DEBUG { 04:53:54 DEBUG { 04:53:54 DEBUG "status": "success", 04:53:54 DEBUG "status": "success", 04:53:54 DEBUG "data": { 04:53:54 DEBUG "data": { 04:53:54 DEBUG "resultType": "vector", 04:53:54 DEBUG "resultType": "vector", 04:53:54 DEBUG "result": [ 04:53:54 DEBUG "result": [ 04:53:54 DEBUG { 04:53:54 DEBUG { 04:53:54 DEBUG "metric": { 04:53:54 DEBUG "metric": { 04:53:54 DEBUG "instance": "10.142.0.107:9100" 04:53:54 DEBUG "instance": "10.142.0.107:9100" 04:53:54 DEBUG }, 04:53:54 DEBUG }, 04:53:54 DEBUG "value": [ 04:53:54 DEBUG "value": [ 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG [...] 04:53:54 DEBUG [...] 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG "instance": "10.142.0.43:9100" 04:53:54 DEBUG "instance": "10.142.0.43:9100" 04:53:54 DEBUG }, 04:53:54 DEBUG }, 04:53:54 DEBUG "value": [ 04:53:54 DEBUG "value": [ 04:53:54 DEBUG 1713498834, 04:53:54 DEBUG 1713498834, 04:53:54 DEBUG "0.048766666666664564" 04:53:54 DEBUG "406789245952" 04:53:54 DEBUG ] 04:53:54 DEBUG ] 04:53:54 DEBUG } 04:53:54 DEBUG } 04:53:54 DEBUG ] 04:53:54 DEBUG ] 04:53:54 DEBUG } 04:53:54 DEBUG } 04:53:54 DEBUG } 04:53:54 DEBUG } 04:53:54 INFO [http_cmd]: http status code OK 04:53:54 DEBUG --- status code --- 04:53:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:53:54 DEBUG --- http response --- 04:53:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:53:54 DEBUG ----- output ----- 04:53:54 DEBUG { 04:53:54 DEBUG "status": "success", 04:53:54 DEBUG "data": { 04:53:54 DEBUG "resultType": "vector", 04:53:54 DEBUG "result": [ 04:53:54 DEBUG { 04:53:54 DEBUG "metric": { 04:53:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 04:53:54 DEBUG }, 04:53:54 DEBUG "value": [ 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG [...] 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 04:53:54 DEBUG }, 04:53:54 DEBUG "value": [ 04:53:54 DEBUG 1713498834, 04:53:54 DEBUG "6.496358701769182" 04:53:54 DEBUG ] 04:53:54 DEBUG } 04:53:54 DEBUG ] 04:53:54 DEBUG } 04:53:54 DEBUG } 04:53:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498834" 04:53:54 INFO [http_cmd]: http status code OK 04:53:54 DEBUG --- status code --- 04:53:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:53:54 DEBUG --- http response --- 04:53:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:53:54 DEBUG ----- output ----- 04:53:54 DEBUG { 04:53:54 DEBUG "status": "success", 04:53:54 DEBUG "data": { 04:53:54 DEBUG "resultType": "vector", 04:53:54 DEBUG "result": [ 04:53:54 DEBUG { 04:53:54 DEBUG "metric": { 04:53:54 DEBUG "instance": "10.142.0.107:9100" 04:53:54 DEBUG }, 04:53:54 DEBUG "value": [ 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG [...] 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG "instance": "10.142.0.43:9100" 04:53:54 DEBUG }, 04:53:54 DEBUG "value": [ 04:53:54 DEBUG 1713498834, 04:53:54 DEBUG "47795934.66666667" 04:53:54 DEBUG ] 04:53:54 DEBUG } 04:53:54 DEBUG ] 04:53:54 DEBUG } 04:53:54 DEBUG } 04:53:54 INFO [http_cmd]: http status code OK 04:53:54 DEBUG --- status code --- 04:53:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:53:54 DEBUG --- http response --- 04:53:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 04:53:54 DEBUG ----- output ----- 04:53:54 DEBUG { 04:53:54 DEBUG "status": "success", 04:53:54 DEBUG "data": { 04:53:54 DEBUG "resultType": "vector", 04:53:54 DEBUG "result": [ 04:53:54 DEBUG { 04:53:54 DEBUG "metric": { 04:53:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:53:54 DEBUG }, 04:53:54 DEBUG "value": [ 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG [...] 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG 04:53:54 DEBUG "pod": "ds-cts-0" 04:53:54 DEBUG }, 04:53:54 DEBUG "value": [ 04:53:54 DEBUG 1713498834, 04:53:54 DEBUG "6.004972151261365" 04:53:54 DEBUG ] 04:53:54 DEBUG } 04:53:54 DEBUG ] 04:53:54 DEBUG } 04:53:54 DEBUG } 04:53:54 INFO 04:53:54 INFO [loop_until]: kubectl --namespace=xlou top pods 04:53:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:53:54 INFO 04:53:54 INFO [loop_until]: kubectl --namespace=xlou top node 04:53:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:53:54 INFO [loop_until]: OK (rc = 0) 04:53:54 DEBUG --- stdout --- 04:53:54 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 9715m 8942Mi am-6fd554f8f7-hvvkw 9328m 8874Mi am-6fd554f8f7-pmj4b 10192m 9085Mi ds-cts-0 6034m 12178Mi ds-cts-1 5625m 12095Mi ds-cts-2 5890m 12137Mi ds-idrepo-0 4674m 3254Mi ds-idrepo-1 4638m 3207Mi ds-idrepo-2 4521m 3065Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 4m 1215Mi lodemon-6b89d774d5-rf7sk 8m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 6657m 1957Mi 04:53:54 DEBUG --- stderr --- 04:53:54 DEBUG 04:53:55 INFO [loop_until]: OK (rc = 0) 04:53:55 DEBUG --- stdout --- 04:53:55 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 78m 0% 6245Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 4575m 15% 6692Mi 5% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9463m 31% 11591Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 86m 0% 3255Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 6008m 20% 14795Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 162m 0% 5187Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 9699m 32% 11722Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 10271m 34% 2728Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 10842m 36% 11172Mi 10% gke-xlou-cdm-ds-0f601751-j488 4870m 16% 6145Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 5663m 18% 16170Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 5093m 17% 6875Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 5916m 19% 14158Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 6661m 41% 5719Mi 9% 04:53:55 DEBUG --- stderr --- 04:53:55 DEBUG 04:54:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498894" 04:54:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498894" 04:54:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498894" 04:54:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713498894" 04:54:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498894" 04:54:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498894" 04:54:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498894" 04:54:54 INFO [http_cmd]: http status code OK 04:54:54 DEBUG --- status code --- 04:54:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:54:54 DEBUG --- http response --- 04:54:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 04:54:54 DEBUG ----- output ----- 04:54:54 DEBUG { 04:54:54 DEBUG "status": "success", 04:54:54 DEBUG "data": { 04:54:54 DEBUG "resultType": "vector", 04:54:54 DEBUG "result": [ 04:54:54 DEBUG { 04:54:54 DEBUG "metric": { 04:54:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:54:54 DEBUG }, 04:54:54 DEBUG "value": [ 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG [...] 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 04:54:54 DEBUG }, 04:54:54 DEBUG "value": [ 04:54:54 DEBUG 1713498894, 04:54:54 DEBUG "0" 04:54:54 DEBUG ] 04:54:54 DEBUG } 04:54:54 DEBUG ] 04:54:54 DEBUG } 04:54:54 DEBUG } 04:54:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713498894" 04:54:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713498894" 04:54:54 INFO [http_cmd]: http status code OK 04:54:54 INFO [http_cmd]: http status code OK 04:54:54 INFO [http_cmd]: http status code OK 04:54:54 DEBUG --- status code --- 04:54:54 INFO [http_cmd]: http status code OK 04:54:54 INFO [http_cmd]: http status code OK 04:54:54 DEBUG --- status code --- 04:54:54 DEBUG --- status code --- 04:54:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:54:54 DEBUG --- status code --- 04:54:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498894" 04:54:54 DEBUG --- status code --- 04:54:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:54:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:54:54 DEBUG --- http response --- 04:54:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:54:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:54:54 DEBUG --- http response --- 04:54:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498894" 04:54:54 DEBUG --- http response --- 04:54:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 04:54:54 DEBUG --- http response --- 04:54:54 DEBUG --- http response --- 04:54:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:54:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:54:54 INFO [http_cmd]: http status code OK 04:54:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:54:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:54:54 DEBUG ----- output ----- 04:54:54 DEBUG ----- output ----- 04:54:54 DEBUG --- status code --- 04:54:54 DEBUG ----- output ----- 04:54:54 DEBUG ----- output ----- 04:54:54 DEBUG { 04:54:54 DEBUG { 04:54:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:54:54 DEBUG { 04:54:54 DEBUG { 04:54:54 DEBUG "status": "success", 04:54:54 DEBUG "status": "success", 04:54:54 DEBUG --- http response --- 04:54:54 DEBUG "status": "success", 04:54:54 DEBUG "status": "success", 04:54:54 DEBUG "data": { 04:54:54 DEBUG "data": { 04:54:54 DEBUG "data": { 04:54:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:54:54 DEBUG "data": { 04:54:54 DEBUG "resultType": "vector", 04:54:54 DEBUG "resultType": "vector", 04:54:54 DEBUG "resultType": "vector", 04:54:54 DEBUG ----- output ----- 04:54:54 DEBUG "resultType": "vector", 04:54:54 DEBUG "result": [ 04:54:54 DEBUG "result": [ 04:54:54 DEBUG "result": [ 04:54:54 DEBUG { 04:54:54 DEBUG "result": [ 04:54:54 DEBUG { 04:54:54 DEBUG { 04:54:54 DEBUG { 04:54:54 DEBUG "status": "success", 04:54:54 DEBUG { 04:54:54 DEBUG "metric": { 04:54:54 DEBUG "metric": { 04:54:54 DEBUG "metric": { 04:54:54 DEBUG "data": { 04:54:54 DEBUG "metric": { 04:54:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:54:54 DEBUG "pod": "ds-cts-0" 04:54:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:54:54 DEBUG "resultType": "vector", 04:54:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 04:54:54 DEBUG }, 04:54:54 DEBUG }, 04:54:54 DEBUG }, 04:54:54 DEBUG "result": [ 04:54:54 DEBUG }, 04:54:54 DEBUG "value": [ 04:54:54 DEBUG "value": [ 04:54:54 DEBUG "value": [ 04:54:54 DEBUG { 04:54:54 DEBUG "value": [ 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG "metric": { 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG }, 04:54:54 DEBUG 04:54:54 DEBUG [...] 04:54:54 DEBUG [...] 04:54:54 DEBUG [...] 04:54:54 DEBUG "value": [ 04:54:54 DEBUG [...] 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 ERROR [http_cmd]: ERROR 04:54:54 DEBUG 04:54:54 INFO [http_cmd]: http status code OK 04:54:54 INFO [http_cmd]: http status code OK 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498894" 04:54:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:54:54 DEBUG 04:54:54 ERROR --- status code --- 04:54:54 DEBUG 04:54:54 DEBUG --- status code --- 04:54:54 DEBUG --- status code --- 04:54:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:54:54 DEBUG "pod": "ds-cts-2" 04:54:54 DEBUG }, 04:54:54 DEBUG [...] 04:54:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 04:54:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 04:54:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:54:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:54:54 DEBUG }, 04:54:54 DEBUG }, 04:54:54 DEBUG "value": [ 04:54:54 DEBUG 04:54:54 ERROR --- http response --- 04:54:54 DEBUG }, 04:54:54 DEBUG --- http response --- 04:54:54 DEBUG --- http response --- 04:54:54 DEBUG "value": [ 04:54:54 DEBUG "value": [ 04:54:54 DEBUG 1713498894, 04:54:54 DEBUG 04:54:54 DEBUG "value": [ 04:54:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 04:54:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498894" 04:54:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:54:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:54:54 DEBUG 1713498894, 04:54:54 DEBUG 1713498894, 04:54:54 DEBUG "1180774.5" 04:54:54 DEBUG 04:54:54 DEBUG 1713498894, 04:54:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 04:54:54 DEBUG ----- output ----- 04:54:54 DEBUG ----- output ----- 04:54:54 INFO [http_cmd]: http status code OK 04:54:54 DEBUG "0" 04:54:54 DEBUG "0.00003420888888888889" 04:54:54 DEBUG ] 04:54:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:54:54 DEBUG "2151624704" 04:54:54 DEBUG { 04:54:54 DEBUG { 04:54:54 DEBUG --- status code --- 04:54:54 DEBUG ] 04:54:54 DEBUG ] 04:54:54 DEBUG } 04:54:54 DEBUG }, 04:54:54 DEBUG ] 04:54:54 DEBUG "status": "success", 04:54:54 DEBUG "status": "success", 04:54:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:54:54 DEBUG } 04:54:54 DEBUG } 04:54:54 DEBUG ] 04:54:54 DEBUG "value": [ 04:54:54 DEBUG } 04:54:54 DEBUG "data": { 04:54:54 DEBUG "data": { 04:54:54 DEBUG --- http response --- 04:54:54 DEBUG ] 04:54:54 DEBUG ] 04:54:54 DEBUG } 04:54:54 DEBUG 1713498894, 04:54:54 DEBUG ] 04:54:54 DEBUG "resultType": "vector", 04:54:54 DEBUG "resultType": "vector", 04:54:54 DEBUG } 04:54:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:54:54 DEBUG } 04:54:54 DEBUG } 04:54:54 DEBUG "0" 04:54:54 DEBUG } 04:54:54 DEBUG "result": [ 04:54:54 DEBUG "result": [ 04:54:54 DEBUG } 04:54:54 DEBUG ----- output ----- 04:54:54 DEBUG } 04:54:54 DEBUG ] 04:54:54 DEBUG } 04:54:54 DEBUG { 04:54:54 DEBUG { 04:54:54 DEBUG { 04:54:54 DEBUG } 04:54:54 DEBUG "metric": { 04:54:54 DEBUG "metric": { 04:54:54 DEBUG "status": "success", 04:54:54 DEBUG ] 04:54:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:54:54 DEBUG "pod": "ds-idrepo-0" 04:54:54 DEBUG "data": { 04:54:54 DEBUG } 04:54:54 DEBUG }, 04:54:54 DEBUG }, 04:54:54 DEBUG "resultType": "vector", 04:54:54 DEBUG } 04:54:54 DEBUG "value": [ 04:54:54 DEBUG "value": [ 04:54:54 DEBUG "result": [ 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG { 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG "metric": { 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG "instance": "10.142.0.107:9100" 04:54:54 DEBUG [...] 04:54:54 DEBUG [...] 04:54:54 DEBUG }, 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG "value": [ 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:54:54 DEBUG "pod": "ds-cts-2" 04:54:54 DEBUG 04:54:54 DEBUG }, 04:54:54 DEBUG }, 04:54:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498894" 04:54:54 DEBUG [...] 04:54:54 DEBUG "value": [ 04:54:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498894" 04:54:54 DEBUG "value": [ 04:54:54 DEBUG 04:54:54 DEBUG 1713498894, 04:54:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498894" 04:54:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498894" 04:54:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713498894" 04:54:54 INFO [http_cmd]: http status code OK 04:54:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498894" 04:54:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498894" 04:54:54 DEBUG 1713498894, 04:54:54 DEBUG "0" 04:54:54 DEBUG 04:54:54 DEBUG "1995.0333333333333" 04:54:54 DEBUG --- status code --- 04:54:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498894" 04:54:54 INFO [http_cmd]: http status code OK 04:54:54 DEBUG ] 04:54:54 DEBUG 04:54:54 DEBUG ] 04:54:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:54:54 DEBUG --- status code --- 04:54:54 DEBUG } 04:54:54 DEBUG "instance": "10.142.0.43:9100" 04:54:54 INFO [http_cmd]: http status code OK 04:54:54 DEBUG } 04:54:54 DEBUG --- http response --- 04:54:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:54:54 DEBUG ] 04:54:54 DEBUG }, 04:54:54 DEBUG --- status code --- 04:54:54 DEBUG ] 04:54:54 INFO [http_cmd]: http status code OK 04:54:54 INFO [http_cmd]: http status code OK 04:54:54 INFO [http_cmd]: http status code OK 04:54:54 INFO [http_cmd]: http status code OK 04:54:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:54:54 DEBUG --- http response --- 04:54:54 DEBUG } 04:54:54 DEBUG "value": [ 04:54:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:54:54 DEBUG } 04:54:54 DEBUG --- status code --- 04:54:54 DEBUG --- status code --- 04:54:54 DEBUG --- status code --- 04:54:54 DEBUG --- status code --- 04:54:54 DEBUG ----- output ----- 04:54:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:54:54 DEBUG ----- output ----- 04:54:54 DEBUG 1713498894, 04:54:54 DEBUG --- http response --- 04:54:54 DEBUG } 04:54:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:54:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:54:54 INFO [http_cmd]: http status code OK 04:54:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:54:54 INFO [http_cmd]: http status code OK 04:54:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:54:54 DEBUG { 04:54:54 DEBUG } 04:54:54 DEBUG { 04:54:54 DEBUG "47782210.5" 04:54:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:54:54 DEBUG --- http response --- 04:54:54 DEBUG --- http response --- 04:54:54 DEBUG --- status code --- 04:54:54 DEBUG --- http response --- 04:54:54 DEBUG --- status code --- 04:54:54 DEBUG --- http response --- 04:54:54 DEBUG "status": "success", 04:54:54 DEBUG "status": "success", 04:54:54 DEBUG ] 04:54:54 DEBUG ----- output ----- 04:54:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:54:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:54:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:54:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:54:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:54:54 DEBUG "data": { 04:54:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:54:54 DEBUG "data": { 04:54:54 DEBUG } 04:54:54 DEBUG { 04:54:54 DEBUG ----- output ----- 04:54:54 DEBUG ----- output ----- 04:54:54 DEBUG --- http response --- 04:54:54 DEBUG --- http response --- 04:54:54 DEBUG ----- output ----- 04:54:54 DEBUG "resultType": "vector", 04:54:54 DEBUG ----- output ----- 04:54:54 DEBUG "resultType": "vector", 04:54:54 DEBUG ] 04:54:54 DEBUG "status": "success", 04:54:54 DEBUG { 04:54:54 DEBUG { 04:54:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:54:54 DEBUG { 04:54:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:54:54 DEBUG "result": [ 04:54:54 DEBUG { 04:54:54 DEBUG "result": [ 04:54:54 DEBUG } 04:54:54 DEBUG "data": { 04:54:54 DEBUG "status": "success", 04:54:54 DEBUG "status": "success", 04:54:54 DEBUG ----- output ----- 04:54:54 DEBUG "status": "success", 04:54:54 DEBUG ----- output ----- 04:54:54 DEBUG { 04:54:54 DEBUG "status": "success", 04:54:54 DEBUG { 04:54:54 DEBUG } 04:54:54 DEBUG "resultType": "vector", 04:54:54 DEBUG "data": { 04:54:54 DEBUG "data": { 04:54:54 DEBUG { 04:54:54 DEBUG "data": { 04:54:54 DEBUG { 04:54:54 DEBUG "metric": { 04:54:54 DEBUG "data": { 04:54:54 DEBUG "metric": { 04:54:54 DEBUG "result": [ 04:54:54 DEBUG "resultType": "vector", 04:54:54 DEBUG "resultType": "vector", 04:54:54 DEBUG "status": "success", 04:54:54 DEBUG "resultType": "vector", 04:54:54 DEBUG "status": "success", 04:54:54 DEBUG "instance": "10.142.0.107:9100" 04:54:54 DEBUG "resultType": "vector", 04:54:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:54:54 DEBUG { 04:54:54 DEBUG "result": [ 04:54:54 DEBUG "result": [ 04:54:54 DEBUG "data": { 04:54:54 DEBUG "result": [ 04:54:54 DEBUG "data": { 04:54:54 DEBUG }, 04:54:54 DEBUG "result": [ 04:54:54 DEBUG }, 04:54:54 DEBUG "metric": { 04:54:54 DEBUG { 04:54:54 DEBUG { 04:54:54 DEBUG "metric": { 04:54:54 DEBUG { 04:54:54 DEBUG "resultType": "vector", 04:54:54 DEBUG "value": [ 04:54:54 DEBUG { 04:54:54 DEBUG "value": [ 04:54:54 DEBUG "pod": "ds-cts-0" 04:54:54 INFO [http_cmd]: http status code OK 04:54:54 DEBUG "metric": { 04:54:54 DEBUG "resultType": "vector", 04:54:54 DEBUG "pod": "ds-cts-0" 04:54:54 DEBUG "metric": { 04:54:54 DEBUG "result": [ 04:54:54 DEBUG 04:54:54 DEBUG "metric": { 04:54:54 DEBUG 04:54:54 DEBUG }, 04:54:54 DEBUG --- status code --- 04:54:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:54:54 DEBUG "result": [ 04:54:54 DEBUG }, 04:54:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:54:54 DEBUG { 04:54:54 DEBUG 04:54:54 DEBUG "pod": "ds-idrepo-0" 04:54:54 DEBUG 04:54:54 DEBUG "value": [ 04:54:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:54:54 DEBUG }, 04:54:54 DEBUG { 04:54:54 DEBUG "value": [ 04:54:54 DEBUG }, 04:54:54 DEBUG "metric": { 04:54:54 DEBUG 04:54:54 DEBUG }, 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG --- http response --- 04:54:54 DEBUG "value": [ 04:54:54 DEBUG "metric": { 04:54:54 DEBUG 04:54:54 DEBUG "value": [ 04:54:54 DEBUG "pod": "ds-idrepo-0" 04:54:54 DEBUG [...] 04:54:54 DEBUG "value": [ 04:54:54 DEBUG [...] 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 04:54:54 DEBUG "pod": "ds-idrepo-0" 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG }, 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG ----- output ----- 04:54:54 DEBUG }, 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG "value": [ 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG [...] 04:54:54 DEBUG 04:54:54 DEBUG { 04:54:54 DEBUG "value": [ 04:54:54 DEBUG [...] 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG [...] 04:54:54 DEBUG "status": "success", 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG [...] 04:54:54 DEBUG 04:54:54 DEBUG "instance": "10.142.0.43:9100" 04:54:54 DEBUG [...] 04:54:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG "data": { 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG }, 04:54:54 DEBUG 04:54:54 DEBUG }, 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG "resultType": "vector", 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG [...] 04:54:54 DEBUG "value": [ 04:54:54 DEBUG 04:54:54 DEBUG "value": [ 04:54:54 DEBUG "pod": "ds-cts-2" 04:54:54 DEBUG 04:54:54 DEBUG "result": [ 04:54:54 DEBUG [...] 04:54:54 DEBUG "pod": "ds-cts-2" 04:54:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498894" 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG 1713498894, 04:54:54 DEBUG 04:54:54 DEBUG 1713498894, 04:54:54 DEBUG "1995.1333333333334" 04:54:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:54:54 DEBUG { 04:54:54 DEBUG 04:54:54 DEBUG }, 04:54:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:54:54 DEBUG 04:54:54 DEBUG "0.0290000000000011" 04:54:54 DEBUG "pod": "ds-idrepo-2" 04:54:54 DEBUG ] 04:54:54 DEBUG }, 04:54:54 DEBUG }, 04:54:54 DEBUG "metric": { 04:54:54 DEBUG 04:54:54 DEBUG "value": [ 04:54:54 DEBUG }, 04:54:54 DEBUG 04:54:54 DEBUG ] 04:54:54 DEBUG }, 04:54:54 DEBUG } 04:54:54 DEBUG "value": [ 04:54:54 DEBUG "value": [ 04:54:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:54:54 DEBUG 04:54:54 DEBUG 1713498894, 04:54:54 DEBUG "value": [ 04:54:54 DEBUG "pod": "ds-idrepo-2" 04:54:54 DEBUG } 04:54:54 DEBUG "value": [ 04:54:54 DEBUG ] 04:54:54 DEBUG 1713498894, 04:54:54 DEBUG 1713498894, 04:54:54 DEBUG }, 04:54:54 DEBUG "pod": "ds-idrepo-2" 04:54:54 DEBUG "0" 04:54:54 DEBUG 1713498894, 04:54:54 DEBUG }, 04:54:54 DEBUG ] 04:54:54 DEBUG 1713498894, 04:54:54 DEBUG } 04:54:54 DEBUG "0.1" 04:54:54 DEBUG "55.171" 04:54:54 DEBUG "value": [ 04:54:54 DEBUG }, 04:54:54 DEBUG ] 04:54:54 DEBUG "1995.4666666666667" 04:54:54 DEBUG "value": [ 04:54:54 DEBUG } 04:54:54 DEBUG "0" 04:54:54 DEBUG } 04:54:54 DEBUG ] 04:54:54 DEBUG ] 04:54:54 DEBUG 04:54:54 DEBUG "value": [ 04:54:54 DEBUG } 04:54:54 DEBUG ] 04:54:54 DEBUG 1713498894, 04:54:54 DEBUG } 04:54:54 DEBUG ] 04:54:54 DEBUG } 04:54:54 DEBUG } 04:54:54 DEBUG 04:54:54 DEBUG 1713498894, 04:54:54 DEBUG ] 04:54:54 DEBUG } 04:54:54 DEBUG "0" 04:54:54 DEBUG } 04:54:54 DEBUG ] 04:54:54 DEBUG ] 04:54:54 DEBUG 04:54:54 DEBUG "0" 04:54:54 DEBUG } 04:54:54 DEBUG ] 04:54:54 DEBUG ] 04:54:54 DEBUG ] 04:54:54 DEBUG } 04:54:54 DEBUG } 04:54:54 DEBUG [...] 04:54:54 DEBUG ] 04:54:54 DEBUG } 04:54:54 DEBUG } 04:54:54 DEBUG } 04:54:54 DEBUG } 04:54:54 DEBUG } 04:54:54 DEBUG } 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG } 04:54:54 DEBUG ] 04:54:54 DEBUG } 04:54:54 DEBUG 04:54:54 DEBUG } 04:54:54 DEBUG ] 04:54:54 DEBUG } 04:54:54 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 04:54:54 DEBUG } 04:54:54 DEBUG } 04:54:54 DEBUG }, 04:54:54 DEBUG } 04:54:54 DEBUG "value": [ 04:54:54 DEBUG 1713498894, 04:54:54 DEBUG "0" 04:54:54 DEBUG ] 04:54:54 DEBUG } 04:54:54 DEBUG ] 04:54:54 DEBUG } 04:54:54 DEBUG } 04:54:54 INFO [http_cmd]: http status code OK 04:54:54 DEBUG --- status code --- 04:54:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:54:54 DEBUG --- http response --- 04:54:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 04:54:54 DEBUG ----- output ----- 04:54:54 DEBUG { 04:54:54 DEBUG "status": "success", 04:54:54 DEBUG "data": { 04:54:54 DEBUG "resultType": "vector", 04:54:54 DEBUG "result": [ 04:54:54 DEBUG { 04:54:54 DEBUG "metric": { 04:54:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:54:54 DEBUG }, 04:54:54 DEBUG "value": [ 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG [...] 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 04:54:54 DEBUG }, 04:54:54 DEBUG "value": [ 04:54:54 DEBUG 1713498894, 04:54:54 DEBUG "0" 04:54:54 DEBUG ] 04:54:54 DEBUG } 04:54:54 DEBUG ] 04:54:54 DEBUG } 04:54:54 DEBUG } 04:54:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498894" 04:54:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498894" 04:54:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713498894" 04:54:54 INFO [http_cmd]: http status code OK 04:54:54 DEBUG --- status code --- 04:54:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:54:54 DEBUG --- http response --- 04:54:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:54:54 DEBUG ----- output ----- 04:54:54 DEBUG { 04:54:54 DEBUG "status": "success", 04:54:54 DEBUG "data": { 04:54:54 DEBUG "resultType": "vector", 04:54:54 DEBUG "result": [ 04:54:54 DEBUG { 04:54:54 DEBUG "metric": { 04:54:54 DEBUG "instance": "10.142.0.107:9100" 04:54:54 DEBUG }, 04:54:54 DEBUG "value": [ 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG [...] 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG "instance": "10.142.0.43:9100" 04:54:54 DEBUG }, 04:54:54 DEBUG "value": [ 04:54:54 DEBUG 1713498894, 04:54:54 DEBUG "0.043266666666672414" 04:54:54 DEBUG ] 04:54:54 DEBUG } 04:54:54 DEBUG ] 04:54:54 DEBUG } 04:54:54 DEBUG } 04:54:54 INFO [http_cmd]: http status code OK 04:54:54 DEBUG --- status code --- 04:54:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:54:54 DEBUG --- http response --- 04:54:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:54:54 DEBUG ----- output ----- 04:54:54 DEBUG { 04:54:54 DEBUG "status": "success", 04:54:54 DEBUG "data": { 04:54:54 DEBUG "resultType": "vector", 04:54:54 DEBUG "result": [ 04:54:54 DEBUG { 04:54:54 DEBUG "metric": { 04:54:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 04:54:54 DEBUG }, 04:54:54 DEBUG "value": [ 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG [...] 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 04:54:54 DEBUG }, 04:54:54 DEBUG "value": [ 04:54:54 DEBUG 1713498894, 04:54:54 DEBUG "6.4949403814044615" 04:54:54 DEBUG ] 04:54:54 DEBUG } 04:54:54 DEBUG ] 04:54:54 DEBUG } 04:54:54 DEBUG } 04:54:54 INFO [http_cmd]: http status code OK 04:54:54 DEBUG --- status code --- 04:54:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:54:54 DEBUG --- http response --- 04:54:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:54:54 DEBUG ----- output ----- 04:54:54 DEBUG { 04:54:54 DEBUG "status": "success", 04:54:54 DEBUG "data": { 04:54:54 DEBUG "resultType": "vector", 04:54:54 DEBUG "result": [ 04:54:54 DEBUG { 04:54:54 DEBUG "metric": { 04:54:54 DEBUG "instance": "10.142.0.107:9100" 04:54:54 DEBUG }, 04:54:54 DEBUG "value": [ 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG [...] 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG "instance": "10.142.0.43:9100" 04:54:54 DEBUG }, 04:54:54 DEBUG "value": [ 04:54:54 DEBUG 1713498894, 04:54:54 DEBUG "409050121216" 04:54:54 DEBUG ] 04:54:54 DEBUG } 04:54:54 DEBUG ] 04:54:54 DEBUG } 04:54:54 DEBUG } 04:54:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498894" 04:54:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498894" 04:54:54 INFO [http_cmd]: http status code OK 04:54:54 DEBUG --- status code --- 04:54:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:54:54 DEBUG --- http response --- 04:54:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:54:54 DEBUG ----- output ----- 04:54:54 DEBUG { 04:54:54 DEBUG "status": "success", 04:54:54 DEBUG "data": { 04:54:54 DEBUG "resultType": "vector", 04:54:54 DEBUG "result": [ 04:54:54 DEBUG { 04:54:54 DEBUG "metric": { 04:54:54 DEBUG "instance": "10.142.0.107:9100" 04:54:54 DEBUG }, 04:54:54 DEBUG "value": [ 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG [...] 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG "instance": "10.142.0.43:9100" 04:54:54 DEBUG }, 04:54:54 DEBUG "value": [ 04:54:54 DEBUG 1713498894, 04:54:54 DEBUG "47786089.43333333" 04:54:54 DEBUG ] 04:54:54 DEBUG } 04:54:54 DEBUG ] 04:54:54 DEBUG } 04:54:54 DEBUG } 04:54:54 INFO [http_cmd]: http status code OK 04:54:54 DEBUG --- status code --- 04:54:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:54:54 DEBUG --- http response --- 04:54:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 04:54:54 DEBUG ----- output ----- 04:54:54 DEBUG { 04:54:54 DEBUG "status": "success", 04:54:54 DEBUG "data": { 04:54:54 DEBUG "resultType": "vector", 04:54:54 DEBUG "result": [ 04:54:54 DEBUG { 04:54:54 DEBUG "metric": { 04:54:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:54:54 DEBUG }, 04:54:54 DEBUG "value": [ 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG [...] 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG 04:54:54 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 04:54:54 DEBUG }, 04:54:54 DEBUG "value": [ 04:54:54 DEBUG 1713498894, 04:54:54 DEBUG "0.0000178976003655436" 04:54:54 DEBUG ] 04:54:54 DEBUG } 04:54:54 DEBUG ] 04:54:54 DEBUG } 04:54:54 DEBUG } 04:54:55 INFO 04:54:55 INFO [loop_until]: kubectl --namespace=xlou top pods 04:54:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:54:55 INFO 04:54:55 INFO [loop_until]: kubectl --namespace=xlou top node 04:54:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:54:55 INFO [loop_until]: OK (rc = 0) 04:54:55 DEBUG --- stdout --- 04:54:55 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 9684m 8941Mi am-6fd554f8f7-hvvkw 9520m 8874Mi am-6fd554f8f7-pmj4b 10222m 8971Mi ds-cts-0 6220m 12183Mi ds-cts-1 6225m 12102Mi ds-cts-2 6182m 12143Mi ds-idrepo-0 4637m 3253Mi ds-idrepo-1 4675m 3206Mi ds-idrepo-2 4473m 3066Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1172Mi idm-b85f9d978-wnhxb 3m 1215Mi lodemon-6b89d774d5-rf7sk 8m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 6440m 1958Mi 04:54:55 DEBUG --- stderr --- 04:54:55 DEBUG 04:54:55 INFO [loop_until]: OK (rc = 0) 04:54:55 DEBUG --- stdout --- 04:54:55 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 79m 0% 6261Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 4567m 15% 6697Mi 5% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9350m 31% 11595Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 86m 0% 3255Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 6529m 21% 14790Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 160m 0% 5177Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 9681m 32% 11712Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 10287m 34% 2734Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 10890m 36% 11164Mi 9% gke-xlou-cdm-ds-0f601751-j488 4839m 16% 6156Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 6141m 20% 16173Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 5109m 17% 6875Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 6306m 21% 14160Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 6636m 41% 5734Mi 9% 04:54:55 DEBUG --- stderr --- 04:54:55 DEBUG 04:55:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498954" 04:55:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498954" 04:55:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713498954" 04:55:54 INFO [http_cmd]: http status code OK 04:55:54 INFO [http_cmd]: http status code OK 04:55:54 DEBUG --- status code --- 04:55:54 DEBUG --- status code --- 04:55:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:55:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:55:54 DEBUG --- http response --- 04:55:54 DEBUG --- http response --- 04:55:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 04:55:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 04:55:54 DEBUG ----- output ----- 04:55:54 DEBUG { 04:55:54 DEBUG "status": "success", 04:55:54 DEBUG "data": { 04:55:54 DEBUG "resultType": "vector", 04:55:54 DEBUG "result": [ 04:55:54 DEBUG { 04:55:54 DEBUG "metric": { 04:55:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:55:54 DEBUG }, 04:55:54 DEBUG "value": [ 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG [...] 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 04:55:54 DEBUG }, 04:55:54 DEBUG "value": [ 04:55:54 DEBUG 1713498954, 04:55:54 DEBUG "0" 04:55:54 DEBUG ] 04:55:54 DEBUG } 04:55:54 DEBUG ] 04:55:54 DEBUG } 04:55:54 DEBUG } 04:55:54 ERROR [http_cmd]: ERROR 04:55:54 ERROR --- status code --- 04:55:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 04:55:54 ERROR --- http response --- 04:55:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 04:55:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 04:55:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498954" 04:55:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498954" 04:55:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498954" 04:55:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498954" 04:55:54 INFO [http_cmd]: http status code OK 04:55:54 INFO [http_cmd]: http status code OK 04:55:54 DEBUG --- status code --- 04:55:54 DEBUG --- status code --- 04:55:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:55:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:55:54 DEBUG --- http response --- 04:55:54 DEBUG --- http response --- 04:55:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:55:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:55:54 DEBUG ----- output ----- 04:55:54 DEBUG ----- output ----- 04:55:54 DEBUG { 04:55:54 DEBUG { 04:55:54 DEBUG "status": "success", 04:55:54 DEBUG "status": "success", 04:55:54 DEBUG "data": { 04:55:54 DEBUG "data": { 04:55:54 DEBUG "resultType": "vector", 04:55:54 DEBUG "resultType": "vector", 04:55:54 DEBUG "result": [ 04:55:54 DEBUG "result": [ 04:55:54 DEBUG { 04:55:54 DEBUG { 04:55:54 DEBUG "metric": { 04:55:54 DEBUG "metric": { 04:55:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:55:54 DEBUG "pod": "ds-cts-0" 04:55:54 DEBUG }, 04:55:54 DEBUG }, 04:55:54 DEBUG "value": [ 04:55:54 DEBUG "value": [ 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG [...] 04:55:54 DEBUG [...] 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:55:54 DEBUG "pod": "ds-cts-2" 04:55:54 DEBUG }, 04:55:54 DEBUG }, 04:55:54 DEBUG "value": [ 04:55:54 DEBUG "value": [ 04:55:54 DEBUG 1713498954, 04:55:54 DEBUG 1713498954, 04:55:54 DEBUG "1300563.5" 04:55:54 DEBUG "0.000017104444444444444" 04:55:54 DEBUG ] 04:55:54 DEBUG ] 04:55:54 DEBUG } 04:55:54 DEBUG } 04:55:54 DEBUG ] 04:55:54 DEBUG ] 04:55:54 DEBUG } 04:55:54 DEBUG } 04:55:54 DEBUG } 04:55:54 DEBUG } 04:55:54 INFO [http_cmd]: http status code OK 04:55:54 DEBUG --- status code --- 04:55:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:55:54 DEBUG --- http response --- 04:55:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:55:54 DEBUG ----- output ----- 04:55:54 DEBUG { 04:55:54 DEBUG "status": "success", 04:55:54 DEBUG "data": { 04:55:54 DEBUG "resultType": "vector", 04:55:54 DEBUG "result": [ 04:55:54 DEBUG { 04:55:54 DEBUG "metric": { 04:55:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:55:54 DEBUG }, 04:55:54 DEBUG "value": [ 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG [...] 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:55:54 DEBUG }, 04:55:54 DEBUG "value": [ 04:55:54 DEBUG 1713498954, 04:55:54 DEBUG "1998.3" 04:55:54 DEBUG ] 04:55:54 DEBUG } 04:55:54 DEBUG ] 04:55:54 DEBUG } 04:55:54 DEBUG } 04:55:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498954" 04:55:54 INFO [http_cmd]: http status code OK 04:55:54 DEBUG --- status code --- 04:55:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:55:54 DEBUG --- http response --- 04:55:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 04:55:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498954" 04:55:54 DEBUG ----- output ----- 04:55:54 DEBUG { 04:55:54 DEBUG "status": "success", 04:55:54 DEBUG "data": { 04:55:54 DEBUG "resultType": "vector", 04:55:54 DEBUG "result": [ 04:55:54 DEBUG { 04:55:54 DEBUG "metric": { 04:55:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:55:54 DEBUG }, 04:55:54 DEBUG "value": [ 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG [...] 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 04:55:54 DEBUG }, 04:55:54 DEBUG "value": [ 04:55:54 DEBUG 1713498954, 04:55:54 DEBUG "43911.9010371361" 04:55:54 DEBUG ] 04:55:54 DEBUG } 04:55:54 DEBUG ] 04:55:54 DEBUG } 04:55:54 DEBUG } 04:55:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713498954" 04:55:54 INFO [http_cmd]: http status code OK 04:55:54 DEBUG --- status code --- 04:55:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:55:54 DEBUG --- http response --- 04:55:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:55:54 DEBUG ----- output ----- 04:55:54 DEBUG { 04:55:54 DEBUG "status": "success", 04:55:54 DEBUG "data": { 04:55:54 DEBUG "resultType": "vector", 04:55:54 DEBUG "result": [ 04:55:54 DEBUG { 04:55:54 DEBUG "metric": { 04:55:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:55:54 DEBUG }, 04:55:54 DEBUG "value": [ 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG [...] 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:55:54 DEBUG }, 04:55:54 DEBUG "value": [ 04:55:54 DEBUG 1713498954, 04:55:54 DEBUG "0" 04:55:54 DEBUG ] 04:55:54 DEBUG } 04:55:54 DEBUG ] 04:55:54 DEBUG } 04:55:54 DEBUG } 04:55:54 INFO [http_cmd]: http status code OK 04:55:54 DEBUG --- status code --- 04:55:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:55:54 DEBUG --- http response --- 04:55:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:55:54 DEBUG ----- output ----- 04:55:54 DEBUG { 04:55:54 DEBUG "status": "success", 04:55:54 DEBUG "data": { 04:55:54 DEBUG "resultType": "vector", 04:55:54 DEBUG "result": [ 04:55:54 DEBUG { 04:55:54 DEBUG "metric": { 04:55:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:55:54 DEBUG }, 04:55:54 DEBUG "value": [ 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG [...] 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:55:54 DEBUG }, 04:55:54 DEBUG "value": [ 04:55:54 DEBUG 1713498954, 04:55:54 DEBUG "0" 04:55:54 DEBUG ] 04:55:54 DEBUG } 04:55:54 DEBUG ] 04:55:54 DEBUG } 04:55:54 DEBUG } 04:55:54 INFO [http_cmd]: http status code OK 04:55:54 DEBUG --- status code --- 04:55:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:55:54 DEBUG --- http response --- 04:55:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:55:54 DEBUG ----- output ----- 04:55:54 DEBUG { 04:55:54 DEBUG "status": "success", 04:55:54 DEBUG "data": { 04:55:54 DEBUG "resultType": "vector", 04:55:54 DEBUG "result": [ 04:55:54 DEBUG { 04:55:54 DEBUG "metric": { 04:55:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 04:55:54 DEBUG }, 04:55:54 DEBUG "value": [ 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG [...] 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 04:55:54 DEBUG }, 04:55:54 DEBUG "value": [ 04:55:54 DEBUG 1713498954, 04:55:54 DEBUG "2153480192" 04:55:54 DEBUG ] 04:55:54 DEBUG } 04:55:54 DEBUG ] 04:55:54 DEBUG } 04:55:54 DEBUG } 04:55:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498954" 04:55:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498954" 04:55:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713498954" 04:55:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498954" 04:55:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498954" 04:55:54 INFO [http_cmd]: http status code OK 04:55:54 DEBUG --- status code --- 04:55:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:55:54 DEBUG --- http response --- 04:55:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:55:54 DEBUG ----- output ----- 04:55:54 DEBUG { 04:55:54 DEBUG "status": "success", 04:55:54 DEBUG "data": { 04:55:54 DEBUG "resultType": "vector", 04:55:54 DEBUG "result": [ 04:55:54 DEBUG { 04:55:54 DEBUG "metric": { 04:55:54 DEBUG "pod": "ds-cts-0" 04:55:54 DEBUG }, 04:55:54 DEBUG "value": [ 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG [...] 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG "pod": "ds-cts-2" 04:55:54 DEBUG }, 04:55:54 DEBUG "value": [ 04:55:54 DEBUG 1713498954, 04:55:54 DEBUG "0.1" 04:55:54 DEBUG ] 04:55:54 DEBUG } 04:55:54 DEBUG ] 04:55:54 DEBUG } 04:55:54 DEBUG } 04:55:54 INFO [http_cmd]: http status code OK 04:55:54 DEBUG --- status code --- 04:55:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:55:54 DEBUG --- http response --- 04:55:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:55:54 DEBUG ----- output ----- 04:55:54 DEBUG { 04:55:54 DEBUG "status": "success", 04:55:54 DEBUG "data": { 04:55:54 DEBUG "resultType": "vector", 04:55:54 DEBUG "result": [ 04:55:54 DEBUG { 04:55:54 DEBUG "metric": { 04:55:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:55:54 DEBUG }, 04:55:54 DEBUG "value": [ 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG [...] 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:55:54 DEBUG }, 04:55:54 DEBUG "value": [ 04:55:54 DEBUG 1713498954, 04:55:54 DEBUG "1998.5333333333333" 04:55:54 DEBUG ] 04:55:54 DEBUG } 04:55:54 DEBUG ] 04:55:54 DEBUG } 04:55:54 DEBUG } 04:55:54 INFO [http_cmd]: http status code OK 04:55:54 INFO [http_cmd]: http status code OK 04:55:54 DEBUG --- status code --- 04:55:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498954" 04:55:54 INFO [http_cmd]: http status code OK 04:55:54 DEBUG --- status code --- 04:55:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:55:54 DEBUG --- status code --- 04:55:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:55:54 DEBUG --- http response --- 04:55:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:55:54 DEBUG --- http response --- 04:55:54 DEBUG --- http response --- 04:55:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:55:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:55:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:55:54 DEBUG ----- output ----- 04:55:54 DEBUG ----- output ----- 04:55:54 DEBUG ----- output ----- 04:55:54 DEBUG { 04:55:54 DEBUG { 04:55:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498954" 04:55:54 DEBUG { 04:55:54 DEBUG "status": "success", 04:55:54 DEBUG "status": "success", 04:55:54 DEBUG "status": "success", 04:55:54 DEBUG "data": { 04:55:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498954" 04:55:54 DEBUG "data": { 04:55:54 DEBUG "data": { 04:55:54 DEBUG "resultType": "vector", 04:55:54 DEBUG "resultType": "vector", 04:55:54 DEBUG "resultType": "vector", 04:55:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498954" 04:55:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713498954" 04:55:54 DEBUG "result": [ 04:55:54 DEBUG "result": [ 04:55:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498954" 04:55:54 DEBUG "result": [ 04:55:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498954" 04:55:54 DEBUG { 04:55:54 DEBUG { 04:55:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713498954" 04:55:54 DEBUG { 04:55:54 DEBUG "metric": { 04:55:54 INFO [http_cmd]: http status code OK 04:55:54 INFO [http_cmd]: http status code OK 04:55:54 DEBUG "metric": { 04:55:54 DEBUG "metric": { 04:55:54 DEBUG "pod": "ds-idrepo-0" 04:55:54 DEBUG --- status code --- 04:55:54 DEBUG --- status code --- 04:55:54 DEBUG "pod": "ds-idrepo-0" 04:55:54 DEBUG }, 04:55:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713498954" 04:55:54 DEBUG }, 04:55:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:55:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:55:54 DEBUG "instance": "10.142.0.107:9100" 04:55:54 DEBUG }, 04:55:54 DEBUG "value": [ 04:55:54 DEBUG --- http response --- 04:55:54 DEBUG --- http response --- 04:55:54 INFO [http_cmd]: http status code OK 04:55:54 DEBUG "value": [ 04:55:54 DEBUG "value": [ 04:55:54 DEBUG 04:55:54 INFO [http_cmd]: http status code OK 04:55:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 04:55:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:55:54 DEBUG ----- output ----- 04:55:54 DEBUG 04:55:54 INFO [http_cmd]: http status code OK 04:55:54 DEBUG 04:55:54 INFO [http_cmd]: http status code OK 04:55:54 DEBUG 04:55:54 DEBUG --- status code --- 04:55:54 DEBUG ----- output ----- 04:55:54 DEBUG --- status code --- 04:55:54 DEBUG { 04:55:54 DEBUG "status": "success", 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG --- status code --- 04:55:54 DEBUG 04:55:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:55:54 DEBUG { 04:55:54 INFO [http_cmd]: http status code OK 04:55:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:55:54 DEBUG --- status code --- 04:55:54 DEBUG "data": { 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:55:54 DEBUG [...] 04:55:54 DEBUG --- http response --- 04:55:54 DEBUG "status": "success", 04:55:54 DEBUG --- status code --- 04:55:54 DEBUG --- http response --- 04:55:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:55:54 DEBUG "resultType": "vector", 04:55:54 DEBUG [...] 04:55:54 DEBUG [...] 04:55:54 DEBUG --- http response --- 04:55:54 DEBUG 04:55:54 DEBUG "data": { 04:55:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:55:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:55:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:55:54 DEBUG --- http response --- 04:55:54 DEBUG "result": [ 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:55:54 DEBUG 04:55:54 DEBUG "resultType": "vector", 04:55:54 DEBUG --- http response --- 04:55:54 DEBUG ----- output ----- 04:55:54 DEBUG ----- output ----- 04:55:54 DEBUG { 04:55:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG ----- output ----- 04:55:54 INFO [http_cmd]: http status code OK 04:55:54 DEBUG 04:55:54 DEBUG "result": [ 04:55:54 DEBUG { 04:55:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:55:54 DEBUG { 04:55:54 DEBUG "metric": { 04:55:54 DEBUG ----- output ----- 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG { 04:55:54 DEBUG --- status code --- 04:55:54 DEBUG "instance": "10.142.0.43:9100" 04:55:54 DEBUG { 04:55:54 DEBUG "status": "success", 04:55:54 DEBUG ----- output ----- 04:55:54 INFO [http_cmd]: http status code OK 04:55:54 DEBUG "status": "success", 04:55:54 DEBUG "pod": "ds-cts-0" 04:55:54 DEBUG { 04:55:54 DEBUG "pod": "ds-cts-2" 04:55:54 DEBUG "pod": "ds-idrepo-2" 04:55:54 DEBUG "status": "success", 04:55:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:55:54 DEBUG }, 04:55:54 DEBUG "metric": { 04:55:54 DEBUG "data": { 04:55:54 DEBUG { 04:55:54 DEBUG --- status code --- 04:55:54 DEBUG "data": { 04:55:54 DEBUG }, 04:55:54 DEBUG "status": "success", 04:55:54 DEBUG }, 04:55:54 DEBUG }, 04:55:54 DEBUG "data": { 04:55:54 DEBUG --- http response --- 04:55:54 DEBUG "value": [ 04:55:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:55:54 DEBUG "resultType": "vector", 04:55:54 DEBUG "status": "success", 04:55:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:55:54 DEBUG "resultType": "vector", 04:55:54 DEBUG "value": [ 04:55:54 DEBUG "data": { 04:55:54 DEBUG "value": [ 04:55:54 DEBUG "value": [ 04:55:54 DEBUG "resultType": "vector", 04:55:54 DEBUG 1713498954, 04:55:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 04:55:54 DEBUG }, 04:55:54 DEBUG "result": [ 04:55:54 DEBUG "data": { 04:55:54 DEBUG --- http response --- 04:55:54 DEBUG "result": [ 04:55:54 DEBUG 04:55:54 DEBUG "resultType": "vector", 04:55:54 DEBUG 1713498954, 04:55:54 DEBUG 1713498954, 04:55:54 DEBUG "result": [ 04:55:54 DEBUG "47766393.13333333" 04:55:54 DEBUG ----- output ----- 04:55:54 DEBUG "value": [ 04:55:54 DEBUG { 04:55:54 DEBUG "resultType": "vector", 04:55:54 DEBUG { 04:55:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:55:54 DEBUG 04:55:54 DEBUG "result": [ 04:55:54 DEBUG "0" 04:55:54 DEBUG "0" 04:55:54 DEBUG { 04:55:54 DEBUG ] 04:55:54 DEBUG { 04:55:54 DEBUG 04:55:54 DEBUG "metric": { 04:55:54 DEBUG "result": [ 04:55:54 DEBUG "metric": { 04:55:54 DEBUG ----- output ----- 04:55:54 DEBUG 04:55:54 DEBUG { 04:55:54 DEBUG ] 04:55:54 DEBUG ] 04:55:54 DEBUG "metric": { 04:55:54 DEBUG } 04:55:54 DEBUG "status": "success", 04:55:54 DEBUG 04:55:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:55:54 DEBUG { 04:55:54 DEBUG "instance": "10.142.0.107:9100" 04:55:54 DEBUG { 04:55:54 DEBUG [...] 04:55:54 DEBUG "metric": { 04:55:54 DEBUG } 04:55:54 DEBUG } 04:55:54 DEBUG "pod": "ds-idrepo-0" 04:55:54 DEBUG ] 04:55:54 DEBUG "data": { 04:55:54 DEBUG 04:55:54 DEBUG }, 04:55:54 DEBUG "metric": { 04:55:54 DEBUG }, 04:55:54 DEBUG "status": "success", 04:55:54 DEBUG 04:55:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:55:54 DEBUG ] 04:55:54 DEBUG ] 04:55:54 DEBUG }, 04:55:54 DEBUG } 04:55:54 DEBUG "resultType": "vector", 04:55:54 DEBUG [...] 04:55:54 DEBUG "value": [ 04:55:54 DEBUG "pod": "ds-idrepo-0" 04:55:54 DEBUG "value": [ 04:55:54 DEBUG "data": { 04:55:54 DEBUG 04:55:54 DEBUG }, 04:55:54 DEBUG } 04:55:54 DEBUG } 04:55:54 DEBUG "value": [ 04:55:54 DEBUG } 04:55:54 DEBUG "result": [ 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG }, 04:55:54 DEBUG 04:55:54 DEBUG "resultType": "vector", 04:55:54 DEBUG 04:55:54 DEBUG "value": [ 04:55:54 DEBUG } 04:55:54 DEBUG } 04:55:54 DEBUG 04:55:54 DEBUG { 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG "value": [ 04:55:54 DEBUG 04:55:54 DEBUG "result": [ 04:55:54 DEBUG "pod": "ds-cts-2" 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG "metric": { 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG { 04:55:54 DEBUG }, 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:55:54 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 04:55:54 DEBUG [...] 04:55:54 DEBUG 04:55:54 DEBUG [...] 04:55:54 DEBUG "metric": { 04:55:54 DEBUG "value": [ 04:55:54 DEBUG 04:55:54 DEBUG [...] 04:55:54 DEBUG }, 04:55:54 DEBUG }, 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 04:55:54 DEBUG 1713498954, 04:55:54 DEBUG [...] 04:55:54 DEBUG 04:55:54 DEBUG "value": [ 04:55:54 DEBUG "value": [ 04:55:54 DEBUG 04:55:54 DEBUG [...] 04:55:54 DEBUG 04:55:54 DEBUG }, 04:55:54 DEBUG "0" 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG 1713498954, 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG "value": [ 04:55:54 DEBUG ] 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG "13.00803318743122" 04:55:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:55:54 DEBUG 04:55:54 DEBUG "instance": "10.142.0.43:9100" 04:55:54 DEBUG 04:55:54 DEBUG } 04:55:54 DEBUG 04:55:54 DEBUG "pod": "ds-idrepo-2" 04:55:54 DEBUG 04:55:54 DEBUG ] 04:55:54 DEBUG }, 04:55:54 DEBUG 04:55:54 DEBUG }, 04:55:54 DEBUG 04:55:54 DEBUG ] 04:55:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:55:54 DEBUG }, 04:55:54 DEBUG [...] 04:55:54 DEBUG } 04:55:54 DEBUG "value": [ 04:55:54 DEBUG "pod": "ds-idrepo-2" 04:55:54 DEBUG "value": [ 04:55:54 DEBUG 04:55:54 DEBUG } 04:55:54 DEBUG }, 04:55:54 DEBUG "value": [ 04:55:54 DEBUG 04:55:54 DEBUG ] 04:55:54 DEBUG 1713498954, 04:55:54 DEBUG }, 04:55:54 DEBUG 1713498954, 04:55:54 DEBUG [...] 04:55:54 DEBUG } 04:55:54 DEBUG "value": [ 04:55:54 DEBUG 1713498954, 04:55:54 DEBUG 04:55:54 DEBUG } 04:55:54 DEBUG "55.171" 04:55:54 DEBUG "value": [ 04:55:54 DEBUG "0.033999999999997976" 04:55:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498954" 04:55:54 DEBUG 04:55:54 DEBUG 1713498954, 04:55:54 DEBUG "0" 04:55:54 DEBUG 04:55:54 DEBUG } 04:55:54 DEBUG ] 04:55:54 DEBUG 1713498954, 04:55:54 DEBUG ] 04:55:54 DEBUG 04:55:54 DEBUG "1998.3333333333333" 04:55:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498954" 04:55:54 DEBUG ] 04:55:54 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 04:55:54 DEBUG } 04:55:54 DEBUG "0" 04:55:54 DEBUG } 04:55:54 DEBUG 04:55:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 04:55:54 DEBUG ] 04:55:54 DEBUG }, 04:55:54 DEBUG ] 04:55:54 DEBUG ] 04:55:54 DEBUG ] 04:55:54 DEBUG } 04:55:54 DEBUG }, 04:55:54 DEBUG } 04:55:54 DEBUG "value": [ 04:55:54 DEBUG } 04:55:54 DEBUG } 04:55:54 DEBUG } 04:55:54 DEBUG ] 04:55:54 DEBUG "value": [ 04:55:54 DEBUG ] 04:55:54 DEBUG 1713498954, 04:55:54 DEBUG } 04:55:54 DEBUG ] 04:55:54 DEBUG } 04:55:54 DEBUG } 04:55:54 DEBUG 1713498954, 04:55:54 DEBUG } 04:55:54 DEBUG "0" 04:55:54 DEBUG } 04:55:54 DEBUG } 04:55:54 DEBUG "6.542055370397877" 04:55:54 DEBUG } 04:55:54 DEBUG ] 04:55:54 DEBUG } 04:55:54 DEBUG ] 04:55:54 DEBUG } 04:55:54 DEBUG } 04:55:54 DEBUG ] 04:55:54 DEBUG ] 04:55:54 DEBUG } 04:55:54 DEBUG } 04:55:54 DEBUG } 04:55:54 DEBUG } 04:55:54 INFO [http_cmd]: http status code OK 04:55:54 INFO [http_cmd]: http status code OK 04:55:54 DEBUG --- status code --- 04:55:54 DEBUG --- status code --- 04:55:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:55:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:55:54 DEBUG --- http response --- 04:55:54 DEBUG --- http response --- 04:55:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:55:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:55:54 DEBUG ----- output ----- 04:55:54 DEBUG ----- output ----- 04:55:54 DEBUG { 04:55:54 DEBUG { 04:55:54 DEBUG "status": "success", 04:55:54 DEBUG "status": "success", 04:55:54 DEBUG "data": { 04:55:54 DEBUG "data": { 04:55:54 DEBUG "resultType": "vector", 04:55:54 DEBUG "resultType": "vector", 04:55:54 DEBUG "result": [ 04:55:54 DEBUG "result": [ 04:55:54 DEBUG { 04:55:54 DEBUG { 04:55:54 DEBUG "metric": { 04:55:54 DEBUG "metric": { 04:55:54 DEBUG "instance": "10.142.0.107:9100" 04:55:54 DEBUG "instance": "10.142.0.107:9100" 04:55:54 DEBUG }, 04:55:54 DEBUG }, 04:55:54 DEBUG "value": [ 04:55:54 DEBUG "value": [ 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG [...] 04:55:54 DEBUG [...] 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG "instance": "10.142.0.43:9100" 04:55:54 DEBUG "instance": "10.142.0.43:9100" 04:55:54 DEBUG }, 04:55:54 DEBUG }, 04:55:54 DEBUG "value": [ 04:55:54 DEBUG "value": [ 04:55:54 DEBUG 1713498954, 04:55:54 DEBUG 1713498954, 04:55:54 DEBUG "411352945664" 04:55:54 DEBUG "0.047866666666665954" 04:55:54 DEBUG ] 04:55:54 DEBUG ] 04:55:54 DEBUG } 04:55:54 DEBUG } 04:55:54 DEBUG ] 04:55:54 DEBUG ] 04:55:54 DEBUG } 04:55:54 DEBUG } 04:55:54 DEBUG } 04:55:54 DEBUG } 04:55:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713498954" 04:55:54 INFO [http_cmd]: http status code OK 04:55:54 DEBUG --- status code --- 04:55:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:55:54 DEBUG --- http response --- 04:55:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:55:54 DEBUG ----- output ----- 04:55:54 DEBUG { 04:55:54 DEBUG "status": "success", 04:55:54 DEBUG "data": { 04:55:54 DEBUG "resultType": "vector", 04:55:54 DEBUG "result": [ 04:55:54 DEBUG { 04:55:54 DEBUG "metric": { 04:55:54 DEBUG "instance": "10.142.0.107:9100" 04:55:54 DEBUG }, 04:55:54 DEBUG "value": [ 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG [...] 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG 04:55:54 DEBUG "instance": "10.142.0.43:9100" 04:55:54 DEBUG }, 04:55:54 DEBUG "value": [ 04:55:54 DEBUG 1713498954, 04:55:54 DEBUG "47770139.33333333" 04:55:54 DEBUG ] 04:55:54 DEBUG } 04:55:54 DEBUG ] 04:55:54 DEBUG } 04:55:54 DEBUG } 04:55:55 INFO 04:55:55 INFO [loop_until]: kubectl --namespace=xlou top pods 04:55:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:55:55 INFO [loop_until]: OK (rc = 0) 04:55:55 DEBUG --- stdout --- 04:55:55 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 9389m 8962Mi am-6fd554f8f7-hvvkw 9432m 8873Mi am-6fd554f8f7-pmj4b 10164m 8971Mi ds-cts-0 7016m 12189Mi ds-cts-1 6417m 12106Mi ds-cts-2 6716m 12149Mi ds-idrepo-0 4738m 3253Mi ds-idrepo-1 4669m 3206Mi ds-idrepo-2 4622m 3066Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 4m 1216Mi lodemon-6b89d774d5-rf7sk 13m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 6508m 1960Mi 04:55:55 DEBUG --- stderr --- 04:55:55 DEBUG 04:55:55 INFO 04:55:55 INFO [loop_until]: kubectl --namespace=xlou top node 04:55:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:55:55 INFO [loop_until]: OK (rc = 0) 04:55:55 DEBUG --- stdout --- 04:55:55 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 78m 0% 6259Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 4632m 15% 6700Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9486m 31% 11597Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 98m 0% 3258Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 6935m 23% 14789Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 159m 0% 5189Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 9747m 32% 11743Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 10317m 34% 2738Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 10920m 36% 11179Mi 10% gke-xlou-cdm-ds-0f601751-j488 4865m 16% 6165Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 6423m 21% 16167Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 5117m 17% 6887Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 6795m 22% 14163Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 6688m 42% 5722Mi 9% 04:55:55 DEBUG --- stderr --- 04:55:55 DEBUG 04:56:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499014" 04:56:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499014" 04:56:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499014" 04:56:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713499014" 04:56:54 INFO [http_cmd]: http status code OK 04:56:54 INFO [http_cmd]: http status code OK 04:56:54 DEBUG --- status code --- 04:56:54 DEBUG --- status code --- 04:56:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:56:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:56:54 DEBUG --- http response --- 04:56:54 DEBUG --- http response --- 04:56:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 04:56:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 04:56:54 DEBUG ----- output ----- 04:56:54 DEBUG { 04:56:54 DEBUG "status": "success", 04:56:54 DEBUG "data": { 04:56:54 DEBUG "resultType": "vector", 04:56:54 DEBUG "result": [ 04:56:54 DEBUG { 04:56:54 DEBUG "metric": { 04:56:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:56:54 DEBUG }, 04:56:54 DEBUG "value": [ 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG [...] 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG "pod": "ds-cts-0" 04:56:54 DEBUG }, 04:56:54 DEBUG "value": [ 04:56:54 DEBUG 1713499014, 04:56:54 DEBUG "25944832.897444475" 04:56:54 DEBUG ] 04:56:54 DEBUG } 04:56:54 DEBUG ] 04:56:54 DEBUG } 04:56:54 DEBUG } 04:56:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499014" 04:56:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499014" 04:56:54 ERROR [http_cmd]: ERROR 04:56:54 INFO [http_cmd]: http status code OK 04:56:54 ERROR --- status code --- 04:56:54 DEBUG --- status code --- 04:56:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 04:56:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:56:54 ERROR --- http response --- 04:56:54 DEBUG --- http response --- 04:56:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 04:56:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 04:56:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:56:54 DEBUG ----- output ----- 04:56:54 DEBUG { 04:56:54 DEBUG "status": "success", 04:56:54 DEBUG "data": { 04:56:54 DEBUG "resultType": "vector", 04:56:54 DEBUG "result": [ 04:56:54 DEBUG { 04:56:54 DEBUG "metric": { 04:56:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:56:54 DEBUG }, 04:56:54 DEBUG "value": [ 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG [...] 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:56:54 DEBUG }, 04:56:54 DEBUG "value": [ 04:56:54 DEBUG 1713499014, 04:56:54 DEBUG "0" 04:56:54 DEBUG ] 04:56:54 DEBUG } 04:56:54 DEBUG ] 04:56:54 DEBUG } 04:56:54 DEBUG } 04:56:54 INFO [http_cmd]: http status code OK 04:56:54 DEBUG --- status code --- 04:56:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:56:54 DEBUG --- http response --- 04:56:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:56:54 DEBUG ----- output ----- 04:56:54 DEBUG { 04:56:54 DEBUG "status": "success", 04:56:54 INFO [http_cmd]: http status code OK 04:56:54 DEBUG "data": { 04:56:54 DEBUG --- status code --- 04:56:54 DEBUG "resultType": "vector", 04:56:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:56:54 DEBUG "result": [ 04:56:54 DEBUG --- http response --- 04:56:54 DEBUG { 04:56:54 DEBUG "metric": { 04:56:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:56:54 DEBUG }, 04:56:54 DEBUG "value": [ 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG [...] 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:56:54 DEBUG }, 04:56:54 DEBUG "value": [ 04:56:54 DEBUG 1713499014, 04:56:54 DEBUG "2000.7" 04:56:54 DEBUG ] 04:56:54 DEBUG } 04:56:54 DEBUG ] 04:56:54 DEBUG } 04:56:54 DEBUG } 04:56:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 04:56:54 DEBUG ----- output ----- 04:56:54 DEBUG { 04:56:54 DEBUG "status": "success", 04:56:54 DEBUG "data": { 04:56:54 DEBUG "resultType": "vector", 04:56:54 DEBUG "result": [ 04:56:54 DEBUG { 04:56:54 DEBUG "metric": { 04:56:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:56:54 DEBUG }, 04:56:54 DEBUG "value": [ 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG [...] 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG "pod": "ds-cts-0" 04:56:54 DEBUG }, 04:56:54 DEBUG "value": [ 04:56:54 DEBUG 1713499014, 04:56:54 DEBUG "90288.29234332999" 04:56:54 DEBUG ] 04:56:54 DEBUG } 04:56:54 DEBUG ] 04:56:54 DEBUG } 04:56:54 DEBUG } 04:56:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499014" 04:56:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499014" 04:56:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713499014" 04:56:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499014" 04:56:54 INFO [http_cmd]: http status code OK 04:56:54 DEBUG --- status code --- 04:56:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:56:54 DEBUG --- http response --- 04:56:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:56:54 DEBUG ----- output ----- 04:56:54 DEBUG { 04:56:54 DEBUG "status": "success", 04:56:54 DEBUG "data": { 04:56:54 DEBUG "resultType": "vector", 04:56:54 DEBUG "result": [ 04:56:54 DEBUG { 04:56:54 DEBUG "metric": { 04:56:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:56:54 DEBUG }, 04:56:54 DEBUG "value": [ 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG [...] 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:56:54 DEBUG }, 04:56:54 DEBUG "value": [ 04:56:54 DEBUG 1713499014, 04:56:54 DEBUG "1420766.5" 04:56:54 DEBUG ] 04:56:54 DEBUG } 04:56:54 DEBUG ] 04:56:54 DEBUG } 04:56:54 DEBUG } 04:56:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499014" 04:56:54 INFO [http_cmd]: http status code OK 04:56:54 DEBUG --- status code --- 04:56:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:56:54 DEBUG --- http response --- 04:56:54 INFO [http_cmd]: http status code OK 04:56:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:56:54 INFO [http_cmd]: http status code OK 04:56:54 DEBUG --- status code --- 04:56:54 DEBUG ----- output ----- 04:56:54 DEBUG --- status code --- 04:56:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:56:54 DEBUG { 04:56:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:56:54 DEBUG --- http response --- 04:56:54 DEBUG "status": "success", 04:56:54 DEBUG --- http response --- 04:56:54 DEBUG "data": { 04:56:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:56:54 DEBUG "resultType": "vector", 04:56:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:56:54 DEBUG ----- output ----- 04:56:54 DEBUG "result": [ 04:56:54 DEBUG ----- output ----- 04:56:54 DEBUG { 04:56:54 DEBUG { 04:56:54 DEBUG { 04:56:54 DEBUG "status": "success", 04:56:54 DEBUG "metric": { 04:56:54 DEBUG "status": "success", 04:56:54 DEBUG "data": { 04:56:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 04:56:54 DEBUG "data": { 04:56:54 DEBUG "resultType": "vector", 04:56:54 DEBUG }, 04:56:54 DEBUG "resultType": "vector", 04:56:54 DEBUG "result": [ 04:56:54 DEBUG "value": [ 04:56:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499014" 04:56:54 DEBUG "result": [ 04:56:54 DEBUG { 04:56:54 DEBUG 04:56:54 DEBUG { 04:56:54 DEBUG "metric": { 04:56:54 DEBUG 04:56:54 DEBUG "metric": { 04:56:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:56:54 DEBUG 04:56:54 DEBUG "pod": "ds-cts-0" 04:56:54 DEBUG }, 04:56:54 DEBUG [...] 04:56:54 DEBUG }, 04:56:54 DEBUG "value": [ 04:56:54 DEBUG 04:56:54 DEBUG "value": [ 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 04:56:54 DEBUG 04:56:54 DEBUG [...] 04:56:54 DEBUG }, 04:56:54 DEBUG [...] 04:56:54 DEBUG 04:56:54 DEBUG "value": [ 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG 1713499014, 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG "2154852352" 04:56:54 DEBUG 04:56:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:56:54 DEBUG ] 04:56:54 DEBUG "pod": "ds-cts-2" 04:56:54 DEBUG }, 04:56:54 DEBUG } 04:56:54 DEBUG }, 04:56:54 DEBUG "value": [ 04:56:54 DEBUG ] 04:56:54 DEBUG "value": [ 04:56:54 DEBUG 1713499014, 04:56:54 DEBUG } 04:56:54 DEBUG 1713499014, 04:56:54 DEBUG "2000.7333333333333" 04:56:54 DEBUG } 04:56:54 DEBUG "0.000017104444444444444" 04:56:54 DEBUG ] 04:56:54 DEBUG ] 04:56:54 DEBUG } 04:56:54 DEBUG } 04:56:54 DEBUG ] 04:56:54 DEBUG ] 04:56:54 DEBUG } 04:56:54 DEBUG } 04:56:54 DEBUG } 04:56:54 DEBUG } 04:56:54 INFO [http_cmd]: http status code OK 04:56:54 DEBUG --- status code --- 04:56:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:56:54 DEBUG --- http response --- 04:56:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:56:54 DEBUG ----- output ----- 04:56:54 DEBUG { 04:56:54 DEBUG "status": "success", 04:56:54 DEBUG "data": { 04:56:54 DEBUG "resultType": "vector", 04:56:54 DEBUG "result": [ 04:56:54 DEBUG { 04:56:54 DEBUG "metric": { 04:56:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:56:54 DEBUG }, 04:56:54 DEBUG "value": [ 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG [...] 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:56:54 DEBUG }, 04:56:54 DEBUG "value": [ 04:56:54 DEBUG 1713499014, 04:56:54 DEBUG "0" 04:56:54 DEBUG ] 04:56:54 DEBUG } 04:56:54 DEBUG ] 04:56:54 DEBUG } 04:56:54 DEBUG } 04:56:54 INFO [http_cmd]: http status code OK 04:56:54 DEBUG --- status code --- 04:56:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:56:54 DEBUG --- http response --- 04:56:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:56:54 DEBUG ----- output ----- 04:56:54 DEBUG { 04:56:54 DEBUG "status": "success", 04:56:54 DEBUG "data": { 04:56:54 DEBUG "resultType": "vector", 04:56:54 DEBUG "result": [ 04:56:54 DEBUG { 04:56:54 DEBUG "metric": { 04:56:54 DEBUG "pod": "ds-idrepo-0" 04:56:54 DEBUG }, 04:56:54 DEBUG "value": [ 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG [...] 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG "pod": "ds-idrepo-2" 04:56:54 DEBUG }, 04:56:54 DEBUG "value": [ 04:56:54 DEBUG 1713499014, 04:56:54 DEBUG "0" 04:56:54 DEBUG ] 04:56:54 DEBUG } 04:56:54 DEBUG ] 04:56:54 DEBUG } 04:56:54 DEBUG } 04:56:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499014" 04:56:54 INFO [http_cmd]: http status code OK 04:56:54 DEBUG --- status code --- 04:56:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:56:54 DEBUG --- http response --- 04:56:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:56:54 DEBUG ----- output ----- 04:56:54 DEBUG { 04:56:54 DEBUG "status": "success", 04:56:54 DEBUG "data": { 04:56:54 DEBUG "resultType": "vector", 04:56:54 DEBUG "result": [ 04:56:54 DEBUG { 04:56:54 DEBUG "metric": { 04:56:54 DEBUG "pod": "ds-cts-0" 04:56:54 DEBUG }, 04:56:54 DEBUG "value": [ 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG [...] 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG "pod": "ds-cts-2" 04:56:54 DEBUG }, 04:56:54 DEBUG "value": [ 04:56:54 DEBUG 1713499014, 04:56:54 DEBUG "0.1" 04:56:54 DEBUG ] 04:56:54 DEBUG } 04:56:54 DEBUG ] 04:56:54 DEBUG } 04:56:54 DEBUG } 04:56:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713499014" 04:56:54 INFO [http_cmd]: http status code OK 04:56:54 DEBUG --- status code --- 04:56:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:56:54 DEBUG --- http response --- 04:56:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:56:54 DEBUG ----- output ----- 04:56:54 DEBUG { 04:56:54 DEBUG "status": "success", 04:56:54 DEBUG "data": { 04:56:54 DEBUG "resultType": "vector", 04:56:54 DEBUG "result": [ 04:56:54 DEBUG { 04:56:54 DEBUG "metric": { 04:56:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 04:56:54 DEBUG }, 04:56:54 DEBUG "value": [ 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG [...] 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 04:56:54 DEBUG }, 04:56:54 DEBUG "value": [ 04:56:54 DEBUG 1713499014, 04:56:54 DEBUG "6.543897249059048" 04:56:54 DEBUG ] 04:56:54 DEBUG } 04:56:54 DEBUG ] 04:56:54 DEBUG } 04:56:54 DEBUG } 04:56:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499014" 04:56:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713499014" 04:56:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499014" 04:56:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499014" 04:56:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499014" 04:56:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499014" 04:56:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713499014" 04:56:54 INFO [http_cmd]: http status code OK 04:56:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499014" 04:56:54 DEBUG --- status code --- 04:56:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499014" 04:56:54 INFO [http_cmd]: http status code OK 04:56:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499014" 04:56:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499014" 04:56:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499014" 04:56:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:56:54 INFO [http_cmd]: http status code OK 04:56:54 INFO [http_cmd]: http status code OK 04:56:54 INFO [http_cmd]: http status code OK 04:56:54 DEBUG --- status code --- 04:56:54 INFO [http_cmd]: http status code OK 04:56:54 DEBUG --- http response --- 04:56:54 DEBUG --- status code --- 04:56:54 INFO [http_cmd]: http status code OK 04:56:54 DEBUG --- status code --- 04:56:54 DEBUG --- status code --- 04:56:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:56:54 DEBUG --- status code --- 04:56:54 INFO [http_cmd]: http status code OK 04:56:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:56:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:56:54 DEBUG --- status code --- 04:56:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:56:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:56:54 DEBUG --- http response --- 04:56:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:56:54 DEBUG --- status code --- 04:56:54 DEBUG ----- output ----- 04:56:54 DEBUG { 04:56:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:56:54 DEBUG --- http response --- 04:56:54 DEBUG --- http response --- 04:56:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:56:54 DEBUG --- http response --- 04:56:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:56:54 DEBUG "status": "success", 04:56:54 DEBUG --- http response --- 04:56:54 DEBUG --- http response --- 04:56:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:56:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:56:54 DEBUG ----- output ----- 04:56:54 DEBUG --- http response --- 04:56:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 04:56:54 DEBUG "data": { 04:56:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:56:54 DEBUG ----- output ----- 04:56:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:56:54 INFO [http_cmd]: http status code OK 04:56:54 DEBUG ----- output ----- 04:56:54 INFO [http_cmd]: http status code OK 04:56:54 DEBUG { 04:56:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:56:54 DEBUG ----- output ----- 04:56:54 DEBUG "resultType": "vector", 04:56:54 DEBUG ----- output ----- 04:56:54 DEBUG { 04:56:54 DEBUG ----- output ----- 04:56:54 DEBUG --- status code --- 04:56:54 DEBUG { 04:56:54 DEBUG --- status code --- 04:56:54 DEBUG "status": "success", 04:56:54 INFO [http_cmd]: http status code OK 04:56:54 DEBUG ----- output ----- 04:56:54 DEBUG { 04:56:54 DEBUG "result": [ 04:56:54 DEBUG { 04:56:54 DEBUG "status": "success", 04:56:54 DEBUG { 04:56:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:56:54 DEBUG "status": "success", 04:56:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:56:54 DEBUG "data": { 04:56:54 DEBUG --- status code --- 04:56:54 DEBUG { 04:56:54 DEBUG "status": "success", 04:56:54 DEBUG { 04:56:54 INFO [http_cmd]: http status code OK 04:56:54 DEBUG "status": "success", 04:56:54 DEBUG "data": { 04:56:54 DEBUG "status": "success", 04:56:54 DEBUG --- http response --- 04:56:54 DEBUG "data": { 04:56:54 DEBUG --- http response --- 04:56:54 DEBUG "resultType": "vector", 04:56:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499014" 04:56:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:56:54 DEBUG "status": "success", 04:56:54 DEBUG "data": { 04:56:54 DEBUG "metric": { 04:56:54 DEBUG --- status code --- 04:56:54 DEBUG "data": { 04:56:54 DEBUG "resultType": "vector", 04:56:54 DEBUG "data": { 04:56:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:56:54 DEBUG "resultType": "vector", 04:56:54 DEBUG "result": [ 04:56:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 04:56:54 DEBUG --- http response --- 04:56:54 DEBUG "data": { 04:56:54 DEBUG "resultType": "vector", 04:56:54 DEBUG "pod": "ds-idrepo-0" 04:56:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:56:54 DEBUG "resultType": "vector", 04:56:54 DEBUG "result": [ 04:56:54 DEBUG "resultType": "vector", 04:56:54 DEBUG ----- output ----- 04:56:54 DEBUG "result": [ 04:56:54 DEBUG { 04:56:54 DEBUG ----- output ----- 04:56:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:56:54 DEBUG "resultType": "vector", 04:56:54 DEBUG "result": [ 04:56:54 DEBUG }, 04:56:54 DEBUG --- http response --- 04:56:54 DEBUG "result": [ 04:56:54 DEBUG { 04:56:54 DEBUG "result": [ 04:56:54 DEBUG { 04:56:54 DEBUG { 04:56:54 DEBUG "metric": { 04:56:54 DEBUG { 04:56:54 DEBUG ----- output ----- 04:56:54 DEBUG "result": [ 04:56:54 DEBUG { 04:56:54 DEBUG "value": [ 04:56:54 DEBUG { 04:56:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:56:54 DEBUG "metric": { 04:56:54 DEBUG { 04:56:54 DEBUG "status": "success", 04:56:54 DEBUG "metric": { 04:56:54 DEBUG "instance": "10.142.0.107:9100" 04:56:54 DEBUG "status": "success", 04:56:54 DEBUG { 04:56:54 DEBUG { 04:56:54 DEBUG "metric": { 04:56:54 DEBUG 04:56:54 DEBUG "metric": { 04:56:54 DEBUG ----- output ----- 04:56:54 DEBUG "instance": "10.142.0.107:9100" 04:56:54 DEBUG "metric": { 04:56:54 DEBUG "data": { 04:56:54 DEBUG "instance": "10.142.0.107:9100" 04:56:54 DEBUG }, 04:56:54 DEBUG "data": { 04:56:54 DEBUG "status": "success", 04:56:54 DEBUG "metric": { 04:56:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:56:54 DEBUG 04:56:54 DEBUG "pod": "ds-cts-0" 04:56:54 DEBUG { 04:56:54 DEBUG }, 04:56:54 DEBUG "pod": "ds-idrepo-0" 04:56:54 DEBUG "resultType": "vector", 04:56:54 DEBUG }, 04:56:54 DEBUG "value": [ 04:56:54 DEBUG "resultType": "vector", 04:56:54 DEBUG "data": { 04:56:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:56:54 DEBUG }, 04:56:54 DEBUG 04:56:54 DEBUG }, 04:56:54 DEBUG "status": "success", 04:56:54 DEBUG "value": [ 04:56:54 DEBUG }, 04:56:54 DEBUG "result": [ 04:56:54 DEBUG "value": [ 04:56:54 DEBUG 04:56:54 DEBUG "result": [ 04:56:54 DEBUG "resultType": "vector", 04:56:54 DEBUG }, 04:56:54 DEBUG "value": [ 04:56:54 DEBUG [...] 04:56:54 DEBUG "value": [ 04:56:54 DEBUG "data": { 04:56:54 DEBUG 04:56:54 DEBUG "value": [ 04:56:54 DEBUG { 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG { 04:56:54 DEBUG "result": [ 04:56:54 DEBUG "value": [ 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG "resultType": "vector", 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG "metric": { 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG "metric": { 04:56:54 DEBUG { 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG "result": [ 04:56:54 DEBUG { 04:56:54 DEBUG 04:56:54 DEBUG "pod": "ds-idrepo-0" 04:56:54 DEBUG 04:56:54 DEBUG [...] 04:56:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:56:54 DEBUG "metric": { 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG "metric": { 04:56:54 DEBUG 04:56:54 DEBUG }, 04:56:54 DEBUG [...] 04:56:54 DEBUG 04:56:54 DEBUG }, 04:56:54 DEBUG "instance": "10.142.0.107:9100" 04:56:54 DEBUG 04:56:54 DEBUG [...] 04:56:54 DEBUG "pod": "ds-cts-2" 04:56:54 DEBUG [...] 04:56:54 DEBUG [...] 04:56:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:56:54 DEBUG [...] 04:56:54 DEBUG "value": [ 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG "value": [ 04:56:54 DEBUG }, 04:56:54 DEBUG [...] 04:56:54 DEBUG 04:56:54 DEBUG }, 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG }, 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG "value": [ 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG "value": [ 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG "value": [ 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG "instance": "10.142.0.43:9100" 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG 1713499014, 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG [...] 04:56:54 DEBUG }, 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG "pod": "ds-cts-0" 04:56:54 DEBUG "0" 04:56:54 DEBUG "pod": "ds-cts-2" 04:56:54 DEBUG "instance": "10.142.0.43:9100" 04:56:54 DEBUG 04:56:54 DEBUG "pod": "ds-idrepo-2" 04:56:54 DEBUG "instance": "10.142.0.43:9100" 04:56:54 DEBUG 04:56:54 DEBUG "value": [ 04:56:54 DEBUG [...] 04:56:54 DEBUG 04:56:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:56:54 DEBUG }, 04:56:54 DEBUG "value": [ 04:56:54 DEBUG 1713499014, 04:56:54 DEBUG }, 04:56:54 DEBUG 04:56:54 DEBUG }, 04:56:54 DEBUG }, 04:56:54 DEBUG 04:56:54 INFO [http_cmd]: http status code OK 04:56:54 DEBUG 1713499014, 04:56:54 DEBUG 04:56:54 DEBUG [...] 04:56:54 DEBUG }, 04:56:54 DEBUG }, 04:56:54 DEBUG ] 04:56:54 DEBUG "55.171" 04:56:54 DEBUG "value": [ 04:56:54 DEBUG [...] 04:56:54 DEBUG "value": [ 04:56:54 DEBUG "value": [ 04:56:54 DEBUG 04:56:54 DEBUG --- status code --- 04:56:54 DEBUG "0.051399999999998155" 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG "value": [ 04:56:54 DEBUG "value": [ 04:56:54 DEBUG } 04:56:54 DEBUG ] 04:56:54 DEBUG 1713499014, 04:56:54 DEBUG 04:56:54 DEBUG 1713499014, 04:56:54 DEBUG 1713499014, 04:56:54 DEBUG "pod": "ds-idrepo-2" 04:56:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:56:54 DEBUG ] 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG 1713499014, 04:56:54 DEBUG 1713499014, 04:56:54 DEBUG ] 04:56:54 DEBUG } 04:56:54 DEBUG "413634440192" 04:56:54 DEBUG 04:56:54 DEBUG "0" 04:56:54 DEBUG "47681031.8" 04:56:54 DEBUG }, 04:56:54 DEBUG --- http response --- 04:56:54 DEBUG } 04:56:54 DEBUG "pod": "ds-cts-0" 04:56:54 DEBUG 04:56:54 DEBUG "0" 04:56:54 DEBUG ] 04:56:54 DEBUG } 04:56:54 DEBUG ] 04:56:54 DEBUG } 04:56:54 DEBUG 04:56:54 DEBUG ] 04:56:54 DEBUG ] 04:56:54 DEBUG "value": [ 04:56:54 DEBUG ] 04:56:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:56:54 DEBUG }, 04:56:54 DEBUG "instance": "10.142.0.43:9100" 04:56:54 DEBUG ] 04:56:54 DEBUG ] 04:56:54 DEBUG } 04:56:54 DEBUG "5.062683971872868" 04:56:54 DEBUG } 04:56:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:56:54 DEBUG } 04:56:54 DEBUG } 04:56:54 DEBUG 1713499014, 04:56:54 DEBUG } 04:56:54 DEBUG ----- output ----- 04:56:54 DEBUG "value": [ 04:56:54 DEBUG 1713499014, 04:56:54 DEBUG }, 04:56:54 DEBUG "value": [ 04:56:54 DEBUG } 04:56:54 DEBUG ] 04:56:54 DEBUG }, 04:56:54 DEBUG ] 04:56:54 DEBUG } 04:56:54 DEBUG } 04:56:54 DEBUG "0" 04:56:54 DEBUG ] 04:56:54 DEBUG } 04:56:54 DEBUG "3958.1360387478785" 04:56:54 DEBUG } 04:56:54 DEBUG 1713499014, 04:56:54 DEBUG } 04:56:54 DEBUG "value": [ 04:56:54 DEBUG ] 04:56:54 DEBUG } 04:56:54 DEBUG { 04:56:54 DEBUG } 04:56:54 DEBUG } 04:56:54 DEBUG ] 04:56:54 DEBUG ] 04:56:54 DEBUG "0.0380000000000005" 04:56:54 DEBUG ] 04:56:54 DEBUG 1713499014, 04:56:54 DEBUG } 04:56:54 DEBUG } 04:56:54 DEBUG "status": "success", 04:56:54 DEBUG "data": { 04:56:54 DEBUG } 04:56:54 DEBUG } 04:56:54 DEBUG ] 04:56:54 DEBUG } 04:56:54 DEBUG "2000.7333333333333" 04:56:54 DEBUG ] 04:56:54 DEBUG "resultType": "vector", 04:56:54 DEBUG ] 04:56:54 DEBUG } 04:56:54 DEBUG } 04:56:54 DEBUG } 04:56:54 DEBUG ] 04:56:54 DEBUG } 04:56:54 DEBUG "result": [ 04:56:54 DEBUG } 04:56:54 DEBUG } 04:56:54 DEBUG } 04:56:54 DEBUG } 04:56:54 DEBUG { 04:56:54 DEBUG ] 04:56:54 DEBUG ] 04:56:54 DEBUG "metric": { 04:56:54 DEBUG } 04:56:54 DEBUG } 04:56:54 DEBUG "instance": "10.142.0.107:9100" 04:56:54 DEBUG } 04:56:54 DEBUG } 04:56:54 DEBUG }, 04:56:54 DEBUG "value": [ 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG [...] 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG 04:56:54 DEBUG "instance": "10.142.0.43:9100" 04:56:54 DEBUG }, 04:56:54 DEBUG "value": [ 04:56:54 DEBUG 1713499014, 04:56:54 DEBUG "47682805.5" 04:56:54 DEBUG ] 04:56:54 DEBUG } 04:56:54 DEBUG ] 04:56:54 DEBUG } 04:56:54 DEBUG } 04:56:55 INFO 04:56:55 INFO [loop_until]: kubectl --namespace=xlou top pods 04:56:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:56:55 INFO [loop_until]: OK (rc = 0) 04:56:55 DEBUG --- stdout --- 04:56:55 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 9719m 8963Mi am-6fd554f8f7-hvvkw 9722m 8874Mi am-6fd554f8f7-pmj4b 10423m 8972Mi ds-cts-0 7097m 12196Mi ds-cts-1 6770m 12113Mi ds-cts-2 6994m 12154Mi ds-idrepo-0 4731m 3255Mi ds-idrepo-1 4625m 3207Mi ds-idrepo-2 4662m 3066Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 3m 1215Mi lodemon-6b89d774d5-rf7sk 9m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 6613m 1962Mi 04:56:55 DEBUG --- stderr --- 04:56:55 DEBUG 04:56:55 INFO 04:56:55 INFO [loop_until]: kubectl --namespace=xlou top node 04:56:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:56:55 INFO [loop_until]: OK (rc = 0) 04:56:55 DEBUG --- stdout --- 04:56:55 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 78m 0% 6253Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 4658m 15% 6691Mi 5% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9571m 32% 11586Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 86m 0% 3261Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 7128m 23% 14794Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 158m 0% 5200Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 9801m 32% 11741Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 10332m 34% 2731Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 11059m 37% 11170Mi 10% gke-xlou-cdm-ds-0f601751-j488 4914m 16% 6151Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 6747m 22% 16168Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 5138m 17% 6879Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 7050m 23% 14153Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 6700m 42% 5724Mi 9% 04:56:55 DEBUG --- stderr --- 04:56:55 DEBUG 04:57:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713499074" 04:57:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499074" 04:57:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499074" 04:57:54 ERROR [http_cmd]: ERROR 04:57:54 ERROR --- status code --- 04:57:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 04:57:54 ERROR --- http response --- 04:57:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 04:57:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 04:57:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499074" 04:57:54 INFO [http_cmd]: http status code OK 04:57:54 INFO [http_cmd]: http status code OK 04:57:54 DEBUG --- status code --- 04:57:54 DEBUG --- status code --- 04:57:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:57:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:57:54 DEBUG --- http response --- 04:57:54 DEBUG --- http response --- 04:57:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 04:57:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 04:57:54 DEBUG ----- output ----- 04:57:54 DEBUG ----- output ----- 04:57:54 DEBUG { 04:57:54 DEBUG { 04:57:54 DEBUG "status": "success", 04:57:54 DEBUG "status": "success", 04:57:54 DEBUG "data": { 04:57:54 DEBUG "data": { 04:57:54 DEBUG "resultType": "vector", 04:57:54 DEBUG "resultType": "vector", 04:57:54 DEBUG "result": [ 04:57:54 DEBUG "result": [ 04:57:54 DEBUG { 04:57:54 DEBUG { 04:57:54 DEBUG "metric": { 04:57:54 DEBUG "metric": { 04:57:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:57:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:57:54 DEBUG }, 04:57:54 DEBUG }, 04:57:54 DEBUG "value": [ 04:57:54 DEBUG "value": [ 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG [...] 04:57:54 DEBUG [...] 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG "pod": "ds-cts-0" 04:57:54 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 04:57:54 DEBUG }, 04:57:54 DEBUG }, 04:57:54 DEBUG "value": [ 04:57:54 DEBUG "value": [ 04:57:54 DEBUG 1713499074, 04:57:54 DEBUG 1713499074, 04:57:54 DEBUG "38991176.32055843" 04:57:54 DEBUG "0" 04:57:54 DEBUG ] 04:57:54 DEBUG ] 04:57:54 DEBUG } 04:57:54 DEBUG } 04:57:54 DEBUG ] 04:57:54 DEBUG ] 04:57:54 DEBUG } 04:57:54 DEBUG } 04:57:54 DEBUG } 04:57:54 DEBUG } 04:57:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499074" 04:57:54 INFO [http_cmd]: http status code OK 04:57:54 DEBUG --- status code --- 04:57:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:57:54 DEBUG --- http response --- 04:57:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:57:54 DEBUG ----- output ----- 04:57:54 DEBUG { 04:57:54 DEBUG "status": "success", 04:57:54 DEBUG "data": { 04:57:54 DEBUG "resultType": "vector", 04:57:54 DEBUG "result": [ 04:57:54 DEBUG { 04:57:54 DEBUG "metric": { 04:57:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:57:54 DEBUG }, 04:57:54 DEBUG "value": [ 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG [...] 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:57:54 DEBUG }, 04:57:54 DEBUG "value": [ 04:57:54 DEBUG 1713499074, 04:57:54 DEBUG "0" 04:57:54 DEBUG ] 04:57:54 DEBUG } 04:57:54 DEBUG ] 04:57:54 DEBUG } 04:57:54 DEBUG } 04:57:54 INFO [http_cmd]: http status code OK 04:57:54 DEBUG --- status code --- 04:57:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:57:54 DEBUG --- http response --- 04:57:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 04:57:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499074" 04:57:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499074" 04:57:54 INFO [http_cmd]: http status code OK 04:57:54 DEBUG --- status code --- 04:57:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:57:54 DEBUG --- http response --- 04:57:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:57:54 DEBUG ----- output ----- 04:57:54 DEBUG { 04:57:54 DEBUG "status": "success", 04:57:54 DEBUG "data": { 04:57:54 DEBUG "resultType": "vector", 04:57:54 DEBUG "result": [ 04:57:54 DEBUG { 04:57:54 DEBUG "metric": { 04:57:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:57:54 DEBUG }, 04:57:54 DEBUG "value": [ 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG [...] 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:57:54 DEBUG }, 04:57:54 DEBUG "value": [ 04:57:54 DEBUG 1713499074, 04:57:54 DEBUG "2005.2" 04:57:54 DEBUG ] 04:57:54 DEBUG } 04:57:54 DEBUG ] 04:57:54 DEBUG } 04:57:54 DEBUG } 04:57:54 INFO [http_cmd]: http status code OK 04:57:54 DEBUG --- status code --- 04:57:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:57:54 DEBUG --- http response --- 04:57:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:57:54 DEBUG ----- output ----- 04:57:54 DEBUG { 04:57:54 DEBUG "status": "success", 04:57:54 DEBUG "data": { 04:57:54 DEBUG "resultType": "vector", 04:57:54 DEBUG "result": [ 04:57:54 DEBUG { 04:57:54 DEBUG "metric": { 04:57:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:57:54 DEBUG }, 04:57:54 DEBUG "value": [ 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG [...] 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:57:54 DEBUG }, 04:57:54 DEBUG "value": [ 04:57:54 DEBUG 1713499074, 04:57:54 DEBUG "1540674" 04:57:54 DEBUG ] 04:57:54 DEBUG } 04:57:54 DEBUG ] 04:57:54 DEBUG } 04:57:54 DEBUG } 04:57:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713499074" 04:57:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499074" 04:57:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499074" 04:57:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499074" 04:57:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499074" 04:57:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499074" 04:57:54 INFO [http_cmd]: http status code OK 04:57:54 DEBUG --- status code --- 04:57:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:57:54 DEBUG --- http response --- 04:57:54 INFO [http_cmd]: http status code OK 04:57:54 INFO [http_cmd]: http status code OK 04:57:54 DEBUG --- status code --- 04:57:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:57:54 INFO [http_cmd]: http status code OK 04:57:54 DEBUG --- status code --- 04:57:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:57:54 DEBUG ----- output ----- 04:57:54 DEBUG --- status code --- 04:57:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:57:54 DEBUG --- http response --- 04:57:54 DEBUG { 04:57:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:57:54 DEBUG --- http response --- 04:57:54 DEBUG "status": "success", 04:57:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:57:54 DEBUG --- http response --- 04:57:54 DEBUG "data": { 04:57:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:57:54 DEBUG ----- output ----- 04:57:54 DEBUG "resultType": "vector", 04:57:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:57:54 DEBUG ----- output ----- 04:57:54 DEBUG { 04:57:54 DEBUG "result": [ 04:57:54 DEBUG ----- output ----- 04:57:54 DEBUG { 04:57:54 DEBUG "status": "success", 04:57:54 DEBUG { 04:57:54 DEBUG { 04:57:54 DEBUG "status": "success", 04:57:54 DEBUG "data": { 04:57:54 DEBUG "metric": { 04:57:54 DEBUG "status": "success", 04:57:54 DEBUG "data": { 04:57:54 DEBUG "resultType": "vector", 04:57:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:57:54 DEBUG "data": { 04:57:54 DEBUG "resultType": "vector", 04:57:54 DEBUG "result": [ 04:57:54 DEBUG }, 04:57:54 DEBUG "resultType": "vector", 04:57:54 DEBUG "result": [ 04:57:54 DEBUG { 04:57:54 DEBUG "value": [ 04:57:54 DEBUG "result": [ 04:57:54 DEBUG { 04:57:54 DEBUG "metric": { 04:57:54 DEBUG 04:57:54 DEBUG { 04:57:54 DEBUG "metric": { 04:57:54 DEBUG "pod": "ds-cts-0" 04:57:54 DEBUG 04:57:54 DEBUG "metric": { 04:57:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:57:54 DEBUG }, 04:57:54 DEBUG 04:57:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 04:57:54 DEBUG }, 04:57:54 DEBUG "value": [ 04:57:54 DEBUG [...] 04:57:54 DEBUG }, 04:57:54 INFO [http_cmd]: http status code OK 04:57:54 DEBUG "value": [ 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG "value": [ 04:57:54 INFO [http_cmd]: http status code OK 04:57:54 DEBUG --- status code --- 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG --- status code --- 04:57:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:57:54 DEBUG --- http response --- 04:57:54 DEBUG 04:57:54 DEBUG [...] 04:57:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:57:54 DEBUG 04:57:54 DEBUG --- http response --- 04:57:54 DEBUG [...] 04:57:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:57:54 DEBUG 04:57:54 DEBUG }, 04:57:54 DEBUG [...] 04:57:54 DEBUG 04:57:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:57:54 DEBUG ----- output ----- 04:57:54 DEBUG 04:57:54 DEBUG "value": [ 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG ----- output ----- 04:57:54 DEBUG { 04:57:54 DEBUG 04:57:54 DEBUG 1713499074, 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG { 04:57:54 DEBUG "status": "success", 04:57:54 DEBUG "pod": "ds-cts-2" 04:57:54 DEBUG "0" 04:57:54 DEBUG 04:57:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:57:54 DEBUG "status": "success", 04:57:54 DEBUG "data": { 04:57:54 DEBUG }, 04:57:54 DEBUG ] 04:57:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 04:57:54 DEBUG }, 04:57:54 DEBUG "data": { 04:57:54 DEBUG "resultType": "vector", 04:57:54 DEBUG "value": [ 04:57:54 DEBUG } 04:57:54 DEBUG }, 04:57:54 DEBUG "value": [ 04:57:54 DEBUG "resultType": "vector", 04:57:54 DEBUG "result": [ 04:57:54 DEBUG 1713499074, 04:57:54 DEBUG ] 04:57:54 DEBUG "value": [ 04:57:54 DEBUG 1713499074, 04:57:54 DEBUG "result": [ 04:57:54 DEBUG { 04:57:54 DEBUG "0" 04:57:54 DEBUG } 04:57:54 DEBUG 1713499074, 04:57:54 DEBUG "2005.2333333333333" 04:57:54 DEBUG { 04:57:54 DEBUG "metric": { 04:57:54 DEBUG ] 04:57:54 DEBUG } 04:57:54 DEBUG "2157543424" 04:57:54 DEBUG ] 04:57:54 DEBUG "metric": { 04:57:54 DEBUG "pod": "ds-idrepo-0" 04:57:54 DEBUG } 04:57:54 DEBUG ] 04:57:54 DEBUG } 04:57:54 DEBUG "pod": "ds-cts-0" 04:57:54 DEBUG }, 04:57:54 DEBUG ] 04:57:54 DEBUG } 04:57:54 DEBUG ] 04:57:54 DEBUG }, 04:57:54 DEBUG "value": [ 04:57:54 DEBUG } 04:57:54 DEBUG ] 04:57:54 DEBUG } 04:57:54 DEBUG "value": [ 04:57:54 DEBUG 04:57:54 DEBUG } 04:57:54 DEBUG } 04:57:54 DEBUG } 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG } 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG [...] 04:57:54 DEBUG [...] 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG "pod": "ds-idrepo-2" 04:57:54 DEBUG "pod": "ds-cts-2" 04:57:54 DEBUG }, 04:57:54 DEBUG }, 04:57:54 DEBUG "value": [ 04:57:54 DEBUG "value": [ 04:57:54 DEBUG 1713499074, 04:57:54 DEBUG 1713499074, 04:57:54 DEBUG "0" 04:57:54 DEBUG "0.1" 04:57:54 DEBUG ] 04:57:54 DEBUG ] 04:57:54 DEBUG } 04:57:54 DEBUG } 04:57:54 DEBUG ] 04:57:54 DEBUG ] 04:57:54 DEBUG } 04:57:54 DEBUG } 04:57:54 DEBUG } 04:57:54 DEBUG } 04:57:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713499074" 04:57:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713499074" 04:57:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499074" 04:57:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499074" 04:57:54 INFO [http_cmd]: http status code OK 04:57:54 DEBUG --- status code --- 04:57:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:57:54 DEBUG --- http response --- 04:57:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:57:54 DEBUG ----- output ----- 04:57:54 DEBUG { 04:57:54 DEBUG "status": "success", 04:57:54 DEBUG "data": { 04:57:54 DEBUG "resultType": "vector", 04:57:54 DEBUG "result": [ 04:57:54 DEBUG { 04:57:54 DEBUG "metric": { 04:57:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 04:57:54 DEBUG }, 04:57:54 DEBUG "value": [ 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG [...] 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 04:57:54 DEBUG }, 04:57:54 DEBUG "value": [ 04:57:54 DEBUG 1713499074, 04:57:54 DEBUG "6.509330491952813" 04:57:54 DEBUG ] 04:57:54 DEBUG } 04:57:54 DEBUG ] 04:57:54 DEBUG } 04:57:54 DEBUG } 04:57:54 INFO [http_cmd]: http status code OK 04:57:54 DEBUG --- status code --- 04:57:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:57:54 DEBUG --- http response --- 04:57:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:57:54 DEBUG ----- output ----- 04:57:54 DEBUG { 04:57:54 DEBUG "status": "success", 04:57:54 DEBUG "data": { 04:57:54 DEBUG "resultType": "vector", 04:57:54 DEBUG "result": [ 04:57:54 DEBUG { 04:57:54 DEBUG "metric": { 04:57:54 DEBUG "pod": "ds-idrepo-0" 04:57:54 DEBUG }, 04:57:54 DEBUG "value": [ 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG [...] 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG "pod": "ds-cts-2" 04:57:54 DEBUG }, 04:57:54 DEBUG "value": [ 04:57:54 DEBUG 1713499074, 04:57:54 DEBUG "0" 04:57:54 DEBUG ] 04:57:54 DEBUG } 04:57:54 DEBUG ] 04:57:54 DEBUG } 04:57:54 DEBUG } 04:57:54 INFO [http_cmd]: http status code OK 04:57:54 DEBUG --- status code --- 04:57:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:57:54 DEBUG --- http response --- 04:57:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:57:54 DEBUG ----- output ----- 04:57:54 DEBUG { 04:57:54 DEBUG "status": "success", 04:57:54 DEBUG "data": { 04:57:54 DEBUG "resultType": "vector", 04:57:54 DEBUG "result": [ 04:57:54 DEBUG { 04:57:54 DEBUG "metric": { 04:57:54 DEBUG "pod": "ds-cts-0" 04:57:54 DEBUG }, 04:57:54 DEBUG "value": [ 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG [...] 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG "pod": "ds-cts-2" 04:57:54 DEBUG }, 04:57:54 DEBUG "value": [ 04:57:54 DEBUG 1713499074, 04:57:54 DEBUG "0" 04:57:54 DEBUG ] 04:57:54 DEBUG } 04:57:54 DEBUG ] 04:57:54 DEBUG } 04:57:54 DEBUG } 04:57:54 INFO [http_cmd]: http status code OK 04:57:54 DEBUG --- status code --- 04:57:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:57:54 DEBUG --- http response --- 04:57:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 04:57:54 DEBUG ----- output ----- 04:57:54 DEBUG { 04:57:54 DEBUG "status": "success", 04:57:54 DEBUG "data": { 04:57:54 DEBUG "resultType": "vector", 04:57:54 DEBUG "result": [ 04:57:54 DEBUG { 04:57:54 DEBUG "metric": { 04:57:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:57:54 DEBUG }, 04:57:54 DEBUG "value": [ 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG [...] 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG "pod": "ds-cts-0" 04:57:54 DEBUG }, 04:57:54 DEBUG "value": [ 04:57:54 DEBUG 1713499074, 04:57:54 DEBUG "8797.368950936305" 04:57:54 DEBUG ] 04:57:54 DEBUG } 04:57:54 DEBUG ] 04:57:54 DEBUG } 04:57:54 DEBUG } 04:57:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499074" 04:57:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499074" 04:57:54 INFO [http_cmd]: http status code OK 04:57:54 INFO [http_cmd]: http status code OK 04:57:54 DEBUG --- status code --- 04:57:54 DEBUG --- status code --- 04:57:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:57:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:57:54 DEBUG --- http response --- 04:57:54 DEBUG --- http response --- 04:57:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:57:54 DEBUG ----- output ----- 04:57:54 DEBUG { 04:57:54 DEBUG "status": "success", 04:57:54 DEBUG "data": { 04:57:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:57:54 DEBUG "resultType": "vector", 04:57:54 DEBUG ----- output ----- 04:57:54 DEBUG "result": [ 04:57:54 DEBUG { 04:57:54 DEBUG { 04:57:54 DEBUG "status": "success", 04:57:54 DEBUG "metric": { 04:57:54 DEBUG "data": { 04:57:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:57:54 DEBUG "resultType": "vector", 04:57:54 DEBUG }, 04:57:54 DEBUG "result": [ 04:57:54 DEBUG "value": [ 04:57:54 DEBUG { 04:57:54 DEBUG 04:57:54 DEBUG "metric": { 04:57:54 DEBUG 04:57:54 DEBUG "instance": "10.142.0.107:9100" 04:57:54 DEBUG 04:57:54 DEBUG }, 04:57:54 DEBUG [...] 04:57:54 DEBUG "value": [ 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:57:54 DEBUG [...] 04:57:54 DEBUG }, 04:57:54 DEBUG 04:57:54 DEBUG "value": [ 04:57:54 DEBUG 04:57:54 DEBUG 1713499074, 04:57:54 DEBUG 04:57:54 DEBUG "2004.8666666666666" 04:57:54 DEBUG "instance": "10.142.0.43:9100" 04:57:54 DEBUG ] 04:57:54 DEBUG }, 04:57:54 DEBUG } 04:57:54 DEBUG "value": [ 04:57:54 DEBUG ] 04:57:54 DEBUG 1713499074, 04:57:54 DEBUG } 04:57:54 DEBUG "47934851.4" 04:57:54 DEBUG } 04:57:54 DEBUG ] 04:57:54 DEBUG } 04:57:54 DEBUG ] 04:57:54 DEBUG } 04:57:54 DEBUG } 04:57:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499074" 04:57:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499074" 04:57:54 INFO [http_cmd]: http status code OK 04:57:54 DEBUG --- status code --- 04:57:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:57:54 DEBUG --- http response --- 04:57:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:57:54 DEBUG ----- output ----- 04:57:54 DEBUG { 04:57:54 DEBUG "status": "success", 04:57:54 DEBUG "data": { 04:57:54 DEBUG "resultType": "vector", 04:57:54 DEBUG "result": [ 04:57:54 DEBUG { 04:57:54 DEBUG "metric": { 04:57:54 DEBUG "instance": "10.142.0.107:9100" 04:57:54 DEBUG }, 04:57:54 DEBUG "value": [ 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG [...] 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG "instance": "10.142.0.43:9100" 04:57:54 DEBUG }, 04:57:54 DEBUG "value": [ 04:57:54 DEBUG 1713499074, 04:57:54 DEBUG "0.03699999999999903" 04:57:54 DEBUG ] 04:57:54 DEBUG } 04:57:54 DEBUG ] 04:57:54 DEBUG } 04:57:54 DEBUG } 04:57:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713499074" 04:57:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499074" 04:57:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499074" 04:57:54 INFO [http_cmd]: http status code OK 04:57:54 DEBUG --- status code --- 04:57:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:57:54 DEBUG --- http response --- 04:57:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:57:54 DEBUG ----- output ----- 04:57:54 DEBUG { 04:57:54 DEBUG "status": "success", 04:57:54 DEBUG "data": { 04:57:54 DEBUG "resultType": "vector", 04:57:54 DEBUG "result": [ 04:57:54 DEBUG { 04:57:54 DEBUG "metric": { 04:57:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499074" 04:57:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499074" 04:57:54 DEBUG "instance": "10.142.0.107:9100" 04:57:54 DEBUG }, 04:57:54 DEBUG "value": [ 04:57:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499074" 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG [...] 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG "instance": "10.142.0.43:9100" 04:57:54 DEBUG }, 04:57:54 DEBUG "value": [ 04:57:54 DEBUG 1713499074, 04:57:54 DEBUG "0.051633333333330714" 04:57:54 DEBUG ] 04:57:54 DEBUG } 04:57:54 DEBUG ] 04:57:54 DEBUG } 04:57:54 DEBUG } 04:57:54 INFO [http_cmd]: http status code OK 04:57:54 INFO [http_cmd]: http status code OK 04:57:54 DEBUG --- status code --- 04:57:54 DEBUG --- status code --- 04:57:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:57:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:57:54 DEBUG --- http response --- 04:57:54 DEBUG --- http response --- 04:57:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:57:54 DEBUG ----- output ----- 04:57:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:57:54 DEBUG { 04:57:54 DEBUG ----- output ----- 04:57:54 DEBUG "status": "success", 04:57:54 DEBUG { 04:57:54 DEBUG "data": { 04:57:54 DEBUG "status": "success", 04:57:54 DEBUG "resultType": "vector", 04:57:54 DEBUG "data": { 04:57:54 DEBUG "result": [ 04:57:54 DEBUG "resultType": "vector", 04:57:54 DEBUG { 04:57:54 DEBUG "result": [ 04:57:54 DEBUG "metric": { 04:57:54 INFO [http_cmd]: http status code OK 04:57:54 DEBUG { 04:57:54 DEBUG "pod": "ds-idrepo-0" 04:57:54 DEBUG --- status code --- 04:57:54 DEBUG "metric": { 04:57:54 DEBUG }, 04:57:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:57:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:57:54 DEBUG "value": [ 04:57:54 DEBUG --- http response --- 04:57:54 DEBUG }, 04:57:54 DEBUG 04:57:54 DEBUG "value": [ 04:57:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG ----- output ----- 04:57:54 INFO [http_cmd]: http status code OK 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG { 04:57:54 DEBUG --- status code --- 04:57:54 DEBUG [...] 04:57:54 DEBUG 04:57:54 DEBUG "status": "success", 04:57:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:57:54 DEBUG 04:57:54 DEBUG [...] 04:57:54 DEBUG "data": { 04:57:54 DEBUG --- http response --- 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG "resultType": "vector", 04:57:54 DEBUG 04:57:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:57:54 DEBUG 04:57:54 DEBUG "result": [ 04:57:54 DEBUG "pod": "ds-idrepo-2" 04:57:54 DEBUG ----- output ----- 04:57:54 DEBUG 04:57:54 DEBUG { 04:57:54 DEBUG }, 04:57:54 DEBUG { 04:57:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:57:54 DEBUG "metric": { 04:57:54 DEBUG "value": [ 04:57:54 DEBUG "status": "success", 04:57:54 DEBUG }, 04:57:54 DEBUG "instance": "10.142.0.107:9100" 04:57:54 DEBUG 1713499074, 04:57:54 DEBUG "data": { 04:57:54 DEBUG "value": [ 04:57:54 DEBUG }, 04:57:54 DEBUG "0" 04:57:54 DEBUG "resultType": "vector", 04:57:54 DEBUG 1713499074, 04:57:54 DEBUG "value": [ 04:57:54 DEBUG ] 04:57:54 DEBUG "result": [ 04:57:54 DEBUG "55.171" 04:57:54 DEBUG 04:57:54 DEBUG } 04:57:54 DEBUG { 04:57:54 DEBUG ] 04:57:54 DEBUG 04:57:54 INFO [http_cmd]: http status code OK 04:57:54 DEBUG ] 04:57:54 DEBUG "metric": { 04:57:54 DEBUG } 04:57:54 DEBUG 04:57:54 DEBUG --- status code --- 04:57:54 DEBUG } 04:57:54 INFO [http_cmd]: http status code OK 04:57:54 DEBUG "instance": "10.142.0.107:9100" 04:57:54 DEBUG ] 04:57:54 DEBUG [...] 04:57:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:57:54 DEBUG } 04:57:54 DEBUG --- status code --- 04:57:54 DEBUG }, 04:57:54 DEBUG } 04:57:54 DEBUG 04:57:54 DEBUG --- http response --- 04:57:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:57:54 DEBUG "value": [ 04:57:54 DEBUG } 04:57:54 DEBUG 04:57:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:57:54 DEBUG --- http response --- 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG ----- output ----- 04:57:54 DEBUG 04:57:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 04:57:54 DEBUG "instance": "10.142.0.43:9100" 04:57:54 DEBUG { 04:57:54 DEBUG 04:57:54 DEBUG ----- output ----- 04:57:54 DEBUG }, 04:57:54 DEBUG "status": "success", 04:57:54 DEBUG [...] 04:57:54 DEBUG { 04:57:54 DEBUG "value": [ 04:57:54 DEBUG "data": { 04:57:54 DEBUG 04:57:54 DEBUG "status": "success", 04:57:54 DEBUG 1713499074, 04:57:54 DEBUG "resultType": "vector", 04:57:54 DEBUG 04:57:54 DEBUG "data": { 04:57:54 DEBUG "47931326.266666666" 04:57:54 DEBUG "result": [ 04:57:54 DEBUG 04:57:54 DEBUG "resultType": "vector", 04:57:54 DEBUG ] 04:57:54 DEBUG { 04:57:54 DEBUG "instance": "10.142.0.43:9100" 04:57:54 DEBUG "result": [ 04:57:54 DEBUG } 04:57:54 DEBUG "metric": { 04:57:54 DEBUG }, 04:57:54 DEBUG { 04:57:54 DEBUG ] 04:57:54 DEBUG "pod": "ds-idrepo-0" 04:57:54 DEBUG "value": [ 04:57:54 DEBUG "metric": { 04:57:54 DEBUG } 04:57:54 DEBUG }, 04:57:54 DEBUG 1713499074, 04:57:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:57:54 DEBUG } 04:57:54 DEBUG "value": [ 04:57:54 DEBUG "415960148992" 04:57:54 DEBUG }, 04:57:54 DEBUG 04:57:54 DEBUG ] 04:57:54 DEBUG "value": [ 04:57:54 DEBUG 04:57:54 DEBUG } 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG ] 04:57:54 DEBUG 04:57:54 DEBUG [...] 04:57:54 DEBUG } 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG } 04:57:54 DEBUG [...] 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG 04:57:54 DEBUG "pod": "ds-idrepo-2" 04:57:54 DEBUG 04:57:54 DEBUG }, 04:57:54 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 04:57:54 DEBUG "value": [ 04:57:54 DEBUG }, 04:57:54 DEBUG 1713499074, 04:57:54 DEBUG "value": [ 04:57:54 DEBUG "0" 04:57:54 DEBUG 1713499074, 04:57:54 DEBUG ] 04:57:54 DEBUG "0" 04:57:54 DEBUG } 04:57:54 DEBUG ] 04:57:54 DEBUG ] 04:57:54 DEBUG } 04:57:54 DEBUG } 04:57:54 DEBUG ] 04:57:54 DEBUG } 04:57:54 DEBUG } 04:57:54 DEBUG } 04:57:55 INFO 04:57:55 INFO [loop_until]: kubectl --namespace=xlou top pods 04:57:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:57:55 INFO [loop_until]: OK (rc = 0) 04:57:55 DEBUG --- stdout --- 04:57:55 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 9855m 8962Mi am-6fd554f8f7-hvvkw 9352m 8874Mi am-6fd554f8f7-pmj4b 10356m 8971Mi ds-cts-0 7266m 12202Mi ds-cts-1 7012m 12118Mi ds-cts-2 7185m 12160Mi ds-idrepo-0 4843m 3255Mi ds-idrepo-1 4752m 3206Mi ds-idrepo-2 4482m 3066Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 3m 1215Mi lodemon-6b89d774d5-rf7sk 10m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 6551m 1963Mi 04:57:55 DEBUG --- stderr --- 04:57:55 DEBUG 04:57:55 INFO 04:57:55 INFO [loop_until]: kubectl --namespace=xlou top node 04:57:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:57:55 INFO [loop_until]: OK (rc = 0) 04:57:55 DEBUG --- stdout --- 04:57:55 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 78m 0% 6256Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 4627m 15% 6687Mi 5% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9352m 31% 11581Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 79m 0% 3266Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 7365m 24% 14785Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 154m 0% 5194Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 9516m 31% 11742Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 10261m 34% 2726Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 10980m 36% 11178Mi 10% gke-xlou-cdm-ds-0f601751-j488 4848m 16% 6147Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 7139m 23% 16177Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 5109m 17% 6880Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 7055m 23% 14159Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 6643m 41% 5724Mi 9% 04:57:55 DEBUG --- stderr --- 04:57:55 DEBUG 04:58:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713499134" 04:58:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499134" 04:58:54 ERROR [http_cmd]: ERROR 04:58:54 ERROR --- status code --- 04:58:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 04:58:54 ERROR --- http response --- 04:58:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 04:58:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 04:58:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499134" 04:58:54 INFO [http_cmd]: http status code OK 04:58:54 DEBUG --- status code --- 04:58:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:58:54 DEBUG --- http response --- 04:58:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 04:58:54 DEBUG ----- output ----- 04:58:54 DEBUG { 04:58:54 DEBUG "status": "success", 04:58:54 DEBUG "data": { 04:58:54 DEBUG "resultType": "vector", 04:58:54 DEBUG "result": [ 04:58:54 DEBUG { 04:58:54 DEBUG "metric": { 04:58:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:58:54 DEBUG }, 04:58:54 DEBUG "value": [ 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG [...] 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 04:58:54 DEBUG }, 04:58:54 DEBUG "value": [ 04:58:54 DEBUG 1713499134, 04:58:54 DEBUG "0" 04:58:54 DEBUG ] 04:58:54 DEBUG } 04:58:54 DEBUG ] 04:58:54 DEBUG } 04:58:54 DEBUG } 04:58:54 INFO [http_cmd]: http status code OK 04:58:54 DEBUG --- status code --- 04:58:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:58:54 DEBUG --- http response --- 04:58:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:58:54 DEBUG ----- output ----- 04:58:54 DEBUG { 04:58:54 DEBUG "status": "success", 04:58:54 DEBUG "data": { 04:58:54 DEBUG "resultType": "vector", 04:58:54 DEBUG "result": [ 04:58:54 DEBUG { 04:58:54 DEBUG "metric": { 04:58:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:58:54 DEBUG }, 04:58:54 DEBUG "value": [ 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG [...] 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:58:54 DEBUG }, 04:58:54 DEBUG "value": [ 04:58:54 DEBUG 1713499134, 04:58:54 DEBUG "552.7292355441604" 04:58:54 DEBUG ] 04:58:54 DEBUG } 04:58:54 DEBUG ] 04:58:54 DEBUG } 04:58:54 DEBUG } 04:58:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499134" 04:58:54 INFO [http_cmd]: http status code OK 04:58:54 DEBUG --- status code --- 04:58:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:58:54 DEBUG --- http response --- 04:58:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:58:54 DEBUG ----- output ----- 04:58:54 DEBUG { 04:58:54 DEBUG "status": "success", 04:58:54 DEBUG "data": { 04:58:54 DEBUG "resultType": "vector", 04:58:54 DEBUG "result": [ 04:58:54 DEBUG { 04:58:54 DEBUG "metric": { 04:58:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:58:54 DEBUG }, 04:58:54 DEBUG "value": [ 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG [...] 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:58:54 DEBUG }, 04:58:54 DEBUG "value": [ 04:58:54 DEBUG 1713499134, 04:58:54 DEBUG "0" 04:58:54 DEBUG ] 04:58:54 DEBUG } 04:58:54 DEBUG ] 04:58:54 DEBUG } 04:58:54 DEBUG } 04:58:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499134" 04:58:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499134" 04:58:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713499134" 04:58:54 INFO [http_cmd]: http status code OK 04:58:54 DEBUG --- status code --- 04:58:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:58:54 DEBUG --- http response --- 04:58:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 04:58:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499134" 04:58:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499134" 04:58:54 INFO [http_cmd]: http status code OK 04:58:54 DEBUG --- status code --- 04:58:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:58:54 DEBUG --- http response --- 04:58:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:58:54 DEBUG ----- output ----- 04:58:54 DEBUG { 04:58:54 DEBUG "status": "success", 04:58:54 DEBUG "data": { 04:58:54 DEBUG "resultType": "vector", 04:58:54 DEBUG "result": [ 04:58:54 DEBUG { 04:58:54 DEBUG "metric": { 04:58:54 DEBUG "pod": "ds-idrepo-0" 04:58:54 DEBUG }, 04:58:54 DEBUG "value": [ 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG [...] 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG "pod": "ds-idrepo-2" 04:58:54 DEBUG }, 04:58:54 DEBUG "value": [ 04:58:54 DEBUG 1713499134, 04:58:54 DEBUG "0" 04:58:54 DEBUG ] 04:58:54 DEBUG } 04:58:54 DEBUG ] 04:58:54 DEBUG } 04:58:54 DEBUG } 04:58:54 INFO [http_cmd]: http status code OK 04:58:54 DEBUG --- status code --- 04:58:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:58:54 DEBUG --- http response --- 04:58:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:58:54 DEBUG ----- output ----- 04:58:54 DEBUG { 04:58:54 DEBUG "status": "success", 04:58:54 DEBUG "data": { 04:58:54 DEBUG "resultType": "vector", 04:58:54 DEBUG "result": [ 04:58:54 DEBUG { 04:58:54 DEBUG "metric": { 04:58:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 04:58:54 DEBUG }, 04:58:54 DEBUG "value": [ 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG [...] 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 04:58:54 DEBUG }, 04:58:54 DEBUG "value": [ 04:58:54 DEBUG 1713499134, 04:58:54 DEBUG "2156572672" 04:58:54 DEBUG ] 04:58:54 DEBUG } 04:58:54 DEBUG ] 04:58:54 DEBUG } 04:58:54 DEBUG } 04:58:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499134" 04:58:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499134" 04:58:54 INFO [http_cmd]: http status code OK 04:58:54 INFO [http_cmd]: http status code OK 04:58:54 DEBUG --- status code --- 04:58:54 DEBUG --- status code --- 04:58:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:58:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:58:54 DEBUG --- http response --- 04:58:54 DEBUG --- http response --- 04:58:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:58:54 DEBUG ----- output ----- 04:58:54 DEBUG { 04:58:54 DEBUG "status": "success", 04:58:54 DEBUG "data": { 04:58:54 DEBUG "resultType": "vector", 04:58:54 DEBUG "result": [ 04:58:54 DEBUG { 04:58:54 DEBUG "metric": { 04:58:54 DEBUG "pod": "ds-cts-0" 04:58:54 DEBUG }, 04:58:54 DEBUG "value": [ 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG [...] 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG "pod": "ds-cts-2" 04:58:54 DEBUG }, 04:58:54 DEBUG "value": [ 04:58:54 DEBUG 1713499134, 04:58:54 DEBUG "0.1" 04:58:54 DEBUG ] 04:58:54 DEBUG } 04:58:54 DEBUG ] 04:58:54 DEBUG } 04:58:54 DEBUG } 04:58:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:58:54 DEBUG ----- output ----- 04:58:54 DEBUG { 04:58:54 DEBUG "status": "success", 04:58:54 DEBUG "data": { 04:58:54 DEBUG "resultType": "vector", 04:58:54 DEBUG "result": [ 04:58:54 DEBUG { 04:58:54 DEBUG "metric": { 04:58:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:58:54 DEBUG }, 04:58:54 DEBUG "value": [ 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG [...] 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:58:54 DEBUG }, 04:58:54 DEBUG "value": [ 04:58:54 DEBUG 1713499134, 04:58:54 DEBUG "0" 04:58:54 DEBUG ] 04:58:54 DEBUG } 04:58:54 DEBUG ] 04:58:54 DEBUG } 04:58:54 DEBUG } 04:58:54 INFO [http_cmd]: http status code OK 04:58:54 DEBUG --- status code --- 04:58:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:58:54 DEBUG --- http response --- 04:58:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:58:54 DEBUG ----- output ----- 04:58:54 DEBUG { 04:58:54 DEBUG "status": "success", 04:58:54 DEBUG "data": { 04:58:54 DEBUG "resultType": "vector", 04:58:54 DEBUG "result": [ 04:58:54 DEBUG { 04:58:54 DEBUG "metric": { 04:58:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:58:54 DEBUG }, 04:58:54 DEBUG "value": [ 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG [...] 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:58:54 DEBUG }, 04:58:54 DEBUG "value": [ 04:58:54 DEBUG 1713499134, 04:58:54 DEBUG "2007.0666666666666" 04:58:54 DEBUG ] 04:58:54 DEBUG } 04:58:54 DEBUG ] 04:58:54 DEBUG } 04:58:54 DEBUG } 04:58:54 INFO [http_cmd]: http status code OK 04:58:54 DEBUG --- status code --- 04:58:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:58:54 DEBUG --- http response --- 04:58:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:58:54 DEBUG ----- output ----- 04:58:54 DEBUG { 04:58:54 DEBUG "status": "success", 04:58:54 DEBUG "data": { 04:58:54 DEBUG "resultType": "vector", 04:58:54 DEBUG "result": [ 04:58:54 DEBUG { 04:58:54 DEBUG "metric": { 04:58:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:58:54 DEBUG }, 04:58:54 DEBUG "value": [ 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG [...] 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:58:54 DEBUG }, 04:58:54 DEBUG "value": [ 04:58:54 DEBUG 1713499134, 04:58:54 DEBUG "1660785" 04:58:54 DEBUG ] 04:58:54 DEBUG } 04:58:54 DEBUG ] 04:58:54 DEBUG } 04:58:54 DEBUG } 04:58:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713499134" 04:58:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499134" 04:58:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499134" 04:58:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499134" 04:58:54 INFO [http_cmd]: http status code OK 04:58:54 DEBUG --- status code --- 04:58:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:58:54 DEBUG --- http response --- 04:58:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:58:54 DEBUG ----- output ----- 04:58:54 DEBUG { 04:58:54 DEBUG "status": "success", 04:58:54 DEBUG "data": { 04:58:54 DEBUG "resultType": "vector", 04:58:54 DEBUG "result": [ 04:58:54 DEBUG { 04:58:54 DEBUG "metric": { 04:58:54 DEBUG "pod": "ds-cts-0" 04:58:54 DEBUG }, 04:58:54 DEBUG "value": [ 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG [...] 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG "pod": "ds-cts-2" 04:58:54 DEBUG }, 04:58:54 DEBUG "value": [ 04:58:54 DEBUG 1713499134, 04:58:54 DEBUG "0" 04:58:54 DEBUG ] 04:58:54 DEBUG } 04:58:54 DEBUG ] 04:58:54 DEBUG } 04:58:54 DEBUG } 04:58:54 INFO [http_cmd]: http status code OK 04:58:54 DEBUG --- status code --- 04:58:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:58:54 DEBUG --- http response --- 04:58:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:58:54 DEBUG ----- output ----- 04:58:54 DEBUG { 04:58:54 DEBUG "status": "success", 04:58:54 DEBUG "data": { 04:58:54 DEBUG "resultType": "vector", 04:58:54 DEBUG "result": [ 04:58:54 DEBUG { 04:58:54 DEBUG "metric": { 04:58:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 04:58:54 DEBUG }, 04:58:54 DEBUG "value": [ 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG [...] 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 04:58:54 DEBUG }, 04:58:54 DEBUG "value": [ 04:58:54 DEBUG 1713499134, 04:58:54 DEBUG "6.535506796805062" 04:58:54 DEBUG ] 04:58:54 DEBUG } 04:58:54 DEBUG ] 04:58:54 DEBUG } 04:58:54 DEBUG } 04:58:54 INFO [http_cmd]: http status code OK 04:58:54 DEBUG --- status code --- 04:58:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:58:54 DEBUG --- http response --- 04:58:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:58:54 DEBUG ----- output ----- 04:58:54 DEBUG { 04:58:54 DEBUG "status": "success", 04:58:54 DEBUG "data": { 04:58:54 DEBUG "resultType": "vector", 04:58:54 DEBUG "result": [ 04:58:54 DEBUG { 04:58:54 DEBUG "metric": { 04:58:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:58:54 DEBUG }, 04:58:54 DEBUG "value": [ 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG [...] 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:58:54 DEBUG }, 04:58:54 DEBUG "value": [ 04:58:54 DEBUG 1713499134, 04:58:54 DEBUG "2007.0333333333333" 04:58:54 DEBUG ] 04:58:54 DEBUG } 04:58:54 DEBUG ] 04:58:54 DEBUG } 04:58:54 DEBUG } 04:58:54 INFO [http_cmd]: http status code OK 04:58:54 DEBUG --- status code --- 04:58:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:58:54 DEBUG --- http response --- 04:58:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:58:54 DEBUG ----- output ----- 04:58:54 DEBUG { 04:58:54 DEBUG "status": "success", 04:58:54 DEBUG "data": { 04:58:54 DEBUG "resultType": "vector", 04:58:54 DEBUG "result": [ 04:58:54 DEBUG { 04:58:54 DEBUG "metric": { 04:58:54 DEBUG "pod": "ds-cts-0" 04:58:54 DEBUG }, 04:58:54 DEBUG "value": [ 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG [...] 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG "pod": "ds-cts-2" 04:58:54 DEBUG }, 04:58:54 DEBUG "value": [ 04:58:54 DEBUG 1713499134, 04:58:54 DEBUG "0.000017104444444444444" 04:58:54 DEBUG ] 04:58:54 DEBUG } 04:58:54 DEBUG ] 04:58:54 DEBUG } 04:58:54 DEBUG } 04:58:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713499134" 04:58:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499134" 04:58:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499134" 04:58:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499134" 04:58:54 INFO [http_cmd]: http status code OK 04:58:54 DEBUG --- status code --- 04:58:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:58:54 DEBUG --- http response --- 04:58:54 INFO [http_cmd]: http status code OK 04:58:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:58:54 DEBUG --- status code --- 04:58:54 DEBUG ----- output ----- 04:58:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:58:54 DEBUG { 04:58:54 DEBUG --- http response --- 04:58:54 DEBUG "status": "success", 04:58:54 DEBUG "data": { 04:58:54 DEBUG "resultType": "vector", 04:58:54 DEBUG "result": [ 04:58:54 DEBUG { 04:58:54 DEBUG "metric": { 04:58:54 DEBUG "pod": "ds-idrepo-0" 04:58:54 DEBUG }, 04:58:54 DEBUG "value": [ 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG [...] 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG "pod": "ds-cts-2" 04:58:54 DEBUG }, 04:58:54 DEBUG "value": [ 04:58:54 DEBUG 1713499134, 04:58:54 DEBUG "0" 04:58:54 DEBUG ] 04:58:54 DEBUG } 04:58:54 DEBUG ] 04:58:54 DEBUG } 04:58:54 DEBUG } 04:58:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:58:54 DEBUG ----- output ----- 04:58:54 DEBUG { 04:58:54 DEBUG "status": "success", 04:58:54 DEBUG "data": { 04:58:54 DEBUG "resultType": "vector", 04:58:54 DEBUG "result": [ 04:58:54 DEBUG { 04:58:54 DEBUG "metric": { 04:58:54 DEBUG "instance": "10.142.0.107:9100" 04:58:54 DEBUG }, 04:58:54 DEBUG "value": [ 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG [...] 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG "instance": "10.142.0.43:9100" 04:58:54 DEBUG }, 04:58:54 DEBUG "value": [ 04:58:54 DEBUG 1713499134, 04:58:54 DEBUG "0.052533333333338796" 04:58:54 DEBUG ] 04:58:54 DEBUG } 04:58:54 DEBUG ] 04:58:54 DEBUG } 04:58:54 DEBUG } 04:58:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499134" 04:58:54 INFO [http_cmd]: http status code OK 04:58:54 DEBUG --- status code --- 04:58:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:58:54 DEBUG --- http response --- 04:58:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:58:54 DEBUG ----- output ----- 04:58:54 DEBUG { 04:58:54 DEBUG "status": "success", 04:58:54 DEBUG "data": { 04:58:54 DEBUG "resultType": "vector", 04:58:54 DEBUG "result": [ 04:58:54 DEBUG { 04:58:54 DEBUG "metric": { 04:58:54 DEBUG "instance": "10.142.0.107:9100" 04:58:54 DEBUG }, 04:58:54 DEBUG "value": [ 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG [...] 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG "instance": "10.142.0.43:9100" 04:58:54 DEBUG }, 04:58:54 DEBUG "value": [ 04:58:54 DEBUG 1713499134, 04:58:54 DEBUG "47947883.33333333" 04:58:54 DEBUG ] 04:58:54 DEBUG } 04:58:54 DEBUG ] 04:58:54 DEBUG } 04:58:54 DEBUG } 04:58:54 INFO [http_cmd]: http status code OK 04:58:54 DEBUG --- status code --- 04:58:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:58:54 DEBUG --- http response --- 04:58:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:58:54 DEBUG ----- output ----- 04:58:54 DEBUG { 04:58:54 DEBUG "status": "success", 04:58:54 DEBUG "data": { 04:58:54 DEBUG "resultType": "vector", 04:58:54 DEBUG "result": [ 04:58:54 DEBUG { 04:58:54 DEBUG "metric": { 04:58:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:58:54 DEBUG }, 04:58:54 DEBUG "value": [ 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG [...] 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:58:54 DEBUG }, 04:58:54 DEBUG "value": [ 04:58:54 DEBUG 1713499134, 04:58:54 DEBUG "0" 04:58:54 DEBUG ] 04:58:54 DEBUG } 04:58:54 DEBUG ] 04:58:54 DEBUG } 04:58:54 DEBUG } 04:58:54 INFO [http_cmd]: http status code OK 04:58:54 DEBUG --- status code --- 04:58:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:58:54 DEBUG --- http response --- 04:58:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 04:58:54 DEBUG ----- output ----- 04:58:54 DEBUG { 04:58:54 DEBUG "status": "success", 04:58:54 DEBUG "data": { 04:58:54 DEBUG "resultType": "vector", 04:58:54 DEBUG "result": [ 04:58:54 DEBUG { 04:58:54 DEBUG "metric": { 04:58:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:58:54 DEBUG }, 04:58:54 DEBUG "value": [ 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG [...] 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 04:58:54 DEBUG }, 04:58:54 DEBUG "value": [ 04:58:54 DEBUG 1713499134, 04:58:54 DEBUG "0" 04:58:54 DEBUG ] 04:58:54 DEBUG } 04:58:54 DEBUG ] 04:58:54 DEBUG } 04:58:54 DEBUG } 04:58:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499134" 04:58:54 INFO [http_cmd]: http status code OK 04:58:54 DEBUG --- status code --- 04:58:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:58:54 DEBUG --- http response --- 04:58:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:58:54 DEBUG ----- output ----- 04:58:54 DEBUG { 04:58:54 DEBUG "status": "success", 04:58:54 DEBUG "data": { 04:58:54 DEBUG "resultType": "vector", 04:58:54 DEBUG "result": [ 04:58:54 DEBUG { 04:58:54 DEBUG "metric": { 04:58:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:58:54 DEBUG }, 04:58:54 DEBUG "value": [ 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG [...] 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG 04:58:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:58:54 DEBUG }, 04:58:54 DEBUG "value": [ 04:58:54 DEBUG 1713499134, 04:58:54 DEBUG "2006.7333333333333" 04:58:54 DEBUG ] 04:58:54 DEBUG } 04:58:54 DEBUG ] 04:58:54 DEBUG } 04:58:54 DEBUG } 04:58:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499134" 04:58:54 INFO [http_cmd]: http status code OK 04:58:54 DEBUG --- status code --- 04:58:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:58:54 DEBUG --- http response --- 04:58:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:58:54 DEBUG ----- output ----- 04:58:54 DEBUG { 04:58:54 DEBUG "status": "success", 04:58:54 DEBUG "data": { 04:58:54 DEBUG "resultType": "vector", 04:58:55 DEBUG "result": [ 04:58:55 DEBUG { 04:58:55 DEBUG "metric": { 04:58:55 DEBUG "instance": "10.142.0.107:9100" 04:58:55 DEBUG }, 04:58:55 DEBUG "value": [ 04:58:55 DEBUG 04:58:55 DEBUG 04:58:55 DEBUG 04:58:55 DEBUG [...] 04:58:55 DEBUG 04:58:55 DEBUG 04:58:55 DEBUG 04:58:55 DEBUG "instance": "10.142.0.43:9100" 04:58:55 DEBUG }, 04:58:55 DEBUG "value": [ 04:58:55 DEBUG 1713499134, 04:58:55 DEBUG "0.03499999999999944" 04:58:55 DEBUG ] 04:58:55 DEBUG } 04:58:55 DEBUG ] 04:58:55 DEBUG } 04:58:55 DEBUG } 04:58:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499135" 04:58:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499135" 04:58:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713499135" 04:58:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499135" 04:58:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499135" 04:58:55 INFO [http_cmd]: http status code OK 04:58:55 DEBUG --- status code --- 04:58:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:58:55 DEBUG --- http response --- 04:58:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:58:55 DEBUG ----- output ----- 04:58:55 DEBUG { 04:58:55 DEBUG "status": "success", 04:58:55 DEBUG "data": { 04:58:55 DEBUG "resultType": "vector", 04:58:55 DEBUG "result": [ 04:58:55 DEBUG { 04:58:55 DEBUG "metric": { 04:58:55 DEBUG "instance": "10.142.0.107:9100" 04:58:55 DEBUG }, 04:58:55 DEBUG "value": [ 04:58:55 DEBUG 04:58:55 DEBUG 04:58:55 DEBUG 04:58:55 DEBUG [...] 04:58:55 DEBUG 04:58:55 DEBUG 04:58:55 DEBUG 04:58:55 DEBUG "instance": "10.142.0.43:9100" 04:58:55 DEBUG }, 04:58:55 DEBUG "value": [ 04:58:55 DEBUG 1713499135, 04:58:55 DEBUG "418247920640" 04:58:55 DEBUG ] 04:58:55 DEBUG } 04:58:55 DEBUG ] 04:58:55 DEBUG } 04:58:55 DEBUG } 04:58:55 INFO [http_cmd]: http status code OK 04:58:55 INFO [http_cmd]: http status code OK 04:58:55 DEBUG --- status code --- 04:58:55 DEBUG --- status code --- 04:58:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:58:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:58:55 DEBUG --- http response --- 04:58:55 DEBUG --- http response --- 04:58:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:58:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:58:55 DEBUG ----- output ----- 04:58:55 DEBUG ----- output ----- 04:58:55 DEBUG { 04:58:55 DEBUG { 04:58:55 DEBUG "status": "success", 04:58:55 DEBUG "status": "success", 04:58:55 DEBUG "data": { 04:58:55 DEBUG "data": { 04:58:55 DEBUG "resultType": "vector", 04:58:55 DEBUG "resultType": "vector", 04:58:55 DEBUG "result": [ 04:58:55 DEBUG "result": [ 04:58:55 DEBUG { 04:58:55 DEBUG { 04:58:55 DEBUG "metric": { 04:58:55 DEBUG "metric": { 04:58:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:58:55 DEBUG "pod": "ds-idrepo-0" 04:58:55 DEBUG }, 04:58:55 DEBUG }, 04:58:55 DEBUG "value": [ 04:58:55 DEBUG "value": [ 04:58:55 DEBUG 04:58:55 DEBUG 04:58:55 DEBUG 04:58:55 DEBUG 04:58:55 DEBUG 04:58:55 DEBUG 04:58:55 DEBUG [...] 04:58:55 DEBUG [...] 04:58:55 DEBUG 04:58:55 DEBUG 04:58:55 DEBUG 04:58:55 DEBUG 04:58:55 DEBUG 04:58:55 DEBUG 04:58:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:58:55 DEBUG "pod": "ds-idrepo-2" 04:58:55 DEBUG }, 04:58:55 DEBUG }, 04:58:55 DEBUG "value": [ 04:58:55 DEBUG "value": [ 04:58:55 DEBUG 1713499135, 04:58:55 DEBUG 1713499135, 04:58:55 DEBUG "55.171" 04:58:55 DEBUG "0" 04:58:55 DEBUG ] 04:58:55 DEBUG ] 04:58:55 DEBUG } 04:58:55 DEBUG } 04:58:55 DEBUG ] 04:58:55 DEBUG ] 04:58:55 DEBUG } 04:58:55 DEBUG } 04:58:55 DEBUG } 04:58:55 DEBUG } 04:58:55 INFO [http_cmd]: http status code OK 04:58:55 DEBUG --- status code --- 04:58:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:58:55 DEBUG --- http response --- 04:58:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:58:55 DEBUG ----- output ----- 04:58:55 DEBUG { 04:58:55 DEBUG "status": "success", 04:58:55 DEBUG "data": { 04:58:55 DEBUG "resultType": "vector", 04:58:55 DEBUG "result": [ 04:58:55 DEBUG { 04:58:55 DEBUG "metric": { 04:58:55 DEBUG "instance": "10.142.0.107:9100" 04:58:55 DEBUG }, 04:58:55 DEBUG "value": [ 04:58:55 DEBUG 04:58:55 DEBUG 04:58:55 DEBUG 04:58:55 DEBUG [...] 04:58:55 DEBUG 04:58:55 DEBUG 04:58:55 DEBUG 04:58:55 DEBUG "instance": "10.142.0.43:9100" 04:58:55 DEBUG }, 04:58:55 DEBUG "value": [ 04:58:55 DEBUG 1713499135, 04:58:55 DEBUG "47943838.9" 04:58:55 DEBUG ] 04:58:55 DEBUG } 04:58:55 DEBUG ] 04:58:55 DEBUG } 04:58:55 DEBUG } 04:58:55 INFO [http_cmd]: http status code OK 04:58:55 DEBUG --- status code --- 04:58:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:58:55 DEBUG --- http response --- 04:58:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:58:55 DEBUG ----- output ----- 04:58:55 DEBUG { 04:58:55 DEBUG "status": "success", 04:58:55 DEBUG "data": { 04:58:55 DEBUG "resultType": "vector", 04:58:55 DEBUG "result": [ 04:58:55 DEBUG { 04:58:55 DEBUG "metric": { 04:58:55 DEBUG "pod": "ds-idrepo-0" 04:58:55 DEBUG }, 04:58:55 DEBUG "value": [ 04:58:55 DEBUG 04:58:55 DEBUG 04:58:55 DEBUG 04:58:55 DEBUG [...] 04:58:55 DEBUG 04:58:55 DEBUG 04:58:55 DEBUG 04:58:55 DEBUG "pod": "ds-idrepo-2" 04:58:55 DEBUG }, 04:58:55 DEBUG "value": [ 04:58:55 DEBUG 1713499135, 04:58:55 DEBUG "0" 04:58:55 DEBUG ] 04:58:55 DEBUG } 04:58:55 DEBUG ] 04:58:55 DEBUG } 04:58:55 DEBUG } 04:58:55 INFO 04:58:55 INFO [loop_until]: kubectl --namespace=xlou top pods 04:58:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:58:55 INFO [loop_until]: OK (rc = 0) 04:58:55 DEBUG --- stdout --- 04:58:55 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 9840m 8964Mi am-6fd554f8f7-hvvkw 9281m 8874Mi am-6fd554f8f7-pmj4b 10115m 8974Mi ds-cts-0 7121m 12207Mi ds-cts-1 7289m 12124Mi ds-cts-2 7260m 12166Mi ds-idrepo-0 4764m 3255Mi ds-idrepo-1 4498m 3207Mi ds-idrepo-2 4476m 3066Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 3m 1215Mi lodemon-6b89d774d5-rf7sk 8m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 6533m 1965Mi 04:58:55 DEBUG --- stderr --- 04:58:55 DEBUG 04:58:55 INFO 04:58:55 INFO [loop_until]: kubectl --namespace=xlou top node 04:58:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:58:55 INFO [loop_until]: OK (rc = 0) 04:58:55 DEBUG --- stdout --- 04:58:55 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 76m 0% 6255Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 4474m 14% 6683Mi 5% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9377m 31% 11588Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 81m 0% 3253Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 7409m 24% 14790Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 161m 0% 5187Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 9746m 32% 11742Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 10347m 34% 2739Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 10868m 36% 11169Mi 10% gke-xlou-cdm-ds-0f601751-j488 4872m 16% 6162Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 7300m 24% 16168Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 4943m 16% 6879Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 7364m 24% 14159Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 6672m 41% 5723Mi 9% 04:58:55 DEBUG --- stderr --- 04:58:55 DEBUG 04:59:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713499194" 04:59:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499194" 04:59:54 ERROR [http_cmd]: ERROR 04:59:54 ERROR --- status code --- 04:59:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 04:59:54 ERROR --- http response --- 04:59:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499194" 04:59:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499194" 04:59:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 04:59:54 INFO [http_cmd]: http status code OK 04:59:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 04:59:54 DEBUG --- status code --- 04:59:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:59:54 DEBUG --- http response --- 04:59:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 04:59:54 DEBUG ----- output ----- 04:59:54 DEBUG { 04:59:54 DEBUG "status": "success", 04:59:54 DEBUG "data": { 04:59:54 DEBUG "resultType": "vector", 04:59:54 DEBUG "result": [ 04:59:54 DEBUG { 04:59:54 DEBUG "metric": { 04:59:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:59:54 DEBUG }, 04:59:54 DEBUG "value": [ 04:59:54 DEBUG 04:59:54 DEBUG 04:59:54 DEBUG 04:59:54 DEBUG [...] 04:59:54 DEBUG 04:59:54 DEBUG 04:59:54 DEBUG 04:59:54 DEBUG "pod": "ds-idrepo-1" 04:59:54 DEBUG }, 04:59:54 DEBUG "value": [ 04:59:54 DEBUG 1713499194, 04:59:54 DEBUG "10028.326153655436" 04:59:54 DEBUG ] 04:59:54 DEBUG } 04:59:54 DEBUG ] 04:59:54 DEBUG } 04:59:54 DEBUG } 04:59:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499194" 04:59:54 INFO [http_cmd]: http status code OK 04:59:54 DEBUG --- status code --- 04:59:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:59:54 DEBUG --- http response --- 04:59:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 04:59:54 DEBUG ----- output ----- 04:59:54 DEBUG { 04:59:54 DEBUG "status": "success", 04:59:54 DEBUG "data": { 04:59:54 DEBUG "resultType": "vector", 04:59:54 DEBUG "result": [ 04:59:54 DEBUG { 04:59:54 DEBUG "metric": { 04:59:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:59:54 DEBUG }, 04:59:54 DEBUG "value": [ 04:59:54 DEBUG 04:59:54 DEBUG 04:59:54 DEBUG 04:59:54 DEBUG [...] 04:59:54 DEBUG 04:59:54 DEBUG 04:59:54 DEBUG 04:59:54 DEBUG "pod": "ds-idrepo-1" 04:59:54 DEBUG }, 04:59:54 DEBUG "value": [ 04:59:54 DEBUG 1713499194, 04:59:54 DEBUG "5204.574332909784" 04:59:54 DEBUG ] 04:59:54 DEBUG } 04:59:54 DEBUG ] 04:59:54 DEBUG } 04:59:54 DEBUG } 04:59:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499194" 04:59:54 INFO [http_cmd]: http status code OK 04:59:54 DEBUG --- status code --- 04:59:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:59:54 DEBUG --- http response --- 04:59:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 04:59:54 INFO [http_cmd]: http status code OK 04:59:54 INFO [http_cmd]: http status code OK 04:59:54 DEBUG --- status code --- 04:59:54 DEBUG --- status code --- 04:59:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:59:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:59:54 DEBUG --- http response --- 04:59:54 DEBUG --- http response --- 04:59:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:59:54 DEBUG ----- output ----- 04:59:54 DEBUG { 04:59:54 DEBUG "status": "success", 04:59:54 DEBUG "data": { 04:59:54 DEBUG "resultType": "vector", 04:59:54 DEBUG "result": [ 04:59:54 DEBUG { 04:59:54 DEBUG "metric": { 04:59:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:59:54 DEBUG }, 04:59:54 DEBUG "value": [ 04:59:54 DEBUG 04:59:54 DEBUG 04:59:54 DEBUG 04:59:54 DEBUG [...] 04:59:54 DEBUG 04:59:54 DEBUG 04:59:54 DEBUG 04:59:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:59:54 DEBUG }, 04:59:54 DEBUG "value": [ 04:59:54 DEBUG 1713499194, 04:59:54 DEBUG "0" 04:59:54 DEBUG ] 04:59:54 DEBUG } 04:59:54 DEBUG ] 04:59:54 DEBUG } 04:59:54 DEBUG } 04:59:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:59:54 DEBUG ----- output ----- 04:59:54 DEBUG { 04:59:54 DEBUG "status": "success", 04:59:54 DEBUG "data": { 04:59:54 DEBUG "resultType": "vector", 04:59:54 DEBUG "result": [ 04:59:54 DEBUG { 04:59:54 DEBUG "metric": { 04:59:54 DEBUG "pod": "ds-cts-0" 04:59:54 DEBUG }, 04:59:54 DEBUG "value": [ 04:59:54 DEBUG 04:59:54 DEBUG 04:59:54 DEBUG 04:59:54 DEBUG [...] 04:59:54 DEBUG 04:59:54 DEBUG 04:59:54 DEBUG 04:59:54 DEBUG "pod": "ds-cts-2" 04:59:54 DEBUG }, 04:59:54 DEBUG "value": [ 04:59:54 DEBUG 1713499194, 04:59:54 DEBUG "0.1" 04:59:54 DEBUG ] 04:59:54 DEBUG } 04:59:54 DEBUG ] 04:59:54 DEBUG } 04:59:54 DEBUG } 04:59:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713499194" 04:59:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713499194" 04:59:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499194" 04:59:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499194" 04:59:54 INFO [http_cmd]: http status code OK 04:59:54 DEBUG --- status code --- 04:59:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:59:54 DEBUG --- http response --- 04:59:54 INFO [http_cmd]: http status code OK 04:59:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:59:54 DEBUG --- status code --- 04:59:54 DEBUG ----- output ----- 04:59:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:59:54 DEBUG { 04:59:54 DEBUG --- http response --- 04:59:54 DEBUG "status": "success", 04:59:54 DEBUG "data": { 04:59:54 DEBUG "resultType": "vector", 04:59:54 DEBUG "result": [ 04:59:54 DEBUG { 04:59:54 DEBUG "metric": { 04:59:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 04:59:54 DEBUG }, 04:59:54 DEBUG "value": [ 04:59:54 DEBUG 04:59:54 DEBUG 04:59:54 DEBUG 04:59:54 DEBUG [...] 04:59:54 DEBUG 04:59:54 DEBUG 04:59:54 DEBUG 04:59:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 04:59:54 DEBUG }, 04:59:54 DEBUG "value": [ 04:59:54 DEBUG 1713499194, 04:59:54 DEBUG "2159345664" 04:59:54 DEBUG ] 04:59:54 DEBUG } 04:59:54 DEBUG ] 04:59:54 DEBUG } 04:59:54 DEBUG } 04:59:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:59:54 INFO [http_cmd]: http status code OK 04:59:54 DEBUG ----- output ----- 04:59:54 DEBUG --- status code --- 04:59:54 DEBUG { 04:59:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:59:54 DEBUG "status": "success", 04:59:54 DEBUG --- http response --- 04:59:54 DEBUG "data": { 04:59:54 DEBUG "resultType": "vector", 04:59:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:59:54 DEBUG "result": [ 04:59:54 DEBUG ----- output ----- 04:59:54 DEBUG { 04:59:54 DEBUG { 04:59:54 DEBUG "metric": { 04:59:54 DEBUG "status": "success", 04:59:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 04:59:54 DEBUG "data": { 04:59:54 DEBUG }, 04:59:54 DEBUG "resultType": "vector", 04:59:54 DEBUG "value": [ 04:59:54 DEBUG "result": [ 04:59:54 DEBUG 04:59:54 DEBUG { 04:59:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499194" 04:59:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499194" 04:59:54 DEBUG 04:59:54 DEBUG "metric": { 04:59:54 DEBUG 04:59:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499194" 04:59:54 DEBUG "pod": "ds-idrepo-0" 04:59:54 DEBUG [...] 04:59:54 DEBUG }, 04:59:54 DEBUG 04:59:54 DEBUG "value": [ 04:59:54 DEBUG 04:59:54 DEBUG 04:59:54 DEBUG 04:59:54 DEBUG 04:59:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 04:59:54 DEBUG 04:59:54 DEBUG }, 04:59:54 DEBUG [...] 04:59:54 DEBUG "value": [ 04:59:54 DEBUG 04:59:54 DEBUG 1713499194, 04:59:54 DEBUG 04:59:54 DEBUG "6.5492326575233095" 04:59:54 DEBUG 04:59:54 DEBUG ] 04:59:54 DEBUG "pod": "ds-idrepo-2" 04:59:54 DEBUG } 04:59:54 DEBUG }, 04:59:54 INFO [http_cmd]: http status code OK 04:59:54 DEBUG ] 04:59:54 DEBUG "value": [ 04:59:54 DEBUG --- status code --- 04:59:54 DEBUG } 04:59:54 DEBUG 1713499194, 04:59:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:59:54 DEBUG } 04:59:54 DEBUG "0" 04:59:54 DEBUG --- http response --- 04:59:54 DEBUG ] 04:59:54 DEBUG } 04:59:54 DEBUG ] 04:59:54 DEBUG } 04:59:54 DEBUG } 04:59:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:59:54 DEBUG ----- output ----- 04:59:54 DEBUG { 04:59:54 DEBUG "status": "success", 04:59:54 DEBUG "data": { 04:59:54 DEBUG "resultType": "vector", 04:59:54 DEBUG "result": [ 04:59:54 DEBUG { 04:59:54 DEBUG "metric": { 04:59:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:59:54 DEBUG }, 04:59:54 DEBUG "value": [ 04:59:54 DEBUG 04:59:54 DEBUG 04:59:54 DEBUG 04:59:54 DEBUG [...] 04:59:54 DEBUG 04:59:54 DEBUG 04:59:54 DEBUG 04:59:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:59:54 DEBUG }, 04:59:54 DEBUG "value": [ 04:59:54 DEBUG 1713499194, 04:59:54 DEBUG "1781117.5" 04:59:54 DEBUG ] 04:59:54 DEBUG } 04:59:54 DEBUG ] 04:59:54 DEBUG } 04:59:54 DEBUG } 04:59:54 INFO [http_cmd]: http status code OK 04:59:54 INFO [http_cmd]: http status code OK 04:59:54 DEBUG --- status code --- 04:59:54 DEBUG --- status code --- 04:59:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:59:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:59:54 DEBUG --- http response --- 04:59:54 DEBUG --- http response --- 04:59:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:59:54 DEBUG ----- output ----- 04:59:54 DEBUG { 04:59:54 DEBUG "status": "success", 04:59:54 DEBUG "data": { 04:59:54 DEBUG "resultType": "vector", 04:59:54 DEBUG "result": [ 04:59:54 DEBUG { 04:59:54 DEBUG "metric": { 04:59:54 DEBUG "pod": "ds-cts-0" 04:59:54 DEBUG }, 04:59:54 DEBUG "value": [ 04:59:54 DEBUG 04:59:54 DEBUG 04:59:54 DEBUG 04:59:54 DEBUG [...] 04:59:54 DEBUG 04:59:54 DEBUG 04:59:54 DEBUG 04:59:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:59:54 DEBUG "pod": "ds-cts-2" 04:59:54 DEBUG ----- output ----- 04:59:54 DEBUG }, 04:59:54 DEBUG { 04:59:54 DEBUG "value": [ 04:59:54 DEBUG "status": "success", 04:59:54 DEBUG 1713499194, 04:59:54 DEBUG "data": { 04:59:54 DEBUG "0" 04:59:54 DEBUG "resultType": "vector", 04:59:54 DEBUG ] 04:59:54 DEBUG "result": [ 04:59:54 DEBUG } 04:59:54 DEBUG { 04:59:54 DEBUG ] 04:59:54 DEBUG "metric": { 04:59:54 DEBUG } 04:59:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:59:54 DEBUG } 04:59:54 DEBUG }, 04:59:54 DEBUG "value": [ 04:59:54 DEBUG 04:59:54 DEBUG 04:59:54 DEBUG 04:59:54 DEBUG [...] 04:59:54 DEBUG 04:59:54 DEBUG 04:59:54 DEBUG 04:59:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:59:54 DEBUG }, 04:59:54 DEBUG "value": [ 04:59:54 DEBUG 1713499194, 04:59:54 DEBUG "0" 04:59:54 DEBUG ] 04:59:54 DEBUG } 04:59:54 DEBUG ] 04:59:54 DEBUG } 04:59:54 DEBUG } 04:59:54 INFO [http_cmd]: http status code OK 04:59:54 DEBUG --- status code --- 04:59:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:59:54 DEBUG --- http response --- 04:59:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:59:54 DEBUG ----- output ----- 04:59:54 DEBUG { 04:59:54 DEBUG "status": "success", 04:59:54 DEBUG "data": { 04:59:54 DEBUG "resultType": "vector", 04:59:54 DEBUG "result": [ 04:59:54 DEBUG { 04:59:54 DEBUG "metric": { 04:59:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:59:54 DEBUG }, 04:59:54 DEBUG "value": [ 04:59:54 DEBUG 04:59:54 DEBUG 04:59:54 DEBUG 04:59:54 DEBUG [...] 04:59:54 DEBUG 04:59:54 DEBUG 04:59:54 DEBUG 04:59:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:59:54 DEBUG }, 04:59:54 DEBUG "value": [ 04:59:54 DEBUG 1713499194, 04:59:54 DEBUG "2003.8666666666666" 04:59:54 DEBUG ] 04:59:54 DEBUG } 04:59:54 DEBUG ] 04:59:54 DEBUG } 04:59:54 DEBUG } 04:59:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713499194" 04:59:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499194" 04:59:54 INFO [http_cmd]: http status code OK 04:59:54 DEBUG --- status code --- 04:59:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:59:54 DEBUG --- http response --- 04:59:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499194" 04:59:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499194" 04:59:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:59:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499194" 04:59:54 DEBUG ----- output ----- 04:59:54 DEBUG { 04:59:54 DEBUG "status": "success", 04:59:54 DEBUG "data": { 04:59:54 DEBUG "resultType": "vector", 04:59:54 DEBUG "result": [ 04:59:54 DEBUG { 04:59:54 DEBUG "metric": { 04:59:54 DEBUG "pod": "ds-idrepo-0" 04:59:54 DEBUG }, 04:59:54 DEBUG "value": [ 04:59:54 DEBUG 04:59:54 DEBUG 04:59:54 DEBUG 04:59:54 DEBUG [...] 04:59:54 DEBUG 04:59:54 DEBUG 04:59:54 DEBUG 04:59:54 DEBUG "pod": "ds-cts-2" 04:59:54 DEBUG }, 04:59:54 DEBUG "value": [ 04:59:54 DEBUG 1713499194, 04:59:54 DEBUG "0" 04:59:54 DEBUG ] 04:59:54 DEBUG } 04:59:54 DEBUG ] 04:59:54 DEBUG } 04:59:54 DEBUG } 04:59:55 INFO [http_cmd]: http status code OK 04:59:55 DEBUG --- status code --- 04:59:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:59:55 DEBUG --- http response --- 04:59:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:59:55 DEBUG ----- output ----- 04:59:55 DEBUG { 04:59:55 DEBUG "status": "success", 04:59:55 DEBUG "data": { 04:59:55 DEBUG "resultType": "vector", 04:59:55 DEBUG "result": [ 04:59:55 DEBUG { 04:59:55 DEBUG "metric": { 04:59:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:59:55 DEBUG }, 04:59:55 DEBUG "value": [ 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG [...] 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:59:55 DEBUG }, 04:59:55 DEBUG "value": [ 04:59:55 DEBUG 1713499194, 04:59:55 DEBUG "2004.0666666666666" 04:59:55 DEBUG ] 04:59:55 DEBUG } 04:59:55 DEBUG ] 04:59:55 DEBUG } 04:59:55 DEBUG } 04:59:55 INFO [http_cmd]: http status code OK 04:59:55 DEBUG --- status code --- 04:59:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:59:55 DEBUG --- http response --- 04:59:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 04:59:55 DEBUG ----- output ----- 04:59:55 DEBUG { 04:59:55 DEBUG "status": "success", 04:59:55 DEBUG "data": { 04:59:55 DEBUG "resultType": "vector", 04:59:55 DEBUG "result": [ 04:59:55 DEBUG { 04:59:55 DEBUG "metric": { 04:59:55 DEBUG "pod": "ds-cts-0" 04:59:55 DEBUG }, 04:59:55 DEBUG "value": [ 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG [...] 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG "pod": "ds-cts-2" 04:59:55 DEBUG }, 04:59:55 DEBUG "value": [ 04:59:55 DEBUG 1713499194, 04:59:55 DEBUG "0" 04:59:55 DEBUG ] 04:59:55 DEBUG } 04:59:55 DEBUG ] 04:59:55 DEBUG } 04:59:55 DEBUG } 04:59:55 INFO [http_cmd]: http status code OK 04:59:55 DEBUG --- status code --- 04:59:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:59:55 DEBUG --- http response --- 04:59:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:59:55 DEBUG ----- output ----- 04:59:55 DEBUG { 04:59:55 DEBUG "status": "success", 04:59:55 DEBUG "data": { 04:59:55 DEBUG "resultType": "vector", 04:59:55 DEBUG "result": [ 04:59:55 DEBUG { 04:59:55 DEBUG "metric": { 04:59:55 DEBUG "instance": "10.142.0.107:9100" 04:59:55 DEBUG }, 04:59:55 DEBUG "value": [ 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG [...] 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG "instance": "10.142.0.43:9100" 04:59:55 DEBUG }, 04:59:55 DEBUG "value": [ 04:59:55 DEBUG 1713499194, 04:59:55 DEBUG "47925914.13333333" 04:59:55 DEBUG ] 04:59:55 DEBUG } 04:59:55 DEBUG ] 04:59:55 DEBUG } 04:59:55 DEBUG } 04:59:55 INFO [http_cmd]: http status code OK 04:59:55 DEBUG --- status code --- 04:59:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:59:55 DEBUG --- http response --- 04:59:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:59:55 DEBUG ----- output ----- 04:59:55 DEBUG { 04:59:55 DEBUG "status": "success", 04:59:55 DEBUG "data": { 04:59:55 DEBUG "resultType": "vector", 04:59:55 DEBUG "result": [ 04:59:55 DEBUG { 04:59:55 DEBUG "metric": { 04:59:55 DEBUG "instance": "10.142.0.107:9100" 04:59:55 DEBUG }, 04:59:55 DEBUG "value": [ 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG [...] 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG "instance": "10.142.0.43:9100" 04:59:55 DEBUG }, 04:59:55 DEBUG "value": [ 04:59:55 DEBUG 1713499194, 04:59:55 DEBUG "0.0531666666666742" 04:59:55 DEBUG ] 04:59:55 DEBUG } 04:59:55 DEBUG ] 04:59:55 DEBUG } 04:59:55 DEBUG } 04:59:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499195" 04:59:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499195" 04:59:55 INFO [http_cmd]: http status code OK 04:59:55 DEBUG --- status code --- 04:59:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:59:55 DEBUG --- http response --- 04:59:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 04:59:55 DEBUG ----- output ----- 04:59:55 DEBUG { 04:59:55 DEBUG "status": "success", 04:59:55 DEBUG "data": { 04:59:55 DEBUG "resultType": "vector", 04:59:55 DEBUG "result": [ 04:59:55 DEBUG { 04:59:55 DEBUG "metric": { 04:59:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:59:55 DEBUG }, 04:59:55 DEBUG "value": [ 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG [...] 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG "pod": "ds-idrepo-1" 04:59:55 DEBUG }, 04:59:55 DEBUG "value": [ 04:59:55 DEBUG 1713499195, 04:59:55 DEBUG "0" 04:59:55 DEBUG ] 04:59:55 DEBUG } 04:59:55 DEBUG ] 04:59:55 DEBUG } 04:59:55 DEBUG } 04:59:55 INFO [http_cmd]: http status code OK 04:59:55 DEBUG --- status code --- 04:59:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:59:55 DEBUG --- http response --- 04:59:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 04:59:55 DEBUG ----- output ----- 04:59:55 DEBUG { 04:59:55 DEBUG "status": "success", 04:59:55 DEBUG "data": { 04:59:55 DEBUG "resultType": "vector", 04:59:55 DEBUG "result": [ 04:59:55 DEBUG { 04:59:55 DEBUG "metric": { 04:59:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 04:59:55 DEBUG }, 04:59:55 DEBUG "value": [ 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG [...] 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG "pod": "ds-idrepo-1" 04:59:55 DEBUG }, 04:59:55 DEBUG "value": [ 04:59:55 DEBUG 1713499195, 04:59:55 DEBUG "4.708113211640309" 04:59:55 DEBUG ] 04:59:55 DEBUG } 04:59:55 DEBUG ] 04:59:55 DEBUG } 04:59:55 DEBUG } 04:59:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499195" 04:59:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499195" 04:59:55 INFO [http_cmd]: http status code OK 04:59:55 DEBUG --- status code --- 04:59:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:59:55 DEBUG --- http response --- 04:59:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:59:55 DEBUG ----- output ----- 04:59:55 DEBUG { 04:59:55 DEBUG "status": "success", 04:59:55 DEBUG "data": { 04:59:55 DEBUG "resultType": "vector", 04:59:55 DEBUG "result": [ 04:59:55 DEBUG { 04:59:55 DEBUG "metric": { 04:59:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:59:55 DEBUG }, 04:59:55 DEBUG "value": [ 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG [...] 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:59:55 DEBUG }, 04:59:55 DEBUG "value": [ 04:59:55 DEBUG 1713499195, 04:59:55 DEBUG "2005.4666666666667" 04:59:55 DEBUG ] 04:59:55 DEBUG } 04:59:55 DEBUG ] 04:59:55 DEBUG } 04:59:55 DEBUG } 04:59:55 INFO [http_cmd]: http status code OK 04:59:55 DEBUG --- status code --- 04:59:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:59:55 DEBUG --- http response --- 04:59:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:59:55 DEBUG ----- output ----- 04:59:55 DEBUG { 04:59:55 DEBUG "status": "success", 04:59:55 DEBUG "data": { 04:59:55 DEBUG "resultType": "vector", 04:59:55 DEBUG "result": [ 04:59:55 DEBUG { 04:59:55 DEBUG "metric": { 04:59:55 DEBUG "instance": "10.142.0.107:9100" 04:59:55 DEBUG }, 04:59:55 DEBUG "value": [ 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG [...] 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG "instance": "10.142.0.43:9100" 04:59:55 DEBUG }, 04:59:55 DEBUG "value": [ 04:59:55 DEBUG 1713499195, 04:59:55 DEBUG "420597771264" 04:59:55 DEBUG ] 04:59:55 DEBUG } 04:59:55 DEBUG ] 04:59:55 DEBUG } 04:59:55 DEBUG } 04:59:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499195" 04:59:55 INFO [http_cmd]: http status code OK 04:59:55 DEBUG --- status code --- 04:59:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:59:55 DEBUG --- http response --- 04:59:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:59:55 DEBUG ----- output ----- 04:59:55 DEBUG { 04:59:55 DEBUG "status": "success", 04:59:55 DEBUG "data": { 04:59:55 DEBUG "resultType": "vector", 04:59:55 DEBUG "result": [ 04:59:55 DEBUG { 04:59:55 DEBUG "metric": { 04:59:55 DEBUG "instance": "10.142.0.107:9100" 04:59:55 DEBUG }, 04:59:55 DEBUG "value": [ 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG [...] 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG "instance": "10.142.0.43:9100" 04:59:55 DEBUG }, 04:59:55 DEBUG "value": [ 04:59:55 DEBUG 1713499195, 04:59:55 DEBUG "0.036999999999999984" 04:59:55 DEBUG ] 04:59:55 DEBUG } 04:59:55 DEBUG ] 04:59:55 DEBUG } 04:59:55 DEBUG } 04:59:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713499195" 04:59:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499195" 04:59:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499195" 04:59:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499195" 04:59:55 INFO [http_cmd]: http status code OK 04:59:55 DEBUG --- status code --- 04:59:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:59:55 DEBUG --- http response --- 04:59:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:59:55 DEBUG ----- output ----- 04:59:55 DEBUG { 04:59:55 DEBUG "status": "success", 04:59:55 DEBUG "data": { 04:59:55 DEBUG "resultType": "vector", 04:59:55 DEBUG "result": [ 04:59:55 DEBUG { 04:59:55 DEBUG "metric": { 04:59:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 04:59:55 DEBUG }, 04:59:55 DEBUG "value": [ 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG [...] 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 04:59:55 DEBUG }, 04:59:55 DEBUG "value": [ 04:59:55 DEBUG 1713499195, 04:59:55 DEBUG "55.171" 04:59:55 DEBUG ] 04:59:55 DEBUG } 04:59:55 DEBUG ] 04:59:55 DEBUG } 04:59:55 DEBUG } 04:59:55 INFO [http_cmd]: http status code OK 04:59:55 DEBUG --- status code --- 04:59:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:59:55 DEBUG --- http response --- 04:59:55 INFO [http_cmd]: http status code OK 04:59:55 INFO [http_cmd]: http status code OK 04:59:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 04:59:55 DEBUG --- status code --- 04:59:55 DEBUG --- status code --- 04:59:55 DEBUG ----- output ----- 04:59:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:59:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 04:59:55 DEBUG { 04:59:55 DEBUG --- http response --- 04:59:55 DEBUG --- http response --- 04:59:55 DEBUG "status": "success", 04:59:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:59:55 DEBUG "data": { 04:59:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 04:59:55 DEBUG ----- output ----- 04:59:55 DEBUG "resultType": "vector", 04:59:55 DEBUG ----- output ----- 04:59:55 DEBUG { 04:59:55 DEBUG "result": [ 04:59:55 DEBUG { 04:59:55 DEBUG "status": "success", 04:59:55 DEBUG { 04:59:55 DEBUG "status": "success", 04:59:55 DEBUG "data": { 04:59:55 DEBUG "metric": { 04:59:55 DEBUG "data": { 04:59:55 DEBUG "resultType": "vector", 04:59:55 DEBUG "instance": "10.142.0.107:9100" 04:59:55 DEBUG "resultType": "vector", 04:59:55 DEBUG "result": [ 04:59:55 DEBUG }, 04:59:55 DEBUG "result": [ 04:59:55 DEBUG { 04:59:55 DEBUG "value": [ 04:59:55 DEBUG { 04:59:55 DEBUG "metric": { 04:59:55 DEBUG 04:59:55 DEBUG "metric": { 04:59:55 DEBUG "pod": "ds-idrepo-0" 04:59:55 DEBUG 04:59:55 DEBUG "pod": "ds-idrepo-0" 04:59:55 DEBUG }, 04:59:55 DEBUG 04:59:55 DEBUG }, 04:59:55 DEBUG "value": [ 04:59:55 DEBUG [...] 04:59:55 DEBUG "value": [ 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG [...] 04:59:55 DEBUG "instance": "10.142.0.43:9100" 04:59:55 DEBUG [...] 04:59:55 DEBUG 04:59:55 DEBUG }, 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG "value": [ 04:59:55 DEBUG 04:59:55 DEBUG 04:59:55 DEBUG 1713499195, 04:59:55 DEBUG 04:59:55 DEBUG "pod": "ds-idrepo-2" 04:59:55 DEBUG "47922318.53333333" 04:59:55 DEBUG "pod": "ds-idrepo-2" 04:59:55 DEBUG }, 04:59:55 DEBUG ] 04:59:55 DEBUG }, 04:59:55 DEBUG "value": [ 04:59:55 DEBUG } 04:59:55 DEBUG "value": [ 04:59:55 DEBUG 1713499195, 04:59:55 DEBUG ] 04:59:55 DEBUG 1713499195, 04:59:55 DEBUG "0" 04:59:55 DEBUG } 04:59:55 DEBUG "0" 04:59:55 DEBUG ] 04:59:55 DEBUG } 04:59:55 DEBUG ] 04:59:55 DEBUG } 04:59:55 DEBUG } 04:59:55 DEBUG ] 04:59:55 DEBUG ] 04:59:55 DEBUG } 04:59:55 DEBUG } 04:59:55 DEBUG } 04:59:55 DEBUG } 04:59:55 INFO 04:59:55 INFO [loop_until]: kubectl --namespace=xlou top pods 04:59:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:59:55 INFO [loop_until]: OK (rc = 0) 04:59:55 DEBUG --- stdout --- 04:59:55 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 9668m 8964Mi am-6fd554f8f7-hvvkw 9669m 8875Mi am-6fd554f8f7-pmj4b 10044m 8973Mi ds-cts-0 7737m 12213Mi ds-cts-1 7449m 12131Mi ds-cts-2 7707m 12171Mi ds-idrepo-0 4740m 3255Mi ds-idrepo-1 4759m 3206Mi ds-idrepo-2 4593m 3065Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 3m 1215Mi lodemon-6b89d774d5-rf7sk 8m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 6503m 1967Mi 04:59:55 DEBUG --- stderr --- 04:59:55 DEBUG 04:59:55 INFO 04:59:55 INFO [loop_until]: kubectl --namespace=xlou top node 04:59:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 04:59:55 INFO [loop_until]: OK (rc = 0) 04:59:55 DEBUG --- stdout --- 04:59:55 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 79m 0% 6246Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 4563m 15% 6700Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9497m 31% 11579Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 85m 0% 3248Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 7409m 24% 14785Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 160m 0% 5172Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 9765m 32% 11745Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 10369m 34% 2727Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 10925m 36% 11166Mi 10% gke-xlou-cdm-ds-0f601751-j488 4890m 16% 6166Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 7304m 24% 16173Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 5120m 17% 6880Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 7453m 24% 14163Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 6686m 42% 5721Mi 9% 04:59:55 DEBUG --- stderr --- 04:59:55 DEBUG 05:00:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499254" 05:00:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713499254" 05:00:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499254" 05:00:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499254" 05:00:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499254" 05:00:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713499254" 05:00:54 INFO [http_cmd]: http status code OK 05:00:54 DEBUG --- status code --- 05:00:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:00:54 DEBUG --- http response --- 05:00:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499254" 05:00:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:00:54 DEBUG ----- output ----- 05:00:54 DEBUG { 05:00:54 DEBUG "status": "success", 05:00:54 DEBUG "data": { 05:00:54 DEBUG "resultType": "vector", 05:00:54 DEBUG "result": [ 05:00:54 DEBUG { 05:00:54 DEBUG "metric": { 05:00:54 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:00:54 DEBUG }, 05:00:54 DEBUG "value": [ 05:00:54 DEBUG 05:00:54 DEBUG 05:00:54 DEBUG 05:00:54 DEBUG [...] 05:00:54 DEBUG 05:00:54 DEBUG 05:00:54 DEBUG 05:00:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:00:54 DEBUG }, 05:00:54 DEBUG "value": [ 05:00:54 DEBUG 1713499254, 05:00:54 DEBUG "0" 05:00:54 DEBUG ] 05:00:54 DEBUG } 05:00:54 DEBUG ] 05:00:54 DEBUG } 05:00:54 DEBUG } 05:00:54 ERROR [http_cmd]: ERROR 05:00:54 ERROR --- status code --- 05:00:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:00:54 ERROR --- http response --- 05:00:54 INFO [http_cmd]: http status code OK 05:00:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:00:54 DEBUG --- status code --- 05:00:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:00:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:00:54 DEBUG --- http response --- 05:00:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:00:54 DEBUG ----- output ----- 05:00:54 DEBUG { 05:00:54 DEBUG "status": "success", 05:00:54 DEBUG "data": { 05:00:54 DEBUG "resultType": "vector", 05:00:54 DEBUG "result": [ 05:00:54 DEBUG { 05:00:54 DEBUG "metric": { 05:00:54 DEBUG "pod": "ds-idrepo-1" 05:00:54 DEBUG }, 05:00:54 DEBUG "value": [ 05:00:54 DEBUG 05:00:54 DEBUG 05:00:54 DEBUG 05:00:54 DEBUG [...] 05:00:54 DEBUG 05:00:54 DEBUG 05:00:54 DEBUG 05:00:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:00:54 DEBUG }, 05:00:54 DEBUG "value": [ 05:00:54 DEBUG 1713499254, 05:00:54 DEBUG "34506.184196185284" 05:00:54 DEBUG ] 05:00:54 DEBUG } 05:00:54 DEBUG ] 05:00:54 DEBUG } 05:00:54 DEBUG } 05:00:54 INFO [http_cmd]: http status code OK 05:00:54 DEBUG --- status code --- 05:00:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:00:54 DEBUG --- http response --- 05:00:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:00:54 DEBUG ----- output ----- 05:00:54 DEBUG { 05:00:54 DEBUG "status": "success", 05:00:54 DEBUG "data": { 05:00:54 DEBUG "resultType": "vector", 05:00:54 DEBUG "result": [ 05:00:54 DEBUG { 05:00:54 DEBUG "metric": { 05:00:54 DEBUG "pod": "ds-idrepo-1" 05:00:54 DEBUG }, 05:00:54 DEBUG "value": [ 05:00:54 DEBUG 05:00:54 DEBUG 05:00:54 DEBUG 05:00:54 DEBUG [...] 05:00:54 DEBUG 05:00:54 DEBUG 05:00:54 DEBUG 05:00:54 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:00:54 DEBUG }, 05:00:54 DEBUG "value": [ 05:00:54 DEBUG 1713499254, 05:00:54 DEBUG "0" 05:00:54 DEBUG ] 05:00:54 DEBUG } 05:00:54 DEBUG ] 05:00:54 DEBUG } 05:00:54 DEBUG } 05:00:54 INFO [http_cmd]: http status code OK 05:00:54 DEBUG --- status code --- 05:00:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:00:54 DEBUG --- http response --- 05:00:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:00:54 DEBUG ----- output ----- 05:00:54 DEBUG { 05:00:54 DEBUG "status": "success", 05:00:54 DEBUG "data": { 05:00:54 DEBUG "resultType": "vector", 05:00:54 DEBUG "result": [ 05:00:54 DEBUG { 05:00:54 DEBUG "metric": { 05:00:54 DEBUG "pod": "ds-cts-0" 05:00:54 DEBUG }, 05:00:54 DEBUG "value": [ 05:00:54 DEBUG 05:00:54 DEBUG 05:00:54 DEBUG 05:00:54 DEBUG [...] 05:00:54 DEBUG 05:00:54 DEBUG 05:00:54 DEBUG 05:00:54 DEBUG "pod": "ds-cts-2" 05:00:54 DEBUG }, 05:00:54 DEBUG "value": [ 05:00:54 DEBUG 1713499254, 05:00:54 DEBUG "0.1" 05:00:54 DEBUG ] 05:00:54 DEBUG } 05:00:54 DEBUG ] 05:00:54 DEBUG } 05:00:54 DEBUG } 05:00:54 INFO [http_cmd]: http status code OK 05:00:54 DEBUG --- status code --- 05:00:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:00:54 DEBUG --- http response --- 05:00:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:00:54 DEBUG ----- output ----- 05:00:54 DEBUG { 05:00:54 DEBUG "status": "success", 05:00:54 DEBUG "data": { 05:00:54 DEBUG "resultType": "vector", 05:00:54 DEBUG "result": [ 05:00:54 DEBUG { 05:00:54 DEBUG "metric": { 05:00:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:00:54 DEBUG }, 05:00:54 DEBUG "value": [ 05:00:54 DEBUG 05:00:54 DEBUG 05:00:54 DEBUG 05:00:54 DEBUG [...] 05:00:54 DEBUG 05:00:54 DEBUG 05:00:54 DEBUG 05:00:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:00:54 DEBUG }, 05:00:54 DEBUG "value": [ 05:00:54 DEBUG 1713499254, 05:00:54 DEBUG "2161045504" 05:00:54 DEBUG ] 05:00:54 DEBUG } 05:00:54 DEBUG ] 05:00:54 DEBUG } 05:00:54 DEBUG } 05:00:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713499254" 05:00:54 INFO [http_cmd]: http status code OK 05:00:54 DEBUG --- status code --- 05:00:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:00:54 DEBUG --- http response --- 05:00:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:00:54 INFO [http_cmd]: http status code OK 05:00:54 DEBUG --- status code --- 05:00:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:00:54 DEBUG --- http response --- 05:00:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:00:54 DEBUG ----- output ----- 05:00:54 DEBUG { 05:00:54 DEBUG "status": "success", 05:00:54 DEBUG "data": { 05:00:54 DEBUG "resultType": "vector", 05:00:54 DEBUG "result": [ 05:00:54 DEBUG { 05:00:54 DEBUG "metric": { 05:00:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:00:54 DEBUG }, 05:00:54 DEBUG "value": [ 05:00:54 DEBUG 05:00:54 DEBUG 05:00:54 DEBUG 05:00:54 DEBUG [...] 05:00:54 DEBUG 05:00:54 DEBUG 05:00:54 DEBUG 05:00:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:00:54 DEBUG }, 05:00:54 DEBUG "value": [ 05:00:54 DEBUG 1713499254, 05:00:54 DEBUG "6.6403231479984814" 05:00:54 DEBUG ] 05:00:54 DEBUG } 05:00:54 DEBUG ] 05:00:54 DEBUG } 05:00:54 DEBUG } 05:00:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713499255" 05:00:55 INFO [http_cmd]: http status code OK 05:00:55 DEBUG --- status code --- 05:00:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:00:55 DEBUG --- http response --- 05:00:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499255" 05:00:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499255" 05:00:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:00:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499255" 05:00:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499255" 05:00:55 DEBUG ----- output ----- 05:00:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499255" 05:00:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499255" 05:00:55 DEBUG { 05:00:55 DEBUG "status": "success", 05:00:55 DEBUG "data": { 05:00:55 DEBUG "resultType": "vector", 05:00:55 DEBUG "result": [ 05:00:55 DEBUG { 05:00:55 DEBUG "metric": { 05:00:55 DEBUG "pod": "ds-idrepo-0" 05:00:55 DEBUG }, 05:00:55 DEBUG "value": [ 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG [...] 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG "pod": "ds-cts-2" 05:00:55 DEBUG }, 05:00:55 DEBUG "value": [ 05:00:55 DEBUG 1713499255, 05:00:55 DEBUG "0" 05:00:55 DEBUG ] 05:00:55 DEBUG } 05:00:55 DEBUG ] 05:00:55 DEBUG } 05:00:55 DEBUG } 05:00:55 INFO [http_cmd]: http status code OK 05:00:55 INFO [http_cmd]: http status code OK 05:00:55 DEBUG --- status code --- 05:00:55 DEBUG --- status code --- 05:00:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:00:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:00:55 DEBUG --- http response --- 05:00:55 DEBUG --- http response --- 05:00:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:00:55 DEBUG ----- output ----- 05:00:55 DEBUG { 05:00:55 DEBUG "status": "success", 05:00:55 DEBUG "data": { 05:00:55 DEBUG "resultType": "vector", 05:00:55 DEBUG "result": [ 05:00:55 DEBUG { 05:00:55 DEBUG "metric": { 05:00:55 DEBUG "pod": "ds-idrepo-0" 05:00:55 DEBUG }, 05:00:55 DEBUG "value": [ 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG [...] 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG "pod": "ds-idrepo-2" 05:00:55 DEBUG }, 05:00:55 DEBUG "value": [ 05:00:55 DEBUG 1713499255, 05:00:55 DEBUG "0" 05:00:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:00:55 DEBUG ] 05:00:55 INFO [http_cmd]: http status code OK 05:00:55 DEBUG ----- output ----- 05:00:55 DEBUG } 05:00:55 DEBUG --- status code --- 05:00:55 DEBUG { 05:00:55 DEBUG ] 05:00:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:00:55 DEBUG "status": "success", 05:00:55 DEBUG } 05:00:55 DEBUG --- http response --- 05:00:55 DEBUG "data": { 05:00:55 DEBUG } 05:00:55 DEBUG "resultType": "vector", 05:00:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:00:55 DEBUG "result": [ 05:00:55 DEBUG ----- output ----- 05:00:55 DEBUG { 05:00:55 DEBUG { 05:00:55 DEBUG "metric": { 05:00:55 DEBUG "status": "success", 05:00:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:00:55 DEBUG "data": { 05:00:55 INFO [http_cmd]: http status code OK 05:00:55 DEBUG }, 05:00:55 DEBUG "resultType": "vector", 05:00:55 DEBUG --- status code --- 05:00:55 DEBUG "value": [ 05:00:55 DEBUG "result": [ 05:00:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:00:55 DEBUG 05:00:55 DEBUG { 05:00:55 DEBUG --- http response --- 05:00:55 DEBUG 05:00:55 DEBUG "metric": { 05:00:55 DEBUG 05:00:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:00:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:00:55 DEBUG [...] 05:00:55 DEBUG ----- output ----- 05:00:55 DEBUG }, 05:00:55 DEBUG 05:00:55 DEBUG { 05:00:55 DEBUG "value": [ 05:00:55 DEBUG 05:00:55 DEBUG "status": "success", 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG "data": { 05:00:55 DEBUG 05:00:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:00:55 DEBUG "resultType": "vector", 05:00:55 DEBUG 05:00:55 DEBUG }, 05:00:55 DEBUG "result": [ 05:00:55 DEBUG [...] 05:00:55 DEBUG "value": [ 05:00:55 DEBUG { 05:00:55 DEBUG 05:00:55 DEBUG 1713499255, 05:00:55 DEBUG "metric": { 05:00:55 DEBUG 05:00:55 DEBUG "1901709" 05:00:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:00:55 DEBUG 05:00:55 DEBUG ] 05:00:55 DEBUG }, 05:00:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:00:55 INFO [http_cmd]: http status code OK 05:00:55 DEBUG } 05:00:55 DEBUG "value": [ 05:00:55 DEBUG }, 05:00:55 DEBUG --- status code --- 05:00:55 DEBUG ] 05:00:55 DEBUG 05:00:55 DEBUG "value": [ 05:00:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:00:55 DEBUG } 05:00:55 DEBUG 05:00:55 DEBUG 1713499255, 05:00:55 DEBUG --- http response --- 05:00:55 DEBUG } 05:00:55 DEBUG 05:00:55 DEBUG "2006.6" 05:00:55 INFO [http_cmd]: http status code OK 05:00:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:00:55 DEBUG [...] 05:00:55 DEBUG ] 05:00:55 DEBUG --- status code --- 05:00:55 DEBUG ----- output ----- 05:00:55 DEBUG 05:00:55 DEBUG } 05:00:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:00:55 DEBUG { 05:00:55 DEBUG 05:00:55 DEBUG ] 05:00:55 DEBUG --- http response --- 05:00:55 DEBUG "status": "success", 05:00:55 DEBUG 05:00:55 DEBUG } 05:00:55 DEBUG "data": { 05:00:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:00:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:00:55 DEBUG } 05:00:55 DEBUG "resultType": "vector", 05:00:55 DEBUG ----- output ----- 05:00:55 DEBUG }, 05:00:55 DEBUG "result": [ 05:00:55 DEBUG { 05:00:55 DEBUG "value": [ 05:00:55 DEBUG { 05:00:55 DEBUG "status": "success", 05:00:55 DEBUG 1713499255, 05:00:55 DEBUG "metric": { 05:00:55 DEBUG "data": { 05:00:55 DEBUG "2006.6" 05:00:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:00:55 DEBUG "resultType": "vector", 05:00:55 DEBUG ] 05:00:55 DEBUG }, 05:00:55 DEBUG "result": [ 05:00:55 DEBUG } 05:00:55 DEBUG "value": [ 05:00:55 DEBUG { 05:00:55 DEBUG ] 05:00:55 DEBUG 05:00:55 DEBUG "metric": { 05:00:55 DEBUG } 05:00:55 DEBUG 05:00:55 DEBUG "pod": "ds-cts-0" 05:00:55 DEBUG } 05:00:55 DEBUG 05:00:55 DEBUG }, 05:00:55 DEBUG [...] 05:00:55 DEBUG "value": [ 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:00:55 DEBUG [...] 05:00:55 DEBUG }, 05:00:55 DEBUG 05:00:55 DEBUG "value": [ 05:00:55 DEBUG 05:00:55 DEBUG 1713499255, 05:00:55 DEBUG 05:00:55 DEBUG "0" 05:00:55 DEBUG "pod": "ds-cts-2" 05:00:55 DEBUG ] 05:00:55 DEBUG }, 05:00:55 DEBUG } 05:00:55 DEBUG "value": [ 05:00:55 DEBUG ] 05:00:55 DEBUG 1713499255, 05:00:55 DEBUG } 05:00:55 DEBUG "0" 05:00:55 DEBUG } 05:00:55 DEBUG ] 05:00:55 DEBUG } 05:00:55 DEBUG ] 05:00:55 DEBUG } 05:00:55 DEBUG } 05:00:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499255" 05:00:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499255" 05:00:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499255" 05:00:55 INFO [http_cmd]: http status code OK 05:00:55 DEBUG --- status code --- 05:00:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:00:55 DEBUG --- http response --- 05:00:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499255" 05:00:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:00:55 INFO [http_cmd]: http status code OK 05:00:55 DEBUG ----- output ----- 05:00:55 DEBUG --- status code --- 05:00:55 DEBUG { 05:00:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:00:55 DEBUG "status": "success", 05:00:55 DEBUG --- http response --- 05:00:55 DEBUG "data": { 05:00:55 DEBUG "resultType": "vector", 05:00:55 DEBUG "result": [ 05:00:55 DEBUG { 05:00:55 DEBUG "metric": { 05:00:55 DEBUG "pod": "ds-cts-0" 05:00:55 DEBUG }, 05:00:55 DEBUG "value": [ 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG [...] 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG "pod": "ds-cts-2" 05:00:55 DEBUG }, 05:00:55 DEBUG "value": [ 05:00:55 DEBUG 1713499255, 05:00:55 DEBUG "0" 05:00:55 DEBUG ] 05:00:55 DEBUG } 05:00:55 DEBUG ] 05:00:55 DEBUG } 05:00:55 DEBUG } 05:00:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:00:55 DEBUG ----- output ----- 05:00:55 DEBUG { 05:00:55 DEBUG "status": "success", 05:00:55 DEBUG "data": { 05:00:55 DEBUG "resultType": "vector", 05:00:55 DEBUG "result": [ 05:00:55 DEBUG { 05:00:55 DEBUG "metric": { 05:00:55 DEBUG "pod": "ds-idrepo-1" 05:00:55 DEBUG }, 05:00:55 DEBUG "value": [ 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG [...] 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:00:55 DEBUG }, 05:00:55 DEBUG "value": [ 05:00:55 DEBUG 1713499255, 05:00:55 DEBUG "0" 05:00:55 DEBUG ] 05:00:55 DEBUG } 05:00:55 DEBUG ] 05:00:55 DEBUG } 05:00:55 DEBUG } 05:00:55 INFO [http_cmd]: http status code OK 05:00:55 DEBUG --- status code --- 05:00:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:00:55 DEBUG --- http response --- 05:00:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:00:55 DEBUG ----- output ----- 05:00:55 DEBUG { 05:00:55 DEBUG "status": "success", 05:00:55 DEBUG "data": { 05:00:55 DEBUG "resultType": "vector", 05:00:55 DEBUG "result": [ 05:00:55 DEBUG { 05:00:55 DEBUG "metric": { 05:00:55 DEBUG "instance": "10.142.0.107:9100" 05:00:55 DEBUG }, 05:00:55 DEBUG "value": [ 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG [...] 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG "instance": "10.142.0.43:9100" 05:00:55 DEBUG }, 05:00:55 DEBUG "value": [ 05:00:55 DEBUG 1713499255, 05:00:55 DEBUG "47891725.16666667" 05:00:55 DEBUG ] 05:00:55 DEBUG } 05:00:55 DEBUG ] 05:00:55 DEBUG } 05:00:55 DEBUG } 05:00:55 INFO [http_cmd]: http status code OK 05:00:55 DEBUG --- status code --- 05:00:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:00:55 DEBUG --- http response --- 05:00:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:00:55 DEBUG ----- output ----- 05:00:55 DEBUG { 05:00:55 DEBUG "status": "success", 05:00:55 DEBUG "data": { 05:00:55 DEBUG "resultType": "vector", 05:00:55 DEBUG "result": [ 05:00:55 DEBUG { 05:00:55 DEBUG "metric": { 05:00:55 DEBUG "instance": "10.142.0.107:9100" 05:00:55 DEBUG }, 05:00:55 DEBUG "value": [ 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG [...] 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG "instance": "10.142.0.43:9100" 05:00:55 DEBUG }, 05:00:55 DEBUG "value": [ 05:00:55 DEBUG 1713499255, 05:00:55 DEBUG "0.05606666666666532" 05:00:55 DEBUG ] 05:00:55 DEBUG } 05:00:55 DEBUG ] 05:00:55 DEBUG } 05:00:55 DEBUG } 05:00:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499255" 05:00:55 INFO [http_cmd]: http status code OK 05:00:55 DEBUG --- status code --- 05:00:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:00:55 DEBUG --- http response --- 05:00:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499255" 05:00:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:00:55 DEBUG ----- output ----- 05:00:55 DEBUG { 05:00:55 DEBUG "status": "success", 05:00:55 DEBUG "data": { 05:00:55 DEBUG "resultType": "vector", 05:00:55 DEBUG "result": [ 05:00:55 DEBUG { 05:00:55 DEBUG "metric": { 05:00:55 DEBUG "pod": "ds-idrepo-1" 05:00:55 DEBUG }, 05:00:55 DEBUG "value": [ 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG [...] 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:00:55 DEBUG }, 05:00:55 DEBUG "value": [ 05:00:55 DEBUG 1713499255, 05:00:55 DEBUG "6.121903525789062" 05:00:55 DEBUG ] 05:00:55 DEBUG } 05:00:55 DEBUG ] 05:00:55 DEBUG } 05:00:55 DEBUG } 05:00:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499255" 05:00:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499255" 05:00:55 INFO [http_cmd]: http status code OK 05:00:55 DEBUG --- status code --- 05:00:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:00:55 DEBUG --- http response --- 05:00:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:00:55 DEBUG ----- output ----- 05:00:55 DEBUG { 05:00:55 DEBUG "status": "success", 05:00:55 DEBUG "data": { 05:00:55 DEBUG "resultType": "vector", 05:00:55 DEBUG "result": [ 05:00:55 DEBUG { 05:00:55 DEBUG "metric": { 05:00:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:00:55 DEBUG }, 05:00:55 DEBUG "value": [ 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG [...] 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:00:55 DEBUG }, 05:00:55 DEBUG "value": [ 05:00:55 DEBUG 1713499255, 05:00:55 DEBUG "2006.6666666666667" 05:00:55 DEBUG ] 05:00:55 DEBUG } 05:00:55 DEBUG ] 05:00:55 DEBUG } 05:00:55 DEBUG } 05:00:55 INFO [http_cmd]: http status code OK 05:00:55 DEBUG --- status code --- 05:00:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:00:55 DEBUG --- http response --- 05:00:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:00:55 DEBUG ----- output ----- 05:00:55 DEBUG { 05:00:55 DEBUG "status": "success", 05:00:55 DEBUG "data": { 05:00:55 DEBUG "resultType": "vector", 05:00:55 DEBUG "result": [ 05:00:55 DEBUG { 05:00:55 DEBUG "metric": { 05:00:55 DEBUG "instance": "10.142.0.107:9100" 05:00:55 DEBUG }, 05:00:55 DEBUG "value": [ 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG [...] 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG "instance": "10.142.0.43:9100" 05:00:55 DEBUG }, 05:00:55 DEBUG "value": [ 05:00:55 DEBUG 1713499255, 05:00:55 DEBUG "422969750528" 05:00:55 DEBUG ] 05:00:55 DEBUG } 05:00:55 DEBUG ] 05:00:55 DEBUG } 05:00:55 DEBUG } 05:00:55 INFO [http_cmd]: http status code OK 05:00:55 DEBUG --- status code --- 05:00:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:00:55 DEBUG --- http response --- 05:00:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:00:55 DEBUG ----- output ----- 05:00:55 DEBUG { 05:00:55 DEBUG "status": "success", 05:00:55 DEBUG "data": { 05:00:55 DEBUG "resultType": "vector", 05:00:55 DEBUG "result": [ 05:00:55 DEBUG { 05:00:55 DEBUG "metric": { 05:00:55 DEBUG "pod": "ds-idrepo-0" 05:00:55 DEBUG }, 05:00:55 DEBUG "value": [ 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG [...] 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG "pod": "ds-idrepo-2" 05:00:55 DEBUG }, 05:00:55 DEBUG "value": [ 05:00:55 DEBUG 1713499255, 05:00:55 DEBUG "0" 05:00:55 DEBUG ] 05:00:55 DEBUG } 05:00:55 DEBUG ] 05:00:55 DEBUG } 05:00:55 DEBUG } 05:00:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499255" 05:00:55 INFO [http_cmd]: http status code OK 05:00:55 DEBUG --- status code --- 05:00:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:00:55 DEBUG --- http response --- 05:00:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:00:55 DEBUG ----- output ----- 05:00:55 DEBUG { 05:00:55 DEBUG "status": "success", 05:00:55 DEBUG "data": { 05:00:55 DEBUG "resultType": "vector", 05:00:55 DEBUG "result": [ 05:00:55 DEBUG { 05:00:55 DEBUG "metric": { 05:00:55 DEBUG "instance": "10.142.0.107:9100" 05:00:55 DEBUG }, 05:00:55 DEBUG "value": [ 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG [...] 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG "instance": "10.142.0.43:9100" 05:00:55 DEBUG }, 05:00:55 DEBUG "value": [ 05:00:55 DEBUG 1713499255, 05:00:55 DEBUG "0.04033333333333263" 05:00:55 DEBUG ] 05:00:55 DEBUG } 05:00:55 DEBUG ] 05:00:55 DEBUG } 05:00:55 DEBUG } 05:00:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713499255" 05:00:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499255" 05:00:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499255" 05:00:55 INFO [http_cmd]: http status code OK 05:00:55 DEBUG --- status code --- 05:00:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:00:55 DEBUG --- http response --- 05:00:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:00:55 DEBUG ----- output ----- 05:00:55 DEBUG { 05:00:55 DEBUG "status": "success", 05:00:55 DEBUG "data": { 05:00:55 DEBUG "resultType": "vector", 05:00:55 DEBUG "result": [ 05:00:55 DEBUG { 05:00:55 DEBUG "metric": { 05:00:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:00:55 DEBUG }, 05:00:55 DEBUG "value": [ 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG [...] 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:00:55 DEBUG }, 05:00:55 DEBUG "value": [ 05:00:55 DEBUG 1713499255, 05:00:55 DEBUG "55.171" 05:00:55 DEBUG ] 05:00:55 DEBUG } 05:00:55 DEBUG ] 05:00:55 DEBUG } 05:00:55 DEBUG } 05:00:55 INFO [http_cmd]: http status code OK 05:00:55 DEBUG --- status code --- 05:00:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:00:55 DEBUG --- http response --- 05:00:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:00:55 DEBUG ----- output ----- 05:00:55 DEBUG { 05:00:55 DEBUG "status": "success", 05:00:55 DEBUG "data": { 05:00:55 DEBUG "resultType": "vector", 05:00:55 DEBUG "result": [ 05:00:55 DEBUG { 05:00:55 DEBUG "metric": { 05:00:55 DEBUG "instance": "10.142.0.107:9100" 05:00:55 DEBUG }, 05:00:55 DEBUG "value": [ 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG [...] 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG "instance": "10.142.0.43:9100" 05:00:55 DEBUG }, 05:00:55 DEBUG "value": [ 05:00:55 DEBUG 1713499255, 05:00:55 DEBUG "47888072.166666664" 05:00:55 DEBUG ] 05:00:55 DEBUG } 05:00:55 DEBUG ] 05:00:55 DEBUG } 05:00:55 DEBUG } 05:00:55 INFO [http_cmd]: http status code OK 05:00:55 DEBUG --- status code --- 05:00:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:00:55 DEBUG --- http response --- 05:00:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:00:55 DEBUG ----- output ----- 05:00:55 DEBUG { 05:00:55 DEBUG "status": "success", 05:00:55 DEBUG "data": { 05:00:55 DEBUG "resultType": "vector", 05:00:55 DEBUG "result": [ 05:00:55 DEBUG { 05:00:55 DEBUG "metric": { 05:00:55 DEBUG "pod": "ds-idrepo-0" 05:00:55 DEBUG }, 05:00:55 DEBUG "value": [ 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG [...] 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG 05:00:55 DEBUG "pod": "ds-idrepo-2" 05:00:55 DEBUG }, 05:00:55 DEBUG "value": [ 05:00:55 DEBUG 1713499255, 05:00:55 DEBUG "0" 05:00:55 DEBUG ] 05:00:55 DEBUG } 05:00:55 DEBUG ] 05:00:55 DEBUG } 05:00:55 DEBUG } 05:00:55 INFO 05:00:55 INFO [loop_until]: kubectl --namespace=xlou top pods 05:00:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:00:55 INFO [loop_until]: OK (rc = 0) 05:00:55 DEBUG --- stdout --- 05:00:55 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 9814m 8963Mi am-6fd554f8f7-hvvkw 9337m 8874Mi am-6fd554f8f7-pmj4b 10179m 8974Mi ds-cts-0 7344m 12219Mi ds-cts-1 7457m 12137Mi ds-cts-2 7248m 12177Mi ds-idrepo-0 4669m 3255Mi ds-idrepo-1 4727m 3207Mi ds-idrepo-2 4499m 3065Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1172Mi idm-b85f9d978-wnhxb 3m 1215Mi lodemon-6b89d774d5-rf7sk 1m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 6752m 1968Mi 05:00:55 DEBUG --- stderr --- 05:00:55 DEBUG 05:00:55 INFO 05:00:55 INFO [loop_until]: kubectl --namespace=xlou top node 05:00:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:00:55 INFO [loop_until]: OK (rc = 0) 05:00:55 DEBUG --- stdout --- 05:00:55 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 82m 0% 6248Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 4675m 15% 6696Mi 5% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9479m 31% 11593Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 86m 0% 3244Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 7585m 25% 14787Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 165m 0% 5193Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 9775m 32% 11741Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 10339m 34% 2727Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 11101m 37% 11169Mi 10% gke-xlou-cdm-ds-0f601751-j488 4867m 16% 6158Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 7351m 24% 16178Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 5126m 17% 6882Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 7450m 24% 14162Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 6802m 42% 5720Mi 9% 05:00:55 DEBUG --- stderr --- 05:00:55 DEBUG 05:01:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499314" 05:01:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499314" 05:01:55 INFO [http_cmd]: http status code OK 05:01:55 DEBUG --- status code --- 05:01:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:01:55 DEBUG --- http response --- 05:01:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:01:55 DEBUG ----- output ----- 05:01:55 DEBUG { 05:01:55 DEBUG "status": "success", 05:01:55 DEBUG "data": { 05:01:55 DEBUG "resultType": "vector", 05:01:55 DEBUG "result": [ 05:01:55 DEBUG { 05:01:55 DEBUG "metric": { 05:01:55 DEBUG "pod": "ds-idrepo-1" 05:01:55 DEBUG }, 05:01:55 DEBUG "value": [ 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG [...] 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:01:55 DEBUG }, 05:01:55 DEBUG "value": [ 05:01:55 DEBUG 1713499314, 05:01:55 DEBUG "0" 05:01:55 DEBUG ] 05:01:55 DEBUG } 05:01:55 DEBUG ] 05:01:55 DEBUG } 05:01:55 DEBUG } 05:01:55 INFO [http_cmd]: http status code OK 05:01:55 DEBUG --- status code --- 05:01:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:01:55 DEBUG --- http response --- 05:01:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:01:55 DEBUG ----- output ----- 05:01:55 DEBUG { 05:01:55 DEBUG "status": "success", 05:01:55 DEBUG "data": { 05:01:55 DEBUG "resultType": "vector", 05:01:55 DEBUG "result": [ 05:01:55 DEBUG { 05:01:55 DEBUG "metric": { 05:01:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:01:55 DEBUG }, 05:01:55 DEBUG "value": [ 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG [...] 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:01:55 DEBUG }, 05:01:55 DEBUG "value": [ 05:01:55 DEBUG 1713499314, 05:01:55 DEBUG "0" 05:01:55 DEBUG ] 05:01:55 DEBUG } 05:01:55 DEBUG ] 05:01:55 DEBUG } 05:01:55 DEBUG } 05:01:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713499315" 05:01:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499315" 05:01:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499315" 05:01:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713499315" 05:01:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499315" 05:01:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499315" 05:01:55 INFO [http_cmd]: http status code OK 05:01:55 ERROR [http_cmd]: ERROR 05:01:55 INFO [http_cmd]: http status code OK 05:01:55 DEBUG --- status code --- 05:01:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499315" 05:01:55 ERROR --- status code --- 05:01:55 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:01:55 INFO [http_cmd]: http status code OK 05:01:55 DEBUG --- status code --- 05:01:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:01:55 DEBUG --- http response --- 05:01:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713499315" 05:01:55 DEBUG --- status code --- 05:01:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:01:55 ERROR --- http response --- 05:01:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:01:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:01:55 DEBUG --- http response --- 05:01:55 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:01:55 DEBUG ----- output ----- 05:01:55 DEBUG --- http response --- 05:01:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 05:01:55 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:01:55 DEBUG { 05:01:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:01:55 DEBUG ----- output ----- 05:01:55 DEBUG "status": "success", 05:01:55 DEBUG ----- output ----- 05:01:55 DEBUG { 05:01:55 DEBUG "data": { 05:01:55 DEBUG { 05:01:55 INFO [http_cmd]: http status code OK 05:01:55 DEBUG "status": "success", 05:01:55 DEBUG "resultType": "vector", 05:01:55 DEBUG "status": "success", 05:01:55 DEBUG --- status code --- 05:01:55 DEBUG "data": { 05:01:55 DEBUG "result": [ 05:01:55 DEBUG "data": { 05:01:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:01:55 DEBUG "resultType": "vector", 05:01:55 DEBUG { 05:01:55 DEBUG "resultType": "vector", 05:01:55 DEBUG --- http response --- 05:01:55 DEBUG "result": [ 05:01:55 DEBUG "metric": { 05:01:55 DEBUG "result": [ 05:01:55 DEBUG { 05:01:55 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:01:55 DEBUG "pod": "ds-cts-0" 05:01:55 DEBUG { 05:01:55 DEBUG "metric": { 05:01:55 DEBUG }, 05:01:55 DEBUG "metric": { 05:01:55 DEBUG "pod": "ds-idrepo-1" 05:01:55 DEBUG "value": [ 05:01:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:01:55 DEBUG }, 05:01:55 DEBUG 05:01:55 DEBUG }, 05:01:55 DEBUG "value": [ 05:01:55 DEBUG 05:01:55 DEBUG "value": [ 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG [...] 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG [...] 05:01:55 DEBUG 05:01:55 DEBUG [...] 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG "pod": "ds-cts-2" 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG }, 05:01:55 DEBUG 05:01:55 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:01:55 DEBUG "value": [ 05:01:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:01:55 DEBUG }, 05:01:55 DEBUG 1713499315, 05:01:55 DEBUG }, 05:01:55 DEBUG "value": [ 05:01:55 DEBUG "0.1" 05:01:55 DEBUG "value": [ 05:01:55 DEBUG 1713499315, 05:01:55 DEBUG ] 05:01:55 DEBUG 1713499315, 05:01:55 DEBUG "2160500736" 05:01:55 DEBUG } 05:01:55 DEBUG "4852.791865847182" 05:01:55 DEBUG ] 05:01:55 DEBUG ] 05:01:55 DEBUG ] 05:01:55 DEBUG } 05:01:55 DEBUG } 05:01:55 DEBUG } 05:01:55 DEBUG ] 05:01:55 DEBUG } 05:01:55 DEBUG ] 05:01:55 DEBUG } 05:01:55 DEBUG } 05:01:55 DEBUG } 05:01:55 DEBUG } 05:01:55 INFO [http_cmd]: http status code OK 05:01:55 DEBUG --- status code --- 05:01:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:01:55 INFO [http_cmd]: http status code OK 05:01:55 DEBUG --- http response --- 05:01:55 DEBUG --- status code --- 05:01:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:01:55 DEBUG --- http response --- 05:01:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:01:55 DEBUG ----- output ----- 05:01:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:01:55 INFO [http_cmd]: http status code OK 05:01:55 DEBUG { 05:01:55 DEBUG ----- output ----- 05:01:55 DEBUG --- status code --- 05:01:55 DEBUG "status": "success", 05:01:55 DEBUG { 05:01:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:01:55 DEBUG "data": { 05:01:55 DEBUG "status": "success", 05:01:55 DEBUG --- http response --- 05:01:55 DEBUG "resultType": "vector", 05:01:55 DEBUG "data": { 05:01:55 DEBUG "result": [ 05:01:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:01:55 DEBUG "resultType": "vector", 05:01:55 DEBUG { 05:01:55 DEBUG ----- output ----- 05:01:55 DEBUG "result": [ 05:01:55 DEBUG "metric": { 05:01:55 DEBUG { 05:01:55 DEBUG { 05:01:55 DEBUG "pod": "ds-idrepo-0" 05:01:55 DEBUG "status": "success", 05:01:55 DEBUG "metric": { 05:01:55 DEBUG }, 05:01:55 DEBUG "data": { 05:01:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:01:55 DEBUG "value": [ 05:01:55 DEBUG "resultType": "vector", 05:01:55 DEBUG }, 05:01:55 DEBUG 05:01:55 DEBUG "result": [ 05:01:55 DEBUG "value": [ 05:01:55 DEBUG 05:01:55 DEBUG { 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG "metric": { 05:01:55 DEBUG 05:01:55 DEBUG [...] 05:01:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG }, 05:01:55 DEBUG [...] 05:01:55 DEBUG 05:01:55 DEBUG "value": [ 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG "pod": "ds-idrepo-2" 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG }, 05:01:55 DEBUG 05:01:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:01:55 DEBUG "value": [ 05:01:55 DEBUG [...] 05:01:55 DEBUG }, 05:01:55 DEBUG 1713499315, 05:01:55 DEBUG 05:01:55 DEBUG "value": [ 05:01:55 DEBUG "0" 05:01:55 DEBUG 05:01:55 DEBUG 1713499315, 05:01:55 DEBUG ] 05:01:55 DEBUG 05:01:55 DEBUG "2002.5333333333333" 05:01:55 DEBUG } 05:01:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:01:55 DEBUG ] 05:01:55 DEBUG ] 05:01:55 DEBUG }, 05:01:55 DEBUG } 05:01:55 DEBUG } 05:01:55 DEBUG "value": [ 05:01:55 DEBUG ] 05:01:55 DEBUG } 05:01:55 DEBUG 1713499315, 05:01:55 DEBUG } 05:01:55 DEBUG "6.514258040217504" 05:01:55 DEBUG } 05:01:55 DEBUG ] 05:01:55 DEBUG } 05:01:55 DEBUG ] 05:01:55 DEBUG } 05:01:55 DEBUG } 05:01:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713499315" 05:01:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499315" 05:01:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499315" 05:01:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499315" 05:01:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499315" 05:01:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499315" 05:01:55 INFO [http_cmd]: http status code OK 05:01:55 DEBUG --- status code --- 05:01:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:01:55 DEBUG --- http response --- 05:01:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499315" 05:01:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:01:55 DEBUG ----- output ----- 05:01:55 DEBUG { 05:01:55 DEBUG "status": "success", 05:01:55 DEBUG "data": { 05:01:55 DEBUG "resultType": "vector", 05:01:55 DEBUG "result": [ 05:01:55 DEBUG { 05:01:55 DEBUG "metric": { 05:01:55 DEBUG "pod": "ds-idrepo-0" 05:01:55 DEBUG }, 05:01:55 DEBUG "value": [ 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG [...] 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG "pod": "ds-cts-2" 05:01:55 DEBUG }, 05:01:55 DEBUG "value": [ 05:01:55 DEBUG 1713499315, 05:01:55 DEBUG "0" 05:01:55 DEBUG ] 05:01:55 DEBUG } 05:01:55 DEBUG ] 05:01:55 DEBUG } 05:01:55 DEBUG } 05:01:55 INFO [http_cmd]: http status code OK 05:01:55 INFO [http_cmd]: http status code OK 05:01:55 DEBUG --- status code --- 05:01:55 INFO [http_cmd]: http status code OK 05:01:55 DEBUG --- status code --- 05:01:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:01:55 DEBUG --- http response --- 05:01:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:01:55 DEBUG --- status code --- 05:01:55 DEBUG --- http response --- 05:01:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:01:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:01:55 DEBUG ----- output ----- 05:01:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:01:55 DEBUG --- http response --- 05:01:55 DEBUG { 05:01:55 DEBUG ----- output ----- 05:01:55 DEBUG "status": "success", 05:01:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:01:55 DEBUG { 05:01:55 DEBUG "data": { 05:01:55 DEBUG ----- output ----- 05:01:55 INFO [http_cmd]: http status code OK 05:01:55 INFO [http_cmd]: http status code OK 05:01:55 DEBUG "status": "success", 05:01:55 DEBUG "resultType": "vector", 05:01:55 DEBUG { 05:01:55 DEBUG --- status code --- 05:01:55 DEBUG --- status code --- 05:01:55 DEBUG "data": { 05:01:55 DEBUG "result": [ 05:01:55 DEBUG "status": "success", 05:01:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:01:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:01:55 DEBUG "resultType": "vector", 05:01:55 DEBUG { 05:01:55 DEBUG "data": { 05:01:55 DEBUG --- http response --- 05:01:55 DEBUG --- http response --- 05:01:55 DEBUG "result": [ 05:01:55 DEBUG "metric": { 05:01:55 INFO [http_cmd]: http status code OK 05:01:55 DEBUG "resultType": "vector", 05:01:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:01:55 DEBUG { 05:01:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:01:55 DEBUG "pod": "ds-cts-0" 05:01:55 DEBUG --- status code --- 05:01:55 DEBUG "result": [ 05:01:55 DEBUG ----- output ----- 05:01:55 DEBUG "metric": { 05:01:55 DEBUG ----- output ----- 05:01:55 DEBUG }, 05:01:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:01:55 DEBUG { 05:01:55 DEBUG { 05:01:55 DEBUG "pod": "ds-cts-0" 05:01:55 DEBUG { 05:01:55 DEBUG "value": [ 05:01:55 DEBUG --- http response --- 05:01:55 DEBUG "metric": { 05:01:55 DEBUG "status": "success", 05:01:55 DEBUG }, 05:01:55 DEBUG "status": "success", 05:01:55 DEBUG 05:01:55 DEBUG "instance": "10.142.0.107:9100" 05:01:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:01:55 DEBUG "data": { 05:01:55 DEBUG "value": [ 05:01:55 DEBUG "data": { 05:01:55 DEBUG 05:01:55 DEBUG }, 05:01:55 DEBUG ----- output ----- 05:01:55 DEBUG "resultType": "vector", 05:01:55 DEBUG 05:01:55 DEBUG "resultType": "vector", 05:01:55 DEBUG 05:01:55 DEBUG "value": [ 05:01:55 DEBUG { 05:01:55 DEBUG "result": [ 05:01:55 DEBUG 05:01:55 DEBUG "result": [ 05:01:55 DEBUG [...] 05:01:55 DEBUG 05:01:55 DEBUG "status": "success", 05:01:55 DEBUG { 05:01:55 DEBUG 05:01:55 DEBUG { 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG "data": { 05:01:55 DEBUG "metric": { 05:01:55 DEBUG [...] 05:01:55 DEBUG "metric": { 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG "resultType": "vector", 05:01:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:01:55 DEBUG 05:01:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:01:55 DEBUG 05:01:55 DEBUG [...] 05:01:55 DEBUG "result": [ 05:01:55 DEBUG }, 05:01:55 DEBUG 05:01:55 DEBUG }, 05:01:55 DEBUG "pod": "ds-cts-2" 05:01:55 DEBUG 05:01:55 DEBUG { 05:01:55 DEBUG "value": [ 05:01:55 DEBUG 05:01:55 DEBUG "value": [ 05:01:55 DEBUG }, 05:01:55 DEBUG 05:01:55 DEBUG "metric": { 05:01:55 DEBUG 05:01:55 DEBUG "pod": "ds-cts-2" 05:01:55 DEBUG 05:01:55 DEBUG "value": [ 05:01:55 DEBUG 05:01:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:01:55 DEBUG 05:01:55 DEBUG }, 05:01:55 DEBUG 05:01:55 DEBUG 1713499315, 05:01:55 DEBUG "instance": "10.142.0.43:9100" 05:01:55 DEBUG }, 05:01:55 DEBUG 05:01:55 DEBUG "value": [ 05:01:55 DEBUG 05:01:55 DEBUG "0" 05:01:55 DEBUG }, 05:01:55 DEBUG "value": [ 05:01:55 DEBUG [...] 05:01:55 DEBUG 1713499315, 05:01:55 DEBUG [...] 05:01:55 DEBUG ] 05:01:55 DEBUG "value": [ 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG "0" 05:01:55 DEBUG 05:01:55 DEBUG } 05:01:55 DEBUG 1713499315, 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG ] 05:01:55 DEBUG 05:01:55 DEBUG ] 05:01:55 DEBUG "47665947.46666667" 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG } 05:01:55 DEBUG 05:01:55 DEBUG } 05:01:55 DEBUG ] 05:01:55 DEBUG [...] 05:01:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:01:55 DEBUG ] 05:01:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:01:55 DEBUG } 05:01:55 DEBUG } 05:01:55 DEBUG 05:01:55 DEBUG }, 05:01:55 DEBUG } 05:01:55 DEBUG }, 05:01:55 DEBUG ] 05:01:55 DEBUG 05:01:55 DEBUG "value": [ 05:01:55 DEBUG } 05:01:55 DEBUG "value": [ 05:01:55 DEBUG } 05:01:55 DEBUG 05:01:55 DEBUG 1713499315, 05:01:55 DEBUG 1713499315, 05:01:55 DEBUG } 05:01:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:01:55 DEBUG "0" 05:01:55 DEBUG "2021809.5" 05:01:55 DEBUG }, 05:01:55 DEBUG ] 05:01:55 DEBUG ] 05:01:55 DEBUG "value": [ 05:01:55 DEBUG } 05:01:55 DEBUG } 05:01:55 DEBUG 1713499315, 05:01:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499315" 05:01:55 DEBUG ] 05:01:55 DEBUG ] 05:01:55 DEBUG "2002.3" 05:01:55 DEBUG } 05:01:55 DEBUG } 05:01:55 DEBUG ] 05:01:55 DEBUG } 05:01:55 DEBUG } 05:01:55 DEBUG } 05:01:55 DEBUG ] 05:01:55 DEBUG } 05:01:55 DEBUG } 05:01:55 INFO [http_cmd]: http status code OK 05:01:55 DEBUG --- status code --- 05:01:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:01:55 DEBUG --- http response --- 05:01:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 05:01:55 DEBUG ----- output ----- 05:01:55 DEBUG { 05:01:55 DEBUG "status": "success", 05:01:55 DEBUG "data": { 05:01:55 DEBUG "resultType": "vector", 05:01:55 DEBUG "result": [ 05:01:55 DEBUG { 05:01:55 DEBUG "metric": { 05:01:55 DEBUG "pod": "ds-idrepo-1" 05:01:55 DEBUG }, 05:01:55 DEBUG "value": [ 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG [...] 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:01:55 DEBUG }, 05:01:55 DEBUG "value": [ 05:01:55 DEBUG 1713499315, 05:01:55 DEBUG "0" 05:01:55 DEBUG ] 05:01:55 DEBUG } 05:01:55 DEBUG ] 05:01:55 DEBUG } 05:01:55 DEBUG } 05:01:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499315" 05:01:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499315" 05:01:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499315" 05:01:55 INFO [http_cmd]: http status code OK 05:01:55 DEBUG --- status code --- 05:01:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:01:55 DEBUG --- http response --- 05:01:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:01:55 DEBUG ----- output ----- 05:01:55 DEBUG { 05:01:55 DEBUG "status": "success", 05:01:55 DEBUG "data": { 05:01:55 DEBUG "resultType": "vector", 05:01:55 DEBUG "result": [ 05:01:55 DEBUG { 05:01:55 DEBUG "metric": { 05:01:55 DEBUG "instance": "10.142.0.107:9100" 05:01:55 DEBUG }, 05:01:55 DEBUG "value": [ 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG [...] 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG "instance": "10.142.0.43:9100" 05:01:55 DEBUG }, 05:01:55 DEBUG "value": [ 05:01:55 DEBUG 1713499315, 05:01:55 DEBUG "425282456576" 05:01:55 DEBUG ] 05:01:55 DEBUG } 05:01:55 DEBUG ] 05:01:55 DEBUG } 05:01:55 DEBUG } 05:01:55 INFO [http_cmd]: http status code OK 05:01:55 DEBUG --- status code --- 05:01:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:01:55 DEBUG --- http response --- 05:01:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 05:01:55 DEBUG ----- output ----- 05:01:55 DEBUG { 05:01:55 DEBUG "status": "success", 05:01:55 DEBUG "data": { 05:01:55 DEBUG "resultType": "vector", 05:01:55 DEBUG "result": [ 05:01:55 DEBUG { 05:01:55 DEBUG "metric": { 05:01:55 DEBUG "pod": "ds-idrepo-1" 05:01:55 DEBUG }, 05:01:55 DEBUG "value": [ 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG [...] 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:01:55 DEBUG }, 05:01:55 DEBUG "value": [ 05:01:55 DEBUG 1713499315, 05:01:55 DEBUG "3.3591075152321443" 05:01:55 DEBUG ] 05:01:55 DEBUG } 05:01:55 DEBUG ] 05:01:55 DEBUG } 05:01:55 DEBUG } 05:01:55 INFO [http_cmd]: http status code OK 05:01:55 DEBUG --- status code --- 05:01:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:01:55 DEBUG --- http response --- 05:01:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:01:55 DEBUG ----- output ----- 05:01:55 DEBUG { 05:01:55 DEBUG "status": "success", 05:01:55 DEBUG "data": { 05:01:55 DEBUG "resultType": "vector", 05:01:55 DEBUG "result": [ 05:01:55 DEBUG { 05:01:55 DEBUG "metric": { 05:01:55 DEBUG "instance": "10.142.0.107:9100" 05:01:55 DEBUG }, 05:01:55 DEBUG "value": [ 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG [...] 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG "instance": "10.142.0.43:9100" 05:01:55 DEBUG }, 05:01:55 DEBUG "value": [ 05:01:55 DEBUG 1713499315, 05:01:55 DEBUG "0.05623333333332804" 05:01:55 DEBUG ] 05:01:55 DEBUG } 05:01:55 DEBUG ] 05:01:55 DEBUG } 05:01:55 DEBUG } 05:01:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499315" 05:01:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499315" 05:01:55 INFO [http_cmd]: http status code OK 05:01:55 INFO [http_cmd]: http status code OK 05:01:55 DEBUG --- status code --- 05:01:55 DEBUG --- status code --- 05:01:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:01:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:01:55 DEBUG --- http response --- 05:01:55 DEBUG --- http response --- 05:01:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:01:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:01:55 DEBUG ----- output ----- 05:01:55 DEBUG ----- output ----- 05:01:55 DEBUG { 05:01:55 DEBUG { 05:01:55 DEBUG "status": "success", 05:01:55 DEBUG "status": "success", 05:01:55 DEBUG "data": { 05:01:55 DEBUG "data": { 05:01:55 DEBUG "resultType": "vector", 05:01:55 DEBUG "resultType": "vector", 05:01:55 DEBUG "result": [ 05:01:55 DEBUG "result": [ 05:01:55 DEBUG { 05:01:55 DEBUG { 05:01:55 DEBUG "metric": { 05:01:55 DEBUG "metric": { 05:01:55 DEBUG "pod": "ds-idrepo-0" 05:01:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:01:55 DEBUG }, 05:01:55 DEBUG }, 05:01:55 DEBUG "value": [ 05:01:55 DEBUG "value": [ 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG [...] 05:01:55 DEBUG [...] 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG "pod": "ds-idrepo-2" 05:01:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:01:55 DEBUG }, 05:01:55 DEBUG }, 05:01:55 DEBUG "value": [ 05:01:55 DEBUG "value": [ 05:01:55 DEBUG 1713499315, 05:01:55 DEBUG 1713499315, 05:01:55 DEBUG "0" 05:01:55 DEBUG "2000.3999999999999" 05:01:55 DEBUG ] 05:01:55 DEBUG ] 05:01:55 DEBUG } 05:01:55 DEBUG } 05:01:55 DEBUG ] 05:01:55 DEBUG ] 05:01:55 DEBUG } 05:01:55 DEBUG } 05:01:55 DEBUG } 05:01:55 DEBUG } 05:01:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499315" 05:01:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713499315" 05:01:55 INFO [http_cmd]: http status code OK 05:01:55 DEBUG --- status code --- 05:01:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:01:55 DEBUG --- http response --- 05:01:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:01:55 DEBUG ----- output ----- 05:01:55 DEBUG { 05:01:55 DEBUG "status": "success", 05:01:55 DEBUG "data": { 05:01:55 DEBUG "resultType": "vector", 05:01:55 DEBUG "result": [ 05:01:55 DEBUG { 05:01:55 DEBUG "metric": { 05:01:55 DEBUG "pod": "ds-idrepo-0" 05:01:55 DEBUG }, 05:01:55 DEBUG "value": [ 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG [...] 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG "pod": "ds-idrepo-2" 05:01:55 DEBUG }, 05:01:55 DEBUG "value": [ 05:01:55 DEBUG 1713499315, 05:01:55 DEBUG "0" 05:01:55 DEBUG ] 05:01:55 DEBUG } 05:01:55 DEBUG ] 05:01:55 DEBUG } 05:01:55 DEBUG } 05:01:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499315" 05:01:55 INFO [http_cmd]: http status code OK 05:01:55 DEBUG --- status code --- 05:01:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:01:55 DEBUG --- http response --- 05:01:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:01:55 DEBUG ----- output ----- 05:01:55 DEBUG { 05:01:55 DEBUG "status": "success", 05:01:55 DEBUG "data": { 05:01:55 DEBUG "resultType": "vector", 05:01:55 DEBUG "result": [ 05:01:55 DEBUG { 05:01:55 DEBUG "metric": { 05:01:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:01:55 DEBUG }, 05:01:55 DEBUG "value": [ 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG [...] 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:01:55 DEBUG }, 05:01:55 DEBUG "value": [ 05:01:55 DEBUG 1713499315, 05:01:55 DEBUG "55.171" 05:01:55 DEBUG ] 05:01:55 DEBUG } 05:01:55 DEBUG ] 05:01:55 DEBUG } 05:01:55 DEBUG } 05:01:55 INFO [http_cmd]: http status code OK 05:01:55 DEBUG --- status code --- 05:01:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:01:55 DEBUG --- http response --- 05:01:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:01:55 DEBUG ----- output ----- 05:01:55 DEBUG { 05:01:55 DEBUG "status": "success", 05:01:55 DEBUG "data": { 05:01:55 DEBUG "resultType": "vector", 05:01:55 DEBUG "result": [ 05:01:55 DEBUG { 05:01:55 DEBUG "metric": { 05:01:55 DEBUG "instance": "10.142.0.107:9100" 05:01:55 DEBUG }, 05:01:55 DEBUG "value": [ 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG [...] 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG "instance": "10.142.0.43:9100" 05:01:55 DEBUG }, 05:01:55 DEBUG "value": [ 05:01:55 DEBUG 1713499315, 05:01:55 DEBUG "0.038333333333332574" 05:01:55 DEBUG ] 05:01:55 DEBUG } 05:01:55 DEBUG ] 05:01:55 DEBUG } 05:01:55 DEBUG } 05:01:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499315" 05:01:55 INFO [http_cmd]: http status code OK 05:01:55 DEBUG --- status code --- 05:01:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:01:55 DEBUG --- http response --- 05:01:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:01:55 DEBUG ----- output ----- 05:01:55 DEBUG { 05:01:55 DEBUG "status": "success", 05:01:55 DEBUG "data": { 05:01:55 DEBUG "resultType": "vector", 05:01:55 DEBUG "result": [ 05:01:55 DEBUG { 05:01:55 DEBUG "metric": { 05:01:55 DEBUG "instance": "10.142.0.107:9100" 05:01:55 DEBUG }, 05:01:55 DEBUG "value": [ 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG [...] 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG 05:01:55 DEBUG "instance": "10.142.0.43:9100" 05:01:55 DEBUG }, 05:01:55 DEBUG "value": [ 05:01:55 DEBUG 1713499315, 05:01:55 DEBUG "47663913.166666664" 05:01:55 DEBUG ] 05:01:55 DEBUG } 05:01:55 DEBUG ] 05:01:55 DEBUG } 05:01:55 DEBUG } 05:01:55 INFO 05:01:55 INFO [loop_until]: kubectl --namespace=xlou top pods 05:01:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:01:55 INFO [loop_until]: OK (rc = 0) 05:01:55 DEBUG --- stdout --- 05:01:55 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 10104m 8963Mi am-6fd554f8f7-hvvkw 9261m 8874Mi am-6fd554f8f7-pmj4b 10405m 8973Mi ds-cts-0 7365m 12225Mi ds-cts-1 7451m 12142Mi ds-cts-2 7579m 12184Mi ds-idrepo-0 4676m 3255Mi ds-idrepo-1 4660m 3207Mi ds-idrepo-2 4572m 3066Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 4m 1216Mi lodemon-6b89d774d5-rf7sk 9m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 6487m 1970Mi 05:01:55 DEBUG --- stderr --- 05:01:55 DEBUG 05:01:55 INFO 05:01:55 INFO [loop_until]: kubectl --namespace=xlou top node 05:01:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:01:55 INFO [loop_until]: OK (rc = 0) 05:01:55 DEBUG --- stdout --- 05:01:55 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 79m 0% 6260Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 4623m 15% 6699Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9311m 31% 11597Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 3250Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 7490m 25% 14794Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 162m 0% 5154Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 9712m 32% 11747Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 10308m 34% 2720Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 11039m 36% 11175Mi 10% gke-xlou-cdm-ds-0f601751-j488 4852m 16% 6150Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 7421m 24% 16179Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 5071m 16% 6874Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 7685m 25% 14166Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 6651m 41% 5720Mi 9% 05:01:55 DEBUG --- stderr --- 05:01:55 DEBUG 05:02:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499375" 05:02:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713499375" 05:02:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499375" 05:02:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713499375" 05:02:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499375" 05:02:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499375" 05:02:55 INFO [http_cmd]: http status code OK 05:02:55 DEBUG --- status code --- 05:02:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:02:55 DEBUG --- http response --- 05:02:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 05:02:55 DEBUG ----- output ----- 05:02:55 DEBUG { 05:02:55 DEBUG "status": "success", 05:02:55 DEBUG "data": { 05:02:55 DEBUG "resultType": "vector", 05:02:55 DEBUG "result": [ 05:02:55 DEBUG { 05:02:55 DEBUG "metric": { 05:02:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:02:55 DEBUG }, 05:02:55 DEBUG "value": [ 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG [...] 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG "pod": "idm-b85f9d978-wnhxb" 05:02:55 DEBUG }, 05:02:55 DEBUG "value": [ 05:02:55 DEBUG 1713499375, 05:02:55 DEBUG "1439.0235642433818" 05:02:55 DEBUG ] 05:02:55 DEBUG } 05:02:55 DEBUG ] 05:02:55 DEBUG } 05:02:55 DEBUG } 05:02:55 ERROR [http_cmd]: ERROR 05:02:55 ERROR --- status code --- 05:02:55 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:02:55 ERROR --- http response --- 05:02:55 INFO [http_cmd]: http status code OK 05:02:55 INFO [http_cmd]: http status code OK 05:02:55 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:02:55 DEBUG --- status code --- 05:02:55 DEBUG --- status code --- 05:02:55 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:02:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:02:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:02:55 DEBUG --- http response --- 05:02:55 DEBUG --- http response --- 05:02:55 INFO [http_cmd]: http status code OK 05:02:55 INFO [http_cmd]: http status code OK 05:02:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:02:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:02:55 DEBUG --- status code --- 05:02:55 DEBUG --- status code --- 05:02:55 DEBUG ----- output ----- 05:02:55 DEBUG ----- output ----- 05:02:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:02:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:02:55 DEBUG { 05:02:55 DEBUG { 05:02:55 DEBUG --- http response --- 05:02:55 DEBUG --- http response --- 05:02:55 DEBUG "status": "success", 05:02:55 DEBUG "status": "success", 05:02:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:02:55 DEBUG "data": { 05:02:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:02:55 DEBUG "data": { 05:02:55 DEBUG ----- output ----- 05:02:55 DEBUG "resultType": "vector", 05:02:55 DEBUG ----- output ----- 05:02:55 DEBUG "resultType": "vector", 05:02:55 DEBUG { 05:02:55 DEBUG "result": [ 05:02:55 DEBUG { 05:02:55 DEBUG "result": [ 05:02:55 DEBUG "status": "success", 05:02:55 DEBUG { 05:02:55 DEBUG "status": "success", 05:02:55 DEBUG { 05:02:55 DEBUG "data": { 05:02:55 DEBUG "metric": { 05:02:55 DEBUG "data": { 05:02:55 DEBUG "metric": { 05:02:55 DEBUG "resultType": "vector", 05:02:55 DEBUG "pod": "ds-cts-0" 05:02:55 DEBUG "resultType": "vector", 05:02:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:02:55 DEBUG "result": [ 05:02:55 DEBUG }, 05:02:55 DEBUG "result": [ 05:02:55 DEBUG }, 05:02:55 DEBUG { 05:02:55 DEBUG "value": [ 05:02:55 DEBUG { 05:02:55 DEBUG "value": [ 05:02:55 DEBUG "metric": { 05:02:55 DEBUG 05:02:55 DEBUG "metric": { 05:02:55 DEBUG 05:02:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:02:55 DEBUG 05:02:55 DEBUG "pod": "ds-idrepo-0" 05:02:55 DEBUG 05:02:55 DEBUG }, 05:02:55 DEBUG 05:02:55 DEBUG }, 05:02:55 DEBUG 05:02:55 DEBUG "value": [ 05:02:55 DEBUG [...] 05:02:55 DEBUG "value": [ 05:02:55 DEBUG [...] 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG [...] 05:02:55 DEBUG "pod": "ds-cts-2" 05:02:55 DEBUG [...] 05:02:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:02:55 DEBUG 05:02:55 DEBUG }, 05:02:55 DEBUG 05:02:55 DEBUG }, 05:02:55 DEBUG 05:02:55 DEBUG "value": [ 05:02:55 DEBUG 05:02:55 DEBUG "value": [ 05:02:55 DEBUG 05:02:55 DEBUG 1713499375, 05:02:55 DEBUG 05:02:55 DEBUG 1713499375, 05:02:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:02:55 DEBUG "0.1" 05:02:55 DEBUG "pod": "ds-idrepo-2" 05:02:55 DEBUG "0" 05:02:55 DEBUG }, 05:02:55 DEBUG ] 05:02:55 DEBUG }, 05:02:55 DEBUG ] 05:02:55 DEBUG "value": [ 05:02:55 DEBUG } 05:02:55 DEBUG "value": [ 05:02:55 DEBUG } 05:02:55 DEBUG 1713499375, 05:02:55 DEBUG ] 05:02:55 DEBUG 1713499375, 05:02:55 DEBUG ] 05:02:55 DEBUG "6.521439362725355" 05:02:55 DEBUG } 05:02:55 DEBUG "0" 05:02:55 DEBUG } 05:02:55 DEBUG ] 05:02:55 DEBUG } 05:02:55 DEBUG ] 05:02:55 DEBUG } 05:02:55 DEBUG } 05:02:55 DEBUG } 05:02:55 DEBUG ] 05:02:55 DEBUG ] 05:02:55 DEBUG } 05:02:55 DEBUG } 05:02:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499375" 05:02:55 DEBUG } 05:02:55 DEBUG } 05:02:55 INFO [http_cmd]: http status code OK 05:02:55 DEBUG --- status code --- 05:02:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:02:55 DEBUG --- http response --- 05:02:55 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:02:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713499375" 05:02:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499375" 05:02:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499375" 05:02:55 INFO [http_cmd]: http status code OK 05:02:55 DEBUG --- status code --- 05:02:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:02:55 DEBUG --- http response --- 05:02:55 INFO [http_cmd]: http status code OK 05:02:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 05:02:55 DEBUG --- status code --- 05:02:55 DEBUG ----- output ----- 05:02:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:02:55 DEBUG { 05:02:55 DEBUG --- http response --- 05:02:55 DEBUG "status": "success", 05:02:55 DEBUG "data": { 05:02:55 DEBUG "resultType": "vector", 05:02:55 DEBUG "result": [ 05:02:55 DEBUG { 05:02:55 DEBUG "metric": { 05:02:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:02:55 DEBUG }, 05:02:55 DEBUG "value": [ 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG [...] 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG "pod": "idm-b85f9d978-wnhxb" 05:02:55 DEBUG }, 05:02:55 DEBUG "value": [ 05:02:55 DEBUG 1713499375, 05:02:55 DEBUG "48001869.140093096" 05:02:55 DEBUG ] 05:02:55 DEBUG } 05:02:55 DEBUG ] 05:02:55 DEBUG } 05:02:55 DEBUG } 05:02:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:02:55 DEBUG ----- output ----- 05:02:55 DEBUG { 05:02:55 DEBUG "status": "success", 05:02:55 DEBUG "data": { 05:02:55 DEBUG "resultType": "vector", 05:02:55 DEBUG "result": [ 05:02:55 DEBUG { 05:02:55 DEBUG "metric": { 05:02:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:02:55 DEBUG }, 05:02:55 DEBUG "value": [ 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG [...] 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:02:55 DEBUG }, 05:02:55 DEBUG "value": [ 05:02:55 DEBUG 1713499375, 05:02:55 DEBUG "2164711424" 05:02:55 DEBUG ] 05:02:55 DEBUG } 05:02:55 DEBUG ] 05:02:55 DEBUG } 05:02:55 DEBUG } 05:02:55 INFO [http_cmd]: http status code OK 05:02:55 DEBUG --- status code --- 05:02:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:02:55 DEBUG --- http response --- 05:02:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:02:55 DEBUG ----- output ----- 05:02:55 DEBUG { 05:02:55 DEBUG "status": "success", 05:02:55 DEBUG "data": { 05:02:55 DEBUG "resultType": "vector", 05:02:55 DEBUG "result": [ 05:02:55 DEBUG { 05:02:55 DEBUG "metric": { 05:02:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:02:55 DEBUG }, 05:02:55 DEBUG "value": [ 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG [...] 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:02:55 DEBUG }, 05:02:55 DEBUG "value": [ 05:02:55 DEBUG 1713499375, 05:02:55 DEBUG "2001.6666666666667" 05:02:55 DEBUG ] 05:02:55 DEBUG } 05:02:55 DEBUG ] 05:02:55 DEBUG } 05:02:55 DEBUG } 05:02:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499375" 05:02:55 INFO [http_cmd]: http status code OK 05:02:55 DEBUG --- status code --- 05:02:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:02:55 DEBUG --- http response --- 05:02:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:02:55 DEBUG ----- output ----- 05:02:55 DEBUG { 05:02:55 DEBUG "status": "success", 05:02:55 DEBUG "data": { 05:02:55 DEBUG "resultType": "vector", 05:02:55 DEBUG "result": [ 05:02:55 DEBUG { 05:02:55 DEBUG "metric": { 05:02:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:02:55 DEBUG }, 05:02:55 DEBUG "value": [ 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG [...] 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:02:55 DEBUG }, 05:02:55 DEBUG "value": [ 05:02:55 DEBUG 1713499375, 05:02:55 DEBUG "2001.6" 05:02:55 DEBUG ] 05:02:55 DEBUG } 05:02:55 DEBUG ] 05:02:55 DEBUG } 05:02:55 DEBUG } 05:02:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499375" 05:02:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713499375" 05:02:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499375" 05:02:55 INFO [http_cmd]: http status code OK 05:02:55 DEBUG --- status code --- 05:02:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:02:55 DEBUG --- http response --- 05:02:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 05:02:55 DEBUG ----- output ----- 05:02:55 DEBUG { 05:02:55 DEBUG "status": "success", 05:02:55 DEBUG "data": { 05:02:55 DEBUG "resultType": "vector", 05:02:55 DEBUG "result": [ 05:02:55 DEBUG { 05:02:55 DEBUG "metric": { 05:02:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:02:55 DEBUG }, 05:02:55 DEBUG "value": [ 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG [...] 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG "pod": "idm-b85f9d978-wnhxb" 05:02:55 DEBUG }, 05:02:55 DEBUG "value": [ 05:02:55 DEBUG 1713499375, 05:02:55 DEBUG "0" 05:02:55 DEBUG ] 05:02:55 DEBUG } 05:02:55 DEBUG ] 05:02:55 DEBUG } 05:02:55 DEBUG } 05:02:55 INFO [http_cmd]: http status code OK 05:02:55 DEBUG --- status code --- 05:02:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:02:55 DEBUG --- http response --- 05:02:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:02:55 DEBUG ----- output ----- 05:02:55 DEBUG { 05:02:55 DEBUG "status": "success", 05:02:55 DEBUG "data": { 05:02:55 DEBUG "resultType": "vector", 05:02:55 DEBUG "result": [ 05:02:55 DEBUG { 05:02:55 DEBUG "metric": { 05:02:55 DEBUG "pod": "ds-cts-0" 05:02:55 DEBUG }, 05:02:55 DEBUG "value": [ 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG [...] 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG "pod": "ds-cts-2" 05:02:55 DEBUG }, 05:02:55 DEBUG "value": [ 05:02:55 DEBUG 1713499375, 05:02:55 DEBUG "0" 05:02:55 DEBUG ] 05:02:55 DEBUG } 05:02:55 DEBUG ] 05:02:55 DEBUG } 05:02:55 DEBUG } 05:02:55 INFO [http_cmd]: http status code OK 05:02:55 DEBUG --- status code --- 05:02:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:02:55 DEBUG --- http response --- 05:02:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:02:55 DEBUG ----- output ----- 05:02:55 DEBUG { 05:02:55 DEBUG "status": "success", 05:02:55 DEBUG "data": { 05:02:55 DEBUG "resultType": "vector", 05:02:55 DEBUG "result": [ 05:02:55 DEBUG { 05:02:55 DEBUG "metric": { 05:02:55 DEBUG "pod": "ds-idrepo-0" 05:02:55 DEBUG }, 05:02:55 DEBUG "value": [ 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG [...] 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG "pod": "ds-cts-2" 05:02:55 DEBUG }, 05:02:55 DEBUG "value": [ 05:02:55 DEBUG 1713499375, 05:02:55 DEBUG "0" 05:02:55 DEBUG ] 05:02:55 DEBUG } 05:02:55 DEBUG ] 05:02:55 DEBUG } 05:02:55 DEBUG } 05:02:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499375" 05:02:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499375" 05:02:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499375" 05:02:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499375" 05:02:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499375" 05:02:55 INFO [http_cmd]: http status code OK 05:02:55 DEBUG --- status code --- 05:02:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:02:55 DEBUG --- http response --- 05:02:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:02:55 DEBUG ----- output ----- 05:02:55 DEBUG { 05:02:55 DEBUG "status": "success", 05:02:55 DEBUG "data": { 05:02:55 DEBUG "resultType": "vector", 05:02:55 DEBUG "result": [ 05:02:55 DEBUG { 05:02:55 DEBUG "metric": { 05:02:55 DEBUG "pod": "ds-cts-0" 05:02:55 DEBUG }, 05:02:55 DEBUG "value": [ 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG [...] 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG "pod": "ds-cts-2" 05:02:55 DEBUG }, 05:02:55 DEBUG "value": [ 05:02:55 DEBUG 1713499375, 05:02:55 DEBUG "0.000017660000000000004" 05:02:55 DEBUG ] 05:02:55 DEBUG } 05:02:55 DEBUG ] 05:02:55 DEBUG } 05:02:55 DEBUG } 05:02:55 INFO [http_cmd]: http status code OK 05:02:55 DEBUG --- status code --- 05:02:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:02:55 DEBUG --- http response --- 05:02:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:02:55 DEBUG ----- output ----- 05:02:55 DEBUG { 05:02:55 DEBUG "status": "success", 05:02:55 DEBUG "data": { 05:02:55 DEBUG "resultType": "vector", 05:02:55 DEBUG "result": [ 05:02:55 DEBUG { 05:02:55 DEBUG "metric": { 05:02:55 DEBUG "instance": "10.142.0.107:9100" 05:02:55 DEBUG }, 05:02:55 DEBUG "value": [ 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG [...] 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG "instance": "10.142.0.43:9100" 05:02:55 DEBUG }, 05:02:55 DEBUG "value": [ 05:02:55 DEBUG 1713499375, 05:02:55 DEBUG "427657919488" 05:02:55 DEBUG ] 05:02:55 DEBUG } 05:02:55 DEBUG ] 05:02:55 DEBUG } 05:02:55 DEBUG } 05:02:55 INFO [http_cmd]: http status code OK 05:02:55 DEBUG --- status code --- 05:02:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:02:55 DEBUG --- http response --- 05:02:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:02:55 DEBUG ----- output ----- 05:02:55 DEBUG { 05:02:55 DEBUG "status": "success", 05:02:55 DEBUG "data": { 05:02:55 DEBUG "resultType": "vector", 05:02:55 DEBUG "result": [ 05:02:55 DEBUG { 05:02:55 DEBUG "metric": { 05:02:55 DEBUG "instance": "10.142.0.107:9100" 05:02:55 DEBUG }, 05:02:55 DEBUG "value": [ 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG [...] 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG "instance": "10.142.0.43:9100" 05:02:55 DEBUG }, 05:02:55 DEBUG "value": [ 05:02:55 DEBUG 1713499375, 05:02:55 DEBUG "47714977.86666667" 05:02:55 DEBUG ] 05:02:55 DEBUG } 05:02:55 DEBUG ] 05:02:55 DEBUG } 05:02:55 DEBUG } 05:02:55 INFO [http_cmd]: http status code OK 05:02:55 DEBUG --- status code --- 05:02:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:02:55 DEBUG --- http response --- 05:02:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:02:55 DEBUG ----- output ----- 05:02:55 DEBUG { 05:02:55 DEBUG "status": "success", 05:02:55 DEBUG "data": { 05:02:55 DEBUG "resultType": "vector", 05:02:55 DEBUG "result": [ 05:02:55 DEBUG { 05:02:55 DEBUG "metric": { 05:02:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:02:55 DEBUG }, 05:02:55 DEBUG "value": [ 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG [...] 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:02:55 DEBUG }, 05:02:55 DEBUG "value": [ 05:02:55 DEBUG 1713499375, 05:02:55 DEBUG "0" 05:02:55 DEBUG ] 05:02:55 DEBUG } 05:02:55 DEBUG ] 05:02:55 DEBUG } 05:02:55 DEBUG } 05:02:55 INFO [http_cmd]: http status code OK 05:02:55 DEBUG --- status code --- 05:02:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:02:55 DEBUG --- http response --- 05:02:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:02:55 DEBUG ----- output ----- 05:02:55 DEBUG { 05:02:55 DEBUG "status": "success", 05:02:55 DEBUG "data": { 05:02:55 DEBUG "resultType": "vector", 05:02:55 DEBUG "result": [ 05:02:55 DEBUG { 05:02:55 DEBUG "metric": { 05:02:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:02:55 DEBUG }, 05:02:55 DEBUG "value": [ 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG [...] 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:02:55 DEBUG }, 05:02:55 DEBUG "value": [ 05:02:55 DEBUG 1713499375, 05:02:55 DEBUG "2141902" 05:02:55 DEBUG ] 05:02:55 DEBUG } 05:02:55 DEBUG ] 05:02:55 DEBUG } 05:02:55 DEBUG } 05:02:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499375" 05:02:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499375" 05:02:55 INFO [http_cmd]: http status code OK 05:02:55 DEBUG --- status code --- 05:02:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:02:55 DEBUG --- http response --- 05:02:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 05:02:55 DEBUG ----- output ----- 05:02:55 DEBUG { 05:02:55 DEBUG "status": "success", 05:02:55 DEBUG "data": { 05:02:55 DEBUG "resultType": "vector", 05:02:55 DEBUG "result": [ 05:02:55 DEBUG { 05:02:55 DEBUG "metric": { 05:02:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:02:55 DEBUG }, 05:02:55 DEBUG "value": [ 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG [...] 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG "pod": "idm-b85f9d978-wnhxb" 05:02:55 DEBUG }, 05:02:55 DEBUG "value": [ 05:02:55 DEBUG 1713499375, 05:02:55 DEBUG "0.006014097421196811" 05:02:55 DEBUG ] 05:02:55 DEBUG } 05:02:55 DEBUG ] 05:02:55 DEBUG } 05:02:55 DEBUG } 05:02:55 INFO [http_cmd]: http status code OK 05:02:55 DEBUG --- status code --- 05:02:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:02:55 DEBUG --- http response --- 05:02:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:02:55 DEBUG ----- output ----- 05:02:55 DEBUG { 05:02:55 DEBUG "status": "success", 05:02:55 DEBUG "data": { 05:02:55 DEBUG "resultType": "vector", 05:02:55 DEBUG "result": [ 05:02:55 DEBUG { 05:02:55 DEBUG "metric": { 05:02:55 DEBUG "instance": "10.142.0.107:9100" 05:02:55 DEBUG }, 05:02:55 DEBUG "value": [ 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG [...] 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG "instance": "10.142.0.43:9100" 05:02:55 DEBUG }, 05:02:55 DEBUG "value": [ 05:02:55 DEBUG 1713499375, 05:02:55 DEBUG "0.05650000000001019" 05:02:55 DEBUG ] 05:02:55 DEBUG } 05:02:55 DEBUG ] 05:02:55 DEBUG } 05:02:55 DEBUG } 05:02:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499375" 05:02:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499375" 05:02:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499375" 05:02:55 INFO [http_cmd]: http status code OK 05:02:55 DEBUG --- status code --- 05:02:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:02:55 DEBUG --- http response --- 05:02:55 INFO [http_cmd]: http status code OK 05:02:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:02:55 DEBUG --- status code --- 05:02:55 DEBUG ----- output ----- 05:02:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:02:55 DEBUG { 05:02:55 DEBUG --- http response --- 05:02:55 DEBUG "status": "success", 05:02:55 DEBUG "data": { 05:02:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:02:55 DEBUG "resultType": "vector", 05:02:55 DEBUG ----- output ----- 05:02:55 DEBUG "result": [ 05:02:55 DEBUG { 05:02:55 DEBUG { 05:02:55 DEBUG "status": "success", 05:02:55 DEBUG "metric": { 05:02:55 DEBUG "data": { 05:02:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:02:55 DEBUG "resultType": "vector", 05:02:55 DEBUG }, 05:02:55 DEBUG "result": [ 05:02:55 DEBUG "value": [ 05:02:55 DEBUG { 05:02:55 DEBUG 05:02:55 DEBUG "metric": { 05:02:55 DEBUG 05:02:55 DEBUG "pod": "ds-idrepo-0" 05:02:55 DEBUG 05:02:55 DEBUG }, 05:02:55 DEBUG [...] 05:02:55 DEBUG "value": [ 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:02:55 DEBUG [...] 05:02:55 DEBUG }, 05:02:55 DEBUG 05:02:55 DEBUG "value": [ 05:02:55 DEBUG 05:02:55 DEBUG 1713499375, 05:02:55 DEBUG 05:02:55 DEBUG "2001.3999999999999" 05:02:55 DEBUG "pod": "ds-idrepo-2" 05:02:55 DEBUG ] 05:02:55 DEBUG }, 05:02:55 DEBUG } 05:02:55 DEBUG "value": [ 05:02:55 DEBUG ] 05:02:55 DEBUG 1713499375, 05:02:55 DEBUG } 05:02:55 DEBUG "0" 05:02:55 DEBUG } 05:02:55 DEBUG ] 05:02:55 DEBUG } 05:02:55 DEBUG ] 05:02:55 DEBUG } 05:02:55 DEBUG } 05:02:55 INFO [http_cmd]: http status code OK 05:02:55 DEBUG --- status code --- 05:02:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:02:55 DEBUG --- http response --- 05:02:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713499375" 05:02:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:02:55 DEBUG ----- output ----- 05:02:55 DEBUG { 05:02:55 DEBUG "status": "success", 05:02:55 DEBUG "data": { 05:02:55 DEBUG "resultType": "vector", 05:02:55 DEBUG "result": [ 05:02:55 DEBUG { 05:02:55 DEBUG "metric": { 05:02:55 DEBUG "pod": "ds-idrepo-0" 05:02:55 DEBUG }, 05:02:55 DEBUG "value": [ 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG [...] 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG "pod": "ds-idrepo-2" 05:02:55 DEBUG }, 05:02:55 DEBUG "value": [ 05:02:55 DEBUG 1713499375, 05:02:55 DEBUG "0" 05:02:55 DEBUG ] 05:02:55 DEBUG } 05:02:55 DEBUG ] 05:02:55 DEBUG } 05:02:55 DEBUG } 05:02:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499375" 05:02:55 INFO [http_cmd]: http status code OK 05:02:55 DEBUG --- status code --- 05:02:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:02:55 DEBUG --- http response --- 05:02:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:02:55 DEBUG ----- output ----- 05:02:55 DEBUG { 05:02:55 DEBUG "status": "success", 05:02:55 DEBUG "data": { 05:02:55 DEBUG "resultType": "vector", 05:02:55 DEBUG "result": [ 05:02:55 DEBUG { 05:02:55 DEBUG "metric": { 05:02:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:02:55 DEBUG }, 05:02:55 DEBUG "value": [ 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG [...] 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:02:55 DEBUG }, 05:02:55 DEBUG "value": [ 05:02:55 DEBUG 1713499375, 05:02:55 DEBUG "55.171" 05:02:55 DEBUG ] 05:02:55 DEBUG } 05:02:55 DEBUG ] 05:02:55 DEBUG } 05:02:55 DEBUG } 05:02:55 INFO [http_cmd]: http status code OK 05:02:55 DEBUG --- status code --- 05:02:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:02:55 DEBUG --- http response --- 05:02:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:02:55 DEBUG ----- output ----- 05:02:55 DEBUG { 05:02:55 DEBUG "status": "success", 05:02:55 DEBUG "data": { 05:02:55 DEBUG "resultType": "vector", 05:02:55 DEBUG "result": [ 05:02:55 DEBUG { 05:02:55 DEBUG "metric": { 05:02:55 DEBUG "instance": "10.142.0.107:9100" 05:02:55 DEBUG }, 05:02:55 DEBUG "value": [ 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG [...] 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG "instance": "10.142.0.43:9100" 05:02:55 DEBUG }, 05:02:55 DEBUG "value": [ 05:02:55 DEBUG 1713499375, 05:02:55 DEBUG "0.03833333333333399" 05:02:55 DEBUG ] 05:02:55 DEBUG } 05:02:55 DEBUG ] 05:02:55 DEBUG } 05:02:55 DEBUG } 05:02:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499375" 05:02:55 INFO [http_cmd]: http status code OK 05:02:55 DEBUG --- status code --- 05:02:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:02:55 DEBUG --- http response --- 05:02:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:02:55 DEBUG ----- output ----- 05:02:55 DEBUG { 05:02:55 DEBUG "status": "success", 05:02:55 DEBUG "data": { 05:02:55 DEBUG "resultType": "vector", 05:02:55 DEBUG "result": [ 05:02:55 DEBUG { 05:02:55 DEBUG "metric": { 05:02:55 DEBUG "instance": "10.142.0.107:9100" 05:02:55 DEBUG }, 05:02:55 DEBUG "value": [ 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG [...] 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG 05:02:55 DEBUG "instance": "10.142.0.43:9100" 05:02:55 DEBUG }, 05:02:55 DEBUG "value": [ 05:02:55 DEBUG 1713499375, 05:02:55 DEBUG "47713086.4" 05:02:55 DEBUG ] 05:02:55 DEBUG } 05:02:55 DEBUG ] 05:02:55 DEBUG } 05:02:55 DEBUG } 05:02:55 INFO 05:02:55 INFO [loop_until]: kubectl --namespace=xlou top pods 05:02:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:02:55 INFO [loop_until]: OK (rc = 0) 05:02:55 DEBUG --- stdout --- 05:02:55 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 9794m 8964Mi am-6fd554f8f7-hvvkw 9677m 8874Mi am-6fd554f8f7-pmj4b 10057m 8974Mi ds-cts-0 7647m 12230Mi ds-cts-1 7334m 12151Mi ds-cts-2 8014m 12189Mi ds-idrepo-0 4755m 3255Mi ds-idrepo-1 4688m 3206Mi ds-idrepo-2 4520m 3065Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 4m 1215Mi lodemon-6b89d774d5-rf7sk 4m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 6488m 1972Mi 05:02:55 DEBUG --- stderr --- 05:02:55 DEBUG 05:02:56 INFO 05:02:56 INFO [loop_until]: kubectl --namespace=xlou top node 05:02:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:02:56 INFO [loop_until]: OK (rc = 0) 05:02:56 DEBUG --- stdout --- 05:02:56 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 79m 0% 6264Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 4613m 15% 6699Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9427m 31% 11594Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 86m 0% 3265Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 7628m 25% 14790Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 144m 0% 5174Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 9880m 33% 11759Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 10301m 34% 2724Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 10628m 35% 11179Mi 10% gke-xlou-cdm-ds-0f601751-j488 4851m 16% 6156Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 7546m 25% 16178Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 5066m 16% 6876Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 7725m 25% 14176Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 6665m 41% 5719Mi 9% 05:02:56 DEBUG --- stderr --- 05:02:56 DEBUG 05:03:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713499435" 05:03:55 ERROR [http_cmd]: ERROR 05:03:55 ERROR --- status code --- 05:03:55 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:03:55 ERROR --- http response --- 05:03:55 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:03:55 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:03:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499435" 05:03:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499435" 05:03:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713499435" 05:03:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499435" 05:03:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499435" 05:03:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499435" 05:03:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499435" 05:03:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713499435" 05:03:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499435" 05:03:55 INFO [http_cmd]: http status code OK 05:03:55 INFO [http_cmd]: http status code OK 05:03:55 INFO [http_cmd]: http status code OK 05:03:55 INFO [http_cmd]: http status code OK 05:03:55 INFO [http_cmd]: http status code OK 05:03:55 DEBUG --- status code --- 05:03:55 INFO [http_cmd]: http status code OK 05:03:55 DEBUG --- status code --- 05:03:55 DEBUG --- status code --- 05:03:55 DEBUG --- status code --- 05:03:55 DEBUG --- status code --- 05:03:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:03:55 DEBUG --- status code --- 05:03:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:03:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:03:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:03:55 DEBUG --- http response --- 05:03:55 DEBUG --- http response --- 05:03:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:03:55 DEBUG --- http response --- 05:03:55 DEBUG --- http response --- 05:03:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:03:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:03:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:03:55 DEBUG --- http response --- 05:03:55 INFO [http_cmd]: http status code OK 05:03:55 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:03:55 DEBUG --- http response --- 05:03:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 05:03:55 DEBUG ----- output ----- 05:03:55 DEBUG ----- output ----- 05:03:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:03:55 DEBUG --- status code --- 05:03:55 DEBUG ----- output ----- 05:03:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:03:55 DEBUG { 05:03:55 INFO [http_cmd]: http status code OK 05:03:55 DEBUG { 05:03:55 DEBUG ----- output ----- 05:03:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:03:55 DEBUG { 05:03:55 DEBUG ----- output ----- 05:03:55 DEBUG "status": "success", 05:03:55 DEBUG --- status code --- 05:03:55 DEBUG "status": "success", 05:03:55 DEBUG { 05:03:55 DEBUG --- http response --- 05:03:55 DEBUG "status": "success", 05:03:55 DEBUG { 05:03:55 DEBUG "data": { 05:03:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:03:55 DEBUG "data": { 05:03:55 DEBUG "status": "success", 05:03:55 DEBUG "data": { 05:03:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:03:55 DEBUG "status": "success", 05:03:55 DEBUG "resultType": "vector", 05:03:55 DEBUG --- http response --- 05:03:55 DEBUG "resultType": "vector", 05:03:55 DEBUG "data": { 05:03:55 DEBUG "resultType": "vector", 05:03:55 DEBUG ----- output ----- 05:03:55 DEBUG "data": { 05:03:55 DEBUG "result": [ 05:03:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:03:55 DEBUG "result": [ 05:03:55 DEBUG "resultType": "vector", 05:03:55 DEBUG "result": [ 05:03:55 DEBUG { 05:03:55 DEBUG "resultType": "vector", 05:03:55 DEBUG { 05:03:55 DEBUG ----- output ----- 05:03:55 DEBUG { 05:03:55 DEBUG "result": [ 05:03:55 DEBUG { 05:03:55 DEBUG "status": "success", 05:03:55 DEBUG "result": [ 05:03:55 DEBUG "metric": { 05:03:55 DEBUG { 05:03:55 DEBUG "metric": { 05:03:55 DEBUG { 05:03:55 DEBUG "metric": { 05:03:55 DEBUG "data": { 05:03:55 DEBUG { 05:03:55 DEBUG "pod": "ds-idrepo-1" 05:03:55 DEBUG "status": "success", 05:03:55 DEBUG "pod": "ds-idrepo-0" 05:03:55 DEBUG "metric": { 05:03:55 DEBUG "pod": "ds-idrepo-1" 05:03:55 DEBUG "resultType": "vector", 05:03:55 DEBUG "metric": { 05:03:55 DEBUG }, 05:03:55 DEBUG "data": { 05:03:55 DEBUG }, 05:03:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:03:55 DEBUG }, 05:03:55 DEBUG "result": [ 05:03:55 DEBUG "pod": "ds-cts-0" 05:03:55 DEBUG "value": [ 05:03:55 DEBUG "resultType": "vector", 05:03:55 DEBUG "value": [ 05:03:55 DEBUG }, 05:03:55 DEBUG "value": [ 05:03:55 DEBUG { 05:03:55 DEBUG }, 05:03:55 DEBUG 05:03:55 DEBUG "result": [ 05:03:55 DEBUG 05:03:55 DEBUG "value": [ 05:03:55 DEBUG 05:03:55 DEBUG "metric": { 05:03:55 DEBUG "value": [ 05:03:55 DEBUG 05:03:55 DEBUG { 05:03:55 INFO [http_cmd]: http status code OK 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG "metric": { 05:03:55 DEBUG --- status code --- 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG }, 05:03:55 DEBUG 05:03:55 DEBUG [...] 05:03:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:03:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:03:55 DEBUG [...] 05:03:55 DEBUG 05:03:55 DEBUG [...] 05:03:55 DEBUG "value": [ 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG }, 05:03:55 DEBUG --- http response --- 05:03:55 DEBUG 05:03:55 DEBUG [...] 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG [...] 05:03:55 DEBUG 05:03:55 DEBUG "value": [ 05:03:55 DEBUG 05:03:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG ----- output ----- 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:03:55 DEBUG 05:03:55 DEBUG "pod": "ds-idrepo-2" 05:03:55 DEBUG { 05:03:55 DEBUG 05:03:55 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:03:55 DEBUG [...] 05:03:55 DEBUG 05:03:55 DEBUG }, 05:03:55 DEBUG 05:03:55 DEBUG }, 05:03:55 DEBUG "status": "success", 05:03:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:03:55 DEBUG }, 05:03:55 DEBUG 05:03:55 DEBUG "pod": "ds-cts-2" 05:03:55 DEBUG "value": [ 05:03:55 DEBUG [...] 05:03:55 DEBUG "value": [ 05:03:55 DEBUG "data": { 05:03:55 DEBUG }, 05:03:55 DEBUG "value": [ 05:03:55 DEBUG 05:03:55 DEBUG }, 05:03:55 DEBUG 1713499435, 05:03:55 DEBUG 05:03:55 DEBUG 1713499435, 05:03:55 DEBUG "resultType": "vector", 05:03:55 DEBUG "value": [ 05:03:55 DEBUG 1713499435, 05:03:55 DEBUG 05:03:55 DEBUG "value": [ 05:03:55 DEBUG "3671.5442988840364" 05:03:55 DEBUG 05:03:55 DEBUG "0" 05:03:55 DEBUG "result": [ 05:03:55 DEBUG 1713499435, 05:03:55 DEBUG "0" 05:03:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:03:55 DEBUG 1713499435, 05:03:55 DEBUG ] 05:03:55 DEBUG 05:03:55 DEBUG ] 05:03:55 DEBUG { 05:03:55 DEBUG "2167107584" 05:03:55 DEBUG ] 05:03:55 DEBUG }, 05:03:55 DEBUG "0.3333333333333333" 05:03:55 DEBUG } 05:03:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:03:55 DEBUG } 05:03:55 DEBUG "metric": { 05:03:55 DEBUG ] 05:03:55 DEBUG } 05:03:55 DEBUG "value": [ 05:03:55 DEBUG ] 05:03:55 DEBUG ] 05:03:55 DEBUG }, 05:03:55 DEBUG ] 05:03:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:03:55 DEBUG } 05:03:55 DEBUG ] 05:03:55 DEBUG 1713499435, 05:03:55 DEBUG } 05:03:55 DEBUG } 05:03:55 DEBUG "value": [ 05:03:55 DEBUG } 05:03:55 DEBUG }, 05:03:55 DEBUG ] 05:03:55 DEBUG } 05:03:55 DEBUG "0" 05:03:55 DEBUG ] 05:03:55 DEBUG } 05:03:55 DEBUG 1713499435, 05:03:55 DEBUG } 05:03:55 DEBUG "value": [ 05:03:55 DEBUG } 05:03:55 DEBUG } 05:03:55 DEBUG ] 05:03:55 DEBUG } 05:03:55 DEBUG "6.495727729833706" 05:03:55 DEBUG 05:03:55 DEBUG } 05:03:55 DEBUG } 05:03:55 DEBUG } 05:03:55 DEBUG ] 05:03:55 DEBUG 05:03:55 DEBUG ] 05:03:55 DEBUG } 05:03:55 DEBUG 05:03:55 DEBUG } 05:03:55 DEBUG ] 05:03:55 DEBUG [...] 05:03:55 DEBUG } 05:03:55 DEBUG } 05:03:55 DEBUG 05:03:55 DEBUG } 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:03:55 DEBUG }, 05:03:55 DEBUG "value": [ 05:03:55 DEBUG 1713499435, 05:03:55 DEBUG "2004.2333333333333" 05:03:55 DEBUG ] 05:03:55 DEBUG } 05:03:55 DEBUG ] 05:03:55 DEBUG } 05:03:55 DEBUG } 05:03:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499435" 05:03:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713499435" 05:03:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499435" 05:03:55 INFO [http_cmd]: http status code OK 05:03:55 INFO [http_cmd]: http status code OK 05:03:55 DEBUG --- status code --- 05:03:55 DEBUG --- status code --- 05:03:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:03:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:03:55 DEBUG --- http response --- 05:03:55 DEBUG --- http response --- 05:03:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:03:55 DEBUG ----- output ----- 05:03:55 DEBUG { 05:03:55 DEBUG "status": "success", 05:03:55 DEBUG "data": { 05:03:55 DEBUG "resultType": "vector", 05:03:55 DEBUG "result": [ 05:03:55 DEBUG { 05:03:55 DEBUG "metric": { 05:03:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:03:55 DEBUG }, 05:03:55 DEBUG "value": [ 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG [...] 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:03:55 DEBUG }, 05:03:55 DEBUG "value": [ 05:03:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:03:55 DEBUG 1713499435, 05:03:55 DEBUG ----- output ----- 05:03:55 DEBUG "2004.1666666666667" 05:03:55 DEBUG { 05:03:55 DEBUG ] 05:03:55 DEBUG "status": "success", 05:03:55 DEBUG } 05:03:55 DEBUG "data": { 05:03:55 DEBUG ] 05:03:55 DEBUG "resultType": "vector", 05:03:55 DEBUG } 05:03:55 DEBUG "result": [ 05:03:55 DEBUG } 05:03:55 DEBUG { 05:03:55 DEBUG "metric": { 05:03:55 DEBUG "pod": "ds-idrepo-0" 05:03:55 DEBUG }, 05:03:55 DEBUG "value": [ 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG [...] 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG "pod": "ds-cts-2" 05:03:55 DEBUG }, 05:03:55 DEBUG "value": [ 05:03:55 DEBUG 1713499435, 05:03:55 DEBUG "0" 05:03:55 DEBUG ] 05:03:55 DEBUG } 05:03:55 DEBUG ] 05:03:55 DEBUG } 05:03:55 DEBUG } 05:03:55 INFO [http_cmd]: http status code OK 05:03:55 DEBUG --- status code --- 05:03:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:03:55 DEBUG --- http response --- 05:03:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:03:55 DEBUG ----- output ----- 05:03:55 DEBUG { 05:03:55 DEBUG "status": "success", 05:03:55 DEBUG "data": { 05:03:55 DEBUG "resultType": "vector", 05:03:55 DEBUG "result": [ 05:03:55 DEBUG { 05:03:55 DEBUG "metric": { 05:03:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:03:55 DEBUG }, 05:03:55 DEBUG "value": [ 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG [...] 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:03:55 DEBUG }, 05:03:55 DEBUG "value": [ 05:03:55 DEBUG 1713499435, 05:03:55 DEBUG "0" 05:03:55 DEBUG ] 05:03:55 DEBUG } 05:03:55 DEBUG ] 05:03:55 DEBUG } 05:03:55 DEBUG } 05:03:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499435" 05:03:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499435" 05:03:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499435" 05:03:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499435" 05:03:55 INFO [http_cmd]: http status code OK 05:03:55 DEBUG --- status code --- 05:03:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:03:55 DEBUG --- http response --- 05:03:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:03:55 DEBUG ----- output ----- 05:03:55 DEBUG { 05:03:55 DEBUG "status": "success", 05:03:55 DEBUG "data": { 05:03:55 DEBUG "resultType": "vector", 05:03:55 DEBUG "result": [ 05:03:55 INFO [http_cmd]: http status code OK 05:03:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499435" 05:03:55 DEBUG { 05:03:55 DEBUG --- status code --- 05:03:55 DEBUG "metric": { 05:03:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:03:55 DEBUG "instance": "10.142.0.107:9100" 05:03:55 DEBUG --- http response --- 05:03:55 DEBUG }, 05:03:55 DEBUG "value": [ 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG [...] 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG "instance": "10.142.0.43:9100" 05:03:55 DEBUG }, 05:03:55 DEBUG "value": [ 05:03:55 DEBUG 1713499435, 05:03:55 DEBUG "0.054700000000006334" 05:03:55 DEBUG ] 05:03:55 DEBUG } 05:03:55 DEBUG ] 05:03:55 DEBUG } 05:03:55 DEBUG } 05:03:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 05:03:55 DEBUG ----- output ----- 05:03:55 DEBUG { 05:03:55 DEBUG "status": "success", 05:03:55 DEBUG "data": { 05:03:55 DEBUG "resultType": "vector", 05:03:55 DEBUG "result": [ 05:03:55 DEBUG { 05:03:55 DEBUG "metric": { 05:03:55 DEBUG "pod": "ds-idrepo-1" 05:03:55 DEBUG }, 05:03:55 DEBUG "value": [ 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG [...] 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:03:55 DEBUG }, 05:03:55 DEBUG "value": [ 05:03:55 DEBUG 1713499435, 05:03:55 DEBUG "0" 05:03:55 DEBUG ] 05:03:55 DEBUG } 05:03:55 DEBUG ] 05:03:55 DEBUG } 05:03:55 DEBUG } 05:03:55 INFO [http_cmd]: http status code OK 05:03:55 DEBUG --- status code --- 05:03:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:03:55 DEBUG --- http response --- 05:03:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:03:55 DEBUG ----- output ----- 05:03:55 DEBUG { 05:03:55 DEBUG "status": "success", 05:03:55 DEBUG "data": { 05:03:55 DEBUG "resultType": "vector", 05:03:55 DEBUG "result": [ 05:03:55 DEBUG { 05:03:55 DEBUG "metric": { 05:03:55 DEBUG "pod": "ds-cts-0" 05:03:55 DEBUG }, 05:03:55 DEBUG "value": [ 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG [...] 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG "pod": "ds-cts-2" 05:03:55 DEBUG }, 05:03:55 DEBUG "value": [ 05:03:55 DEBUG 1713499435, 05:03:55 DEBUG "0" 05:03:55 DEBUG ] 05:03:55 DEBUG } 05:03:55 DEBUG ] 05:03:55 DEBUG } 05:03:55 DEBUG } 05:03:55 INFO [http_cmd]: http status code OK 05:03:55 DEBUG --- status code --- 05:03:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:03:55 DEBUG --- http response --- 05:03:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:03:55 DEBUG ----- output ----- 05:03:55 DEBUG { 05:03:55 DEBUG "status": "success", 05:03:55 DEBUG "data": { 05:03:55 DEBUG "resultType": "vector", 05:03:55 DEBUG "result": [ 05:03:55 DEBUG { 05:03:55 DEBUG "metric": { 05:03:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:03:55 DEBUG }, 05:03:55 DEBUG "value": [ 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG [...] 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:03:55 DEBUG }, 05:03:55 DEBUG "value": [ 05:03:55 DEBUG 1713499435, 05:03:55 DEBUG "2261725" 05:03:55 DEBUG ] 05:03:55 DEBUG } 05:03:55 DEBUG ] 05:03:55 DEBUG } 05:03:55 DEBUG } 05:03:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499435" 05:03:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499435" 05:03:55 INFO [http_cmd]: http status code OK 05:03:55 DEBUG --- status code --- 05:03:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:03:55 DEBUG --- http response --- 05:03:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499435" 05:03:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:03:55 DEBUG ----- output ----- 05:03:55 DEBUG { 05:03:55 DEBUG "status": "success", 05:03:55 DEBUG "data": { 05:03:55 DEBUG "resultType": "vector", 05:03:55 DEBUG "result": [ 05:03:55 DEBUG { 05:03:55 DEBUG "metric": { 05:03:55 DEBUG "pod": "ds-idrepo-1" 05:03:55 DEBUG }, 05:03:55 DEBUG "value": [ 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG [...] 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:03:55 DEBUG }, 05:03:55 DEBUG "value": [ 05:03:55 DEBUG 1713499435, 05:03:55 DEBUG "4.001554514078125" 05:03:55 DEBUG ] 05:03:55 DEBUG } 05:03:55 DEBUG ] 05:03:55 DEBUG } 05:03:55 DEBUG } 05:03:55 INFO [http_cmd]: http status code OK 05:03:55 DEBUG --- status code --- 05:03:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:03:55 DEBUG --- http response --- 05:03:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:03:55 DEBUG ----- output ----- 05:03:55 DEBUG { 05:03:55 DEBUG "status": "success", 05:03:55 DEBUG "data": { 05:03:55 DEBUG "resultType": "vector", 05:03:55 DEBUG "result": [ 05:03:55 DEBUG { 05:03:55 DEBUG "metric": { 05:03:55 DEBUG "pod": "ds-cts-0" 05:03:55 DEBUG }, 05:03:55 DEBUG "value": [ 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG [...] 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG "pod": "ds-cts-2" 05:03:55 DEBUG }, 05:03:55 DEBUG "value": [ 05:03:55 DEBUG 1713499435, 05:03:55 DEBUG "0.00012262666666666667" 05:03:55 DEBUG ] 05:03:55 DEBUG } 05:03:55 DEBUG ] 05:03:55 DEBUG } 05:03:55 DEBUG } 05:03:55 INFO [http_cmd]: http status code OK 05:03:55 DEBUG --- status code --- 05:03:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:03:55 DEBUG --- http response --- 05:03:55 INFO [http_cmd]: http status code OK 05:03:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:03:55 DEBUG --- status code --- 05:03:55 DEBUG ----- output ----- 05:03:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:03:55 DEBUG { 05:03:55 DEBUG --- http response --- 05:03:55 DEBUG "status": "success", 05:03:55 DEBUG "data": { 05:03:55 DEBUG "resultType": "vector", 05:03:55 DEBUG "result": [ 05:03:55 DEBUG { 05:03:55 DEBUG "metric": { 05:03:55 DEBUG "instance": "10.142.0.107:9100" 05:03:55 DEBUG }, 05:03:55 DEBUG "value": [ 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG [...] 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG "instance": "10.142.0.43:9100" 05:03:55 DEBUG }, 05:03:55 DEBUG "value": [ 05:03:55 DEBUG 1713499435, 05:03:55 DEBUG "429999139840" 05:03:55 DEBUG ] 05:03:55 DEBUG } 05:03:55 DEBUG ] 05:03:55 DEBUG } 05:03:55 DEBUG } 05:03:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:03:55 DEBUG ----- output ----- 05:03:55 DEBUG { 05:03:55 DEBUG "status": "success", 05:03:55 DEBUG "data": { 05:03:55 DEBUG "resultType": "vector", 05:03:55 DEBUG "result": [ 05:03:55 DEBUG { 05:03:55 DEBUG "metric": { 05:03:55 DEBUG "instance": "10.142.0.107:9100" 05:03:55 DEBUG }, 05:03:55 DEBUG "value": [ 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG [...] 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG "instance": "10.142.0.43:9100" 05:03:55 DEBUG }, 05:03:55 DEBUG "value": [ 05:03:55 DEBUG 1713499435, 05:03:55 DEBUG "47848830.7" 05:03:55 DEBUG ] 05:03:55 DEBUG } 05:03:55 DEBUG ] 05:03:55 DEBUG } 05:03:55 DEBUG } 05:03:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499435" 05:03:55 INFO [http_cmd]: http status code OK 05:03:55 DEBUG --- status code --- 05:03:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:03:55 DEBUG --- http response --- 05:03:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:03:55 DEBUG ----- output ----- 05:03:55 DEBUG { 05:03:55 DEBUG "status": "success", 05:03:55 DEBUG "data": { 05:03:55 DEBUG "resultType": "vector", 05:03:55 DEBUG "result": [ 05:03:55 DEBUG { 05:03:55 DEBUG "metric": { 05:03:55 DEBUG "pod": "ds-idrepo-0" 05:03:55 DEBUG }, 05:03:55 DEBUG "value": [ 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG [...] 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG "pod": "ds-idrepo-2" 05:03:55 DEBUG }, 05:03:55 DEBUG "value": [ 05:03:55 DEBUG 1713499435, 05:03:55 DEBUG "0" 05:03:55 DEBUG ] 05:03:55 DEBUG } 05:03:55 DEBUG ] 05:03:55 DEBUG } 05:03:55 DEBUG } 05:03:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499435" 05:03:55 INFO [http_cmd]: http status code OK 05:03:55 DEBUG --- status code --- 05:03:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:03:55 DEBUG --- http response --- 05:03:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:03:55 DEBUG ----- output ----- 05:03:55 DEBUG { 05:03:55 DEBUG "status": "success", 05:03:55 DEBUG "data": { 05:03:55 DEBUG "resultType": "vector", 05:03:55 DEBUG "result": [ 05:03:55 DEBUG { 05:03:55 DEBUG "metric": { 05:03:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:03:55 DEBUG }, 05:03:55 DEBUG "value": [ 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG [...] 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:03:55 DEBUG }, 05:03:55 DEBUG "value": [ 05:03:55 DEBUG 1713499435, 05:03:55 DEBUG "2003.8999999999999" 05:03:55 DEBUG ] 05:03:55 DEBUG } 05:03:55 DEBUG ] 05:03:55 DEBUG } 05:03:55 DEBUG } 05:03:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499435" 05:03:55 INFO [http_cmd]: http status code OK 05:03:55 DEBUG --- status code --- 05:03:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:03:55 DEBUG --- http response --- 05:03:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:03:55 DEBUG ----- output ----- 05:03:55 DEBUG { 05:03:55 DEBUG "status": "success", 05:03:55 DEBUG "data": { 05:03:55 DEBUG "resultType": "vector", 05:03:55 DEBUG "result": [ 05:03:55 DEBUG { 05:03:55 DEBUG "metric": { 05:03:55 DEBUG "pod": "ds-idrepo-0" 05:03:55 DEBUG }, 05:03:55 DEBUG "value": [ 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG [...] 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG "pod": "ds-idrepo-2" 05:03:55 DEBUG }, 05:03:55 DEBUG "value": [ 05:03:55 DEBUG 1713499435, 05:03:55 DEBUG "0" 05:03:55 DEBUG ] 05:03:55 DEBUG } 05:03:55 DEBUG ] 05:03:55 DEBUG } 05:03:55 DEBUG } 05:03:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713499435" 05:03:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499435" 05:03:55 INFO [http_cmd]: http status code OK 05:03:55 DEBUG --- status code --- 05:03:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:03:55 DEBUG --- http response --- 05:03:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:03:55 DEBUG ----- output ----- 05:03:55 DEBUG { 05:03:55 DEBUG "status": "success", 05:03:55 DEBUG "data": { 05:03:55 DEBUG "resultType": "vector", 05:03:55 DEBUG "result": [ 05:03:55 DEBUG { 05:03:55 DEBUG "metric": { 05:03:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:03:55 DEBUG }, 05:03:55 DEBUG "value": [ 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG [...] 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:03:55 DEBUG }, 05:03:55 DEBUG "value": [ 05:03:55 DEBUG 1713499435, 05:03:55 DEBUG "55.171" 05:03:55 DEBUG ] 05:03:55 DEBUG } 05:03:55 DEBUG ] 05:03:55 DEBUG } 05:03:55 DEBUG } 05:03:55 INFO [http_cmd]: http status code OK 05:03:55 DEBUG --- status code --- 05:03:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:03:55 DEBUG --- http response --- 05:03:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:03:55 DEBUG ----- output ----- 05:03:55 DEBUG { 05:03:55 DEBUG "status": "success", 05:03:55 DEBUG "data": { 05:03:55 DEBUG "resultType": "vector", 05:03:55 DEBUG "result": [ 05:03:55 DEBUG { 05:03:55 DEBUG "metric": { 05:03:55 DEBUG "instance": "10.142.0.107:9100" 05:03:55 DEBUG }, 05:03:55 DEBUG "value": [ 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG [...] 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG "instance": "10.142.0.43:9100" 05:03:55 DEBUG }, 05:03:55 DEBUG "value": [ 05:03:55 DEBUG 1713499435, 05:03:55 DEBUG "0.035999999999999456" 05:03:55 DEBUG ] 05:03:55 DEBUG } 05:03:55 DEBUG ] 05:03:55 DEBUG } 05:03:55 DEBUG } 05:03:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499435" 05:03:55 INFO [http_cmd]: http status code OK 05:03:55 DEBUG --- status code --- 05:03:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:03:55 DEBUG --- http response --- 05:03:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:03:55 DEBUG ----- output ----- 05:03:55 DEBUG { 05:03:55 DEBUG "status": "success", 05:03:55 DEBUG "data": { 05:03:55 DEBUG "resultType": "vector", 05:03:55 DEBUG "result": [ 05:03:55 DEBUG { 05:03:55 DEBUG "metric": { 05:03:55 DEBUG "instance": "10.142.0.107:9100" 05:03:55 DEBUG }, 05:03:55 DEBUG "value": [ 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG [...] 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG 05:03:55 DEBUG "instance": "10.142.0.43:9100" 05:03:55 DEBUG }, 05:03:55 DEBUG "value": [ 05:03:55 DEBUG 1713499435, 05:03:55 DEBUG "47845624.8" 05:03:55 DEBUG ] 05:03:55 DEBUG } 05:03:55 DEBUG ] 05:03:55 DEBUG } 05:03:55 DEBUG } 05:03:55 INFO 05:03:55 INFO [loop_until]: kubectl --namespace=xlou top pods 05:03:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:03:56 INFO [loop_until]: OK (rc = 0) 05:03:56 DEBUG --- stdout --- 05:03:56 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 9798m 8963Mi am-6fd554f8f7-hvvkw 10017m 8875Mi am-6fd554f8f7-pmj4b 10041m 8974Mi ds-cts-0 7590m 12238Mi ds-cts-1 7441m 12156Mi ds-cts-2 7434m 12195Mi ds-idrepo-0 4839m 3255Mi ds-idrepo-1 4946m 3207Mi ds-idrepo-2 4505m 3065Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 4m 1215Mi lodemon-6b89d774d5-rf7sk 7m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 6405m 1973Mi 05:03:56 DEBUG --- stderr --- 05:03:56 DEBUG 05:03:56 INFO 05:03:56 INFO [loop_until]: kubectl --namespace=xlou top node 05:03:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:03:56 INFO [loop_until]: OK (rc = 0) 05:03:56 DEBUG --- stdout --- 05:03:56 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 78m 0% 6263Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 4600m 15% 6691Mi 5% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9547m 31% 11595Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 80m 0% 3271Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 7725m 25% 14804Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 159m 0% 5172Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 9684m 32% 11746Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 10223m 34% 2718Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 10846m 36% 11179Mi 10% gke-xlou-cdm-ds-0f601751-j488 4841m 16% 6160Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 7320m 24% 16185Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 5097m 17% 6878Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 7465m 25% 14176Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 6668m 41% 5722Mi 9% 05:03:56 DEBUG --- stderr --- 05:03:56 DEBUG 05:04:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713499495" 05:04:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713499495" 05:04:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499495" 05:04:55 ERROR [http_cmd]: ERROR 05:04:55 ERROR --- status code --- 05:04:55 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:04:55 ERROR --- http response --- 05:04:55 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:04:55 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:04:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499495" 05:04:55 INFO [http_cmd]: http status code OK 05:04:55 DEBUG --- status code --- 05:04:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:04:55 DEBUG --- http response --- 05:04:55 INFO [http_cmd]: http status code OK 05:04:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:04:55 DEBUG --- status code --- 05:04:55 DEBUG ----- output ----- 05:04:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:04:55 DEBUG { 05:04:55 DEBUG --- http response --- 05:04:55 DEBUG "status": "success", 05:04:55 DEBUG "data": { 05:04:55 DEBUG "resultType": "vector", 05:04:55 DEBUG "result": [ 05:04:55 DEBUG { 05:04:55 DEBUG "metric": { 05:04:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:04:55 DEBUG }, 05:04:55 DEBUG "value": [ 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG [...] 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:04:55 DEBUG }, 05:04:55 DEBUG "value": [ 05:04:55 DEBUG 1713499495, 05:04:55 DEBUG "0" 05:04:55 DEBUG ] 05:04:55 DEBUG } 05:04:55 DEBUG ] 05:04:55 DEBUG } 05:04:55 DEBUG } 05:04:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:04:55 DEBUG ----- output ----- 05:04:55 DEBUG { 05:04:55 DEBUG "status": "success", 05:04:55 DEBUG "data": { 05:04:55 DEBUG "resultType": "vector", 05:04:55 DEBUG "result": [ 05:04:55 DEBUG { 05:04:55 DEBUG "metric": { 05:04:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:04:55 DEBUG }, 05:04:55 DEBUG "value": [ 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG [...] 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:04:55 DEBUG }, 05:04:55 DEBUG "value": [ 05:04:55 DEBUG 1713499495, 05:04:55 DEBUG "2168676352" 05:04:55 DEBUG ] 05:04:55 DEBUG } 05:04:55 DEBUG ] 05:04:55 DEBUG } 05:04:55 DEBUG } 05:04:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499495" 05:04:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499495" 05:04:55 INFO [http_cmd]: http status code OK 05:04:55 DEBUG --- status code --- 05:04:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:04:55 DEBUG --- http response --- 05:04:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 05:04:55 DEBUG ----- output ----- 05:04:55 DEBUG { 05:04:55 DEBUG "status": "success", 05:04:55 DEBUG "data": { 05:04:55 DEBUG "resultType": "vector", 05:04:55 DEBUG "result": [ 05:04:55 DEBUG { 05:04:55 DEBUG "metric": { 05:04:55 DEBUG "pod": "ds-idrepo-1" 05:04:55 DEBUG }, 05:04:55 DEBUG "value": [ 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG [...] 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:04:55 DEBUG }, 05:04:55 DEBUG "value": [ 05:04:55 DEBUG 1713499495, 05:04:55 DEBUG "0" 05:04:55 DEBUG ] 05:04:55 DEBUG } 05:04:55 DEBUG ] 05:04:55 DEBUG } 05:04:55 DEBUG } 05:04:55 INFO [http_cmd]: http status code OK 05:04:55 DEBUG --- status code --- 05:04:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:04:55 DEBUG --- http response --- 05:04:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:04:55 DEBUG ----- output ----- 05:04:55 DEBUG { 05:04:55 DEBUG "status": "success", 05:04:55 DEBUG "data": { 05:04:55 DEBUG "resultType": "vector", 05:04:55 DEBUG "result": [ 05:04:55 DEBUG { 05:04:55 DEBUG "metric": { 05:04:55 DEBUG "pod": "ds-idrepo-0" 05:04:55 DEBUG }, 05:04:55 DEBUG "value": [ 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG [...] 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG "pod": "ds-idrepo-2" 05:04:55 DEBUG }, 05:04:55 DEBUG "value": [ 05:04:55 DEBUG 1713499495, 05:04:55 DEBUG "0" 05:04:55 DEBUG ] 05:04:55 DEBUG } 05:04:55 DEBUG ] 05:04:55 DEBUG } 05:04:55 DEBUG } 05:04:55 INFO [http_cmd]: http status code OK 05:04:55 DEBUG --- status code --- 05:04:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:04:55 DEBUG --- http response --- 05:04:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:04:55 DEBUG ----- output ----- 05:04:55 DEBUG { 05:04:55 DEBUG "status": "success", 05:04:55 DEBUG "data": { 05:04:55 DEBUG "resultType": "vector", 05:04:55 DEBUG "result": [ 05:04:55 DEBUG { 05:04:55 DEBUG "metric": { 05:04:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:04:55 DEBUG }, 05:04:55 DEBUG "value": [ 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG [...] 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:04:55 DEBUG }, 05:04:55 DEBUG "value": [ 05:04:55 DEBUG 1713499495, 05:04:55 DEBUG "2002.6666666666667" 05:04:55 DEBUG ] 05:04:55 DEBUG } 05:04:55 DEBUG ] 05:04:55 DEBUG } 05:04:55 DEBUG } 05:04:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499495" 05:04:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499495" 05:04:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499495" 05:04:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713499495" 05:04:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713499495" 05:04:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499495" 05:04:55 INFO [http_cmd]: http status code OK 05:04:55 INFO [http_cmd]: http status code OK 05:04:55 DEBUG --- status code --- 05:04:55 DEBUG --- status code --- 05:04:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:04:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:04:55 DEBUG --- http response --- 05:04:55 DEBUG --- http response --- 05:04:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:04:55 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:04:55 DEBUG ----- output ----- 05:04:55 DEBUG { 05:04:55 DEBUG "status": "success", 05:04:55 DEBUG "data": { 05:04:55 DEBUG "resultType": "vector", 05:04:55 DEBUG "result": [ 05:04:55 DEBUG { 05:04:55 DEBUG "metric": { 05:04:55 DEBUG "pod": "ds-cts-0" 05:04:55 DEBUG }, 05:04:55 DEBUG "value": [ 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG [...] 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG "pod": "ds-cts-2" 05:04:55 DEBUG }, 05:04:55 DEBUG "value": [ 05:04:55 DEBUG 1713499495, 05:04:55 DEBUG "0.7" 05:04:55 DEBUG ] 05:04:55 DEBUG } 05:04:55 DEBUG ] 05:04:55 DEBUG } 05:04:55 DEBUG } 05:04:55 INFO [http_cmd]: http status code OK 05:04:55 DEBUG --- status code --- 05:04:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:04:55 INFO [http_cmd]: http status code OK 05:04:55 DEBUG --- http response --- 05:04:55 DEBUG --- status code --- 05:04:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:04:55 DEBUG --- http response --- 05:04:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 05:04:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 05:04:55 DEBUG ----- output ----- 05:04:55 DEBUG ----- output ----- 05:04:55 DEBUG { 05:04:55 DEBUG { 05:04:55 DEBUG "status": "success", 05:04:55 DEBUG "status": "success", 05:04:55 DEBUG "data": { 05:04:55 DEBUG "data": { 05:04:55 DEBUG "resultType": "vector", 05:04:55 DEBUG "resultType": "vector", 05:04:55 DEBUG "result": [ 05:04:55 DEBUG "result": [ 05:04:55 DEBUG { 05:04:55 DEBUG { 05:04:55 DEBUG "metric": { 05:04:55 INFO [http_cmd]: http status code OK 05:04:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499495" 05:04:55 DEBUG "metric": { 05:04:55 DEBUG "pod": "ds-idrepo-1" 05:04:55 DEBUG --- status code --- 05:04:55 DEBUG "pod": "ds-idrepo-1" 05:04:55 DEBUG }, 05:04:55 INFO [http_cmd]: http status code OK 05:04:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:04:55 DEBUG }, 05:04:55 DEBUG "value": [ 05:04:55 DEBUG --- status code --- 05:04:55 DEBUG --- http response --- 05:04:55 DEBUG "value": [ 05:04:55 DEBUG 05:04:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:04:55 DEBUG 05:04:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:04:55 DEBUG 05:04:55 DEBUG --- http response --- 05:04:55 DEBUG 05:04:55 DEBUG ----- output ----- 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:04:55 DEBUG { 05:04:55 DEBUG [...] 05:04:55 DEBUG [...] 05:04:55 DEBUG ----- output ----- 05:04:55 DEBUG "status": "success", 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG { 05:04:55 DEBUG "data": { 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG "status": "success", 05:04:55 DEBUG "resultType": "vector", 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG "data": { 05:04:55 DEBUG "result": [ 05:04:55 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:04:55 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:04:55 DEBUG "resultType": "vector", 05:04:55 DEBUG { 05:04:55 DEBUG }, 05:04:55 DEBUG }, 05:04:55 DEBUG "result": [ 05:04:55 DEBUG "metric": { 05:04:55 DEBUG "value": [ 05:04:55 DEBUG "value": [ 05:04:55 DEBUG { 05:04:55 DEBUG "pod": "ds-idrepo-0" 05:04:55 DEBUG 1713499495, 05:04:55 DEBUG 1713499495, 05:04:55 DEBUG "metric": { 05:04:55 DEBUG }, 05:04:55 DEBUG "0" 05:04:55 DEBUG "37762.600837023114" 05:04:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:04:55 DEBUG "value": [ 05:04:55 DEBUG ] 05:04:55 DEBUG ] 05:04:55 DEBUG }, 05:04:55 DEBUG 05:04:55 DEBUG } 05:04:55 DEBUG } 05:04:55 DEBUG "value": [ 05:04:55 DEBUG 05:04:55 DEBUG ] 05:04:55 DEBUG ] 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG } 05:04:55 DEBUG } 05:04:55 DEBUG 05:04:55 DEBUG [...] 05:04:55 DEBUG } 05:04:55 DEBUG } 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG [...] 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG "pod": "ds-cts-2" 05:04:55 DEBUG 05:04:55 DEBUG }, 05:04:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:04:55 DEBUG "value": [ 05:04:55 DEBUG }, 05:04:55 DEBUG 1713499495, 05:04:55 DEBUG "value": [ 05:04:55 DEBUG "0" 05:04:55 DEBUG 1713499495, 05:04:55 DEBUG ] 05:04:55 DEBUG "6.5349926242620375" 05:04:55 DEBUG } 05:04:55 DEBUG ] 05:04:55 DEBUG ] 05:04:55 DEBUG } 05:04:55 DEBUG } 05:04:55 DEBUG ] 05:04:55 DEBUG } 05:04:55 DEBUG } 05:04:55 DEBUG } 05:04:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499495" 05:04:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499495" 05:04:55 INFO [http_cmd]: http status code OK 05:04:55 DEBUG --- status code --- 05:04:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:04:55 DEBUG --- http response --- 05:04:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:04:55 DEBUG ----- output ----- 05:04:55 DEBUG { 05:04:55 DEBUG "status": "success", 05:04:55 DEBUG "data": { 05:04:55 DEBUG "resultType": "vector", 05:04:55 DEBUG "result": [ 05:04:55 DEBUG { 05:04:55 DEBUG "metric": { 05:04:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:04:55 DEBUG }, 05:04:55 DEBUG "value": [ 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG [...] 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:04:55 DEBUG }, 05:04:55 DEBUG "value": [ 05:04:55 DEBUG 1713499495, 05:04:55 DEBUG "2003.0333333333333" 05:04:55 DEBUG ] 05:04:55 DEBUG } 05:04:55 DEBUG ] 05:04:55 DEBUG } 05:04:55 DEBUG } 05:04:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499495" 05:04:55 INFO [http_cmd]: http status code OK 05:04:55 INFO [http_cmd]: http status code OK 05:04:55 DEBUG --- status code --- 05:04:55 DEBUG --- status code --- 05:04:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:04:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:04:55 DEBUG --- http response --- 05:04:55 DEBUG --- http response --- 05:04:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:04:55 DEBUG ----- output ----- 05:04:55 DEBUG { 05:04:55 DEBUG "status": "success", 05:04:55 DEBUG "data": { 05:04:55 DEBUG "resultType": "vector", 05:04:55 DEBUG "result": [ 05:04:55 DEBUG { 05:04:55 DEBUG "metric": { 05:04:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:04:55 DEBUG }, 05:04:55 DEBUG "value": [ 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG [...] 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:04:55 DEBUG }, 05:04:55 DEBUG "value": [ 05:04:55 DEBUG 1713499495, 05:04:55 DEBUG "0" 05:04:55 DEBUG ] 05:04:55 DEBUG } 05:04:55 DEBUG ] 05:04:55 DEBUG } 05:04:55 DEBUG } 05:04:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 05:04:55 DEBUG ----- output ----- 05:04:55 DEBUG { 05:04:55 DEBUG "status": "success", 05:04:55 DEBUG "data": { 05:04:55 DEBUG "resultType": "vector", 05:04:55 DEBUG "result": [ 05:04:55 DEBUG { 05:04:55 DEBUG "metric": { 05:04:55 DEBUG "pod": "ds-idrepo-1" 05:04:55 DEBUG }, 05:04:55 DEBUG "value": [ 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG [...] 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:04:55 DEBUG }, 05:04:55 DEBUG "value": [ 05:04:55 DEBUG 1713499495, 05:04:55 DEBUG "6.4995105869169825" 05:04:55 DEBUG ] 05:04:55 DEBUG } 05:04:55 DEBUG ] 05:04:55 DEBUG } 05:04:55 DEBUG } 05:04:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499495" 05:04:55 INFO [http_cmd]: http status code OK 05:04:55 DEBUG --- status code --- 05:04:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:04:55 DEBUG --- http response --- 05:04:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:04:55 DEBUG ----- output ----- 05:04:55 DEBUG { 05:04:55 DEBUG "status": "success", 05:04:55 DEBUG "data": { 05:04:55 DEBUG "resultType": "vector", 05:04:55 DEBUG "result": [ 05:04:55 DEBUG { 05:04:55 DEBUG "metric": { 05:04:55 DEBUG "pod": "ds-cts-0" 05:04:55 DEBUG }, 05:04:55 DEBUG "value": [ 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG [...] 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG "pod": "ds-cts-2" 05:04:55 DEBUG }, 05:04:55 DEBUG "value": [ 05:04:55 DEBUG 1713499495, 05:04:55 DEBUG "0" 05:04:55 DEBUG ] 05:04:55 DEBUG } 05:04:55 DEBUG ] 05:04:55 DEBUG } 05:04:55 DEBUG } 05:04:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499495" 05:04:55 INFO [http_cmd]: http status code OK 05:04:55 DEBUG --- status code --- 05:04:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:04:55 DEBUG --- http response --- 05:04:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:04:55 DEBUG ----- output ----- 05:04:55 DEBUG { 05:04:55 DEBUG "status": "success", 05:04:55 DEBUG "data": { 05:04:55 DEBUG "resultType": "vector", 05:04:55 DEBUG "result": [ 05:04:55 DEBUG { 05:04:55 DEBUG "metric": { 05:04:55 DEBUG "instance": "10.142.0.107:9100" 05:04:55 DEBUG }, 05:04:55 DEBUG "value": [ 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG [...] 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG "instance": "10.142.0.43:9100" 05:04:55 DEBUG }, 05:04:55 DEBUG "value": [ 05:04:55 DEBUG 1713499495, 05:04:55 DEBUG "432352416768" 05:04:55 DEBUG ] 05:04:55 DEBUG } 05:04:55 DEBUG ] 05:04:55 DEBUG } 05:04:55 DEBUG } 05:04:55 INFO [http_cmd]: http status code OK 05:04:55 DEBUG --- status code --- 05:04:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:04:55 DEBUG --- http response --- 05:04:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:04:55 DEBUG ----- output ----- 05:04:55 DEBUG { 05:04:55 DEBUG "status": "success", 05:04:55 DEBUG "data": { 05:04:55 DEBUG "resultType": "vector", 05:04:55 DEBUG "result": [ 05:04:55 DEBUG { 05:04:55 DEBUG "metric": { 05:04:55 DEBUG "instance": "10.142.0.107:9100" 05:04:55 DEBUG }, 05:04:55 DEBUG "value": [ 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG [...] 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG "instance": "10.142.0.43:9100" 05:04:55 DEBUG }, 05:04:55 DEBUG "value": [ 05:04:55 DEBUG 1713499495, 05:04:55 DEBUG "0.05660000000000025" 05:04:55 DEBUG ] 05:04:55 DEBUG } 05:04:55 DEBUG ] 05:04:55 DEBUG } 05:04:55 DEBUG } 05:04:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499495" 05:04:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499495" 05:04:55 INFO [http_cmd]: http status code OK 05:04:55 DEBUG --- status code --- 05:04:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:04:55 DEBUG --- http response --- 05:04:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499495" 05:04:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:04:55 DEBUG ----- output ----- 05:04:55 DEBUG { 05:04:55 DEBUG "status": "success", 05:04:55 DEBUG "data": { 05:04:55 DEBUG "resultType": "vector", 05:04:55 DEBUG "result": [ 05:04:55 DEBUG { 05:04:55 DEBUG "metric": { 05:04:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:04:55 DEBUG }, 05:04:55 DEBUG "value": [ 05:04:55 DEBUG 05:04:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499495" 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG [...] 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:04:55 DEBUG }, 05:04:55 DEBUG "value": [ 05:04:55 DEBUG 1713499495, 05:04:55 DEBUG "2382045" 05:04:55 DEBUG ] 05:04:55 DEBUG } 05:04:55 DEBUG ] 05:04:55 DEBUG } 05:04:55 DEBUG } 05:04:55 INFO [http_cmd]: http status code OK 05:04:55 INFO [http_cmd]: http status code OK 05:04:55 DEBUG --- status code --- 05:04:55 DEBUG --- status code --- 05:04:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:04:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:04:55 DEBUG --- http response --- 05:04:55 DEBUG --- http response --- 05:04:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:04:55 DEBUG ----- output ----- 05:04:55 DEBUG { 05:04:55 DEBUG "status": "success", 05:04:55 DEBUG "data": { 05:04:55 DEBUG "resultType": "vector", 05:04:55 DEBUG "result": [ 05:04:55 DEBUG { 05:04:55 DEBUG "metric": { 05:04:55 DEBUG "pod": "ds-cts-0" 05:04:55 DEBUG }, 05:04:55 DEBUG "value": [ 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG [...] 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG "pod": "ds-cts-2" 05:04:55 DEBUG }, 05:04:55 DEBUG "value": [ 05:04:55 DEBUG 1713499495, 05:04:55 DEBUG "0.00010496666666666665" 05:04:55 DEBUG ] 05:04:55 DEBUG } 05:04:55 DEBUG ] 05:04:55 DEBUG } 05:04:55 DEBUG } 05:04:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:04:55 DEBUG ----- output ----- 05:04:55 DEBUG { 05:04:55 DEBUG "status": "success", 05:04:55 DEBUG "data": { 05:04:55 DEBUG "resultType": "vector", 05:04:55 DEBUG "result": [ 05:04:55 DEBUG { 05:04:55 DEBUG "metric": { 05:04:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:04:55 DEBUG }, 05:04:55 DEBUG "value": [ 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG [...] 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:04:55 DEBUG }, 05:04:55 DEBUG "value": [ 05:04:55 DEBUG 1713499495, 05:04:55 DEBUG "2002.7" 05:04:55 DEBUG ] 05:04:55 DEBUG } 05:04:55 DEBUG ] 05:04:55 DEBUG } 05:04:55 DEBUG } 05:04:55 INFO [http_cmd]: http status code OK 05:04:55 DEBUG --- status code --- 05:04:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:04:55 DEBUG --- http response --- 05:04:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:04:55 DEBUG ----- output ----- 05:04:55 DEBUG { 05:04:55 DEBUG "status": "success", 05:04:55 DEBUG "data": { 05:04:55 DEBUG "resultType": "vector", 05:04:55 DEBUG "result": [ 05:04:55 DEBUG { 05:04:55 DEBUG "metric": { 05:04:55 DEBUG "instance": "10.142.0.107:9100" 05:04:55 DEBUG }, 05:04:55 DEBUG "value": [ 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG [...] 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG "instance": "10.142.0.43:9100" 05:04:55 DEBUG }, 05:04:55 DEBUG "value": [ 05:04:55 DEBUG 1713499495, 05:04:55 DEBUG "47775514.3" 05:04:55 DEBUG ] 05:04:55 DEBUG } 05:04:55 DEBUG ] 05:04:55 DEBUG } 05:04:55 DEBUG } 05:04:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499495" 05:04:55 INFO [http_cmd]: http status code OK 05:04:55 DEBUG --- status code --- 05:04:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:04:55 DEBUG --- http response --- 05:04:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:04:55 DEBUG ----- output ----- 05:04:55 DEBUG { 05:04:55 DEBUG "status": "success", 05:04:55 DEBUG "data": { 05:04:55 DEBUG "resultType": "vector", 05:04:55 DEBUG "result": [ 05:04:55 DEBUG { 05:04:55 DEBUG "metric": { 05:04:55 DEBUG "pod": "ds-idrepo-0" 05:04:55 DEBUG }, 05:04:55 DEBUG "value": [ 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG [...] 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG "pod": "ds-idrepo-2" 05:04:55 DEBUG }, 05:04:55 DEBUG "value": [ 05:04:55 DEBUG 1713499495, 05:04:55 DEBUG "0" 05:04:55 DEBUG ] 05:04:55 DEBUG } 05:04:55 DEBUG ] 05:04:55 DEBUG } 05:04:55 DEBUG } 05:04:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499495" 05:04:55 INFO [http_cmd]: http status code OK 05:04:55 DEBUG --- status code --- 05:04:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:04:55 DEBUG --- http response --- 05:04:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:04:55 DEBUG ----- output ----- 05:04:55 DEBUG { 05:04:55 DEBUG "status": "success", 05:04:55 DEBUG "data": { 05:04:55 DEBUG "resultType": "vector", 05:04:55 DEBUG "result": [ 05:04:55 DEBUG { 05:04:55 DEBUG "metric": { 05:04:55 DEBUG "pod": "ds-idrepo-0" 05:04:55 DEBUG }, 05:04:55 DEBUG "value": [ 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG [...] 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG "pod": "ds-idrepo-2" 05:04:55 DEBUG }, 05:04:55 DEBUG "value": [ 05:04:55 DEBUG 1713499495, 05:04:55 DEBUG "0" 05:04:55 DEBUG ] 05:04:55 DEBUG } 05:04:55 DEBUG ] 05:04:55 DEBUG } 05:04:55 DEBUG } 05:04:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713499495" 05:04:55 INFO [http_cmd]: http status code OK 05:04:55 DEBUG --- status code --- 05:04:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:04:55 DEBUG --- http response --- 05:04:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:04:55 DEBUG ----- output ----- 05:04:55 DEBUG { 05:04:55 DEBUG "status": "success", 05:04:55 DEBUG "data": { 05:04:55 DEBUG "resultType": "vector", 05:04:55 DEBUG "result": [ 05:04:55 DEBUG { 05:04:55 DEBUG "metric": { 05:04:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:04:55 DEBUG }, 05:04:55 DEBUG "value": [ 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG [...] 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:04:55 DEBUG }, 05:04:55 DEBUG "value": [ 05:04:55 DEBUG 1713499495, 05:04:55 DEBUG "55.171" 05:04:55 DEBUG ] 05:04:55 DEBUG } 05:04:55 DEBUG ] 05:04:55 DEBUG } 05:04:55 DEBUG } 05:04:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499495" 05:04:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499495" 05:04:55 INFO [http_cmd]: http status code OK 05:04:55 DEBUG --- status code --- 05:04:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:04:55 DEBUG --- http response --- 05:04:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:04:55 DEBUG ----- output ----- 05:04:55 DEBUG { 05:04:55 DEBUG "status": "success", 05:04:55 DEBUG "data": { 05:04:55 DEBUG "resultType": "vector", 05:04:55 DEBUG "result": [ 05:04:55 DEBUG { 05:04:55 DEBUG "metric": { 05:04:55 DEBUG "instance": "10.142.0.107:9100" 05:04:55 DEBUG }, 05:04:55 DEBUG "value": [ 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG [...] 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG "instance": "10.142.0.43:9100" 05:04:55 DEBUG }, 05:04:55 DEBUG "value": [ 05:04:55 DEBUG 1713499495, 05:04:55 DEBUG "0.03900000000000053" 05:04:55 DEBUG ] 05:04:55 DEBUG } 05:04:55 DEBUG ] 05:04:55 DEBUG } 05:04:55 DEBUG } 05:04:55 INFO [http_cmd]: http status code OK 05:04:55 DEBUG --- status code --- 05:04:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:04:55 DEBUG --- http response --- 05:04:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:04:55 DEBUG ----- output ----- 05:04:55 DEBUG { 05:04:55 DEBUG "status": "success", 05:04:55 DEBUG "data": { 05:04:55 DEBUG "resultType": "vector", 05:04:55 DEBUG "result": [ 05:04:55 DEBUG { 05:04:55 DEBUG "metric": { 05:04:55 DEBUG "instance": "10.142.0.107:9100" 05:04:55 DEBUG }, 05:04:55 DEBUG "value": [ 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG [...] 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG 05:04:55 DEBUG "instance": "10.142.0.43:9100" 05:04:55 DEBUG }, 05:04:55 DEBUG "value": [ 05:04:55 DEBUG 1713499495, 05:04:55 DEBUG "47772834.03333333" 05:04:55 DEBUG ] 05:04:55 DEBUG } 05:04:55 DEBUG ] 05:04:55 DEBUG } 05:04:55 DEBUG } 05:04:56 INFO 05:04:56 INFO [loop_until]: kubectl --namespace=xlou top pods 05:04:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:04:56 INFO [loop_until]: OK (rc = 0) 05:04:56 DEBUG --- stdout --- 05:04:56 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 10065m 9119Mi am-6fd554f8f7-hvvkw 9350m 8875Mi am-6fd554f8f7-pmj4b 10335m 8974Mi ds-cts-0 7487m 12241Mi ds-cts-1 7373m 12164Mi ds-cts-2 7863m 12200Mi ds-idrepo-0 4762m 3255Mi ds-idrepo-1 4738m 3207Mi ds-idrepo-2 4590m 3065Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 3m 1215Mi lodemon-6b89d774d5-rf7sk 10m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 6649m 1974Mi 05:04:56 DEBUG --- stderr --- 05:04:56 DEBUG 05:04:56 INFO 05:04:56 INFO [loop_until]: kubectl --namespace=xlou top node 05:04:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:04:56 INFO [loop_until]: OK (rc = 0) 05:04:56 DEBUG --- stdout --- 05:04:56 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 74m 0% 6265Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 4655m 15% 6694Mi 5% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9420m 31% 11581Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 80m 0% 3247Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 7755m 25% 14786Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 157m 0% 5169Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 9961m 33% 11890Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 10300m 34% 2728Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 11129m 37% 11170Mi 10% gke-xlou-cdm-ds-0f601751-j488 4850m 16% 6149Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 7318m 24% 16177Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 5137m 17% 6874Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 7359m 24% 14174Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 6695m 42% 5723Mi 9% 05:04:56 DEBUG --- stderr --- 05:04:56 DEBUG 05:05:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499555" 05:05:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713499555" 05:05:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499555" 05:05:55 INFO [http_cmd]: http status code OK 05:05:55 DEBUG --- status code --- 05:05:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:05:55 DEBUG --- http response --- 05:05:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:05:55 DEBUG ----- output ----- 05:05:55 DEBUG { 05:05:55 DEBUG "status": "success", 05:05:55 DEBUG "data": { 05:05:55 DEBUG "resultType": "vector", 05:05:55 DEBUG "result": [ 05:05:55 DEBUG { 05:05:55 DEBUG "metric": { 05:05:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:05:55 DEBUG }, 05:05:55 DEBUG "value": [ 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG [...] 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:05:55 DEBUG }, 05:05:55 DEBUG "value": [ 05:05:55 DEBUG 1713499555, 05:05:55 DEBUG "0" 05:05:55 DEBUG ] 05:05:55 DEBUG } 05:05:55 DEBUG ] 05:05:55 DEBUG } 05:05:55 DEBUG } 05:05:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499555" 05:05:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713499555" 05:05:55 ERROR [http_cmd]: ERROR 05:05:55 ERROR --- status code --- 05:05:55 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:05:55 ERROR --- http response --- 05:05:55 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:05:55 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:05:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713499555" 05:05:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499555" 05:05:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499555" 05:05:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499555" 05:05:55 INFO [http_cmd]: http status code OK 05:05:55 DEBUG --- status code --- 05:05:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:05:55 DEBUG --- http response --- 05:05:55 INFO [http_cmd]: http status code OK 05:05:55 DEBUG --- status code --- 05:05:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:05:55 DEBUG --- http response --- 05:05:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:05:55 DEBUG ----- output ----- 05:05:55 DEBUG { 05:05:55 DEBUG "status": "success", 05:05:55 DEBUG "data": { 05:05:55 DEBUG "resultType": "vector", 05:05:55 DEBUG "result": [ 05:05:55 DEBUG { 05:05:55 DEBUG "metric": { 05:05:55 DEBUG "pod": "ds-cts-0" 05:05:55 DEBUG }, 05:05:55 DEBUG "value": [ 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG [...] 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG "pod": "ds-cts-2" 05:05:55 DEBUG }, 05:05:55 DEBUG "value": [ 05:05:55 DEBUG 1713499555, 05:05:55 DEBUG "1.0666666666666667" 05:05:55 DEBUG ] 05:05:55 DEBUG } 05:05:55 DEBUG ] 05:05:55 DEBUG } 05:05:55 DEBUG } 05:05:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:05:55 INFO [http_cmd]: http status code OK 05:05:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499555" 05:05:55 DEBUG ----- output ----- 05:05:55 DEBUG --- status code --- 05:05:55 DEBUG { 05:05:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:05:55 DEBUG "status": "success", 05:05:55 DEBUG --- http response --- 05:05:55 DEBUG "data": { 05:05:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 05:05:55 DEBUG "resultType": "vector", 05:05:55 DEBUG ----- output ----- 05:05:55 DEBUG "result": [ 05:05:55 DEBUG { 05:05:55 DEBUG { 05:05:55 DEBUG "status": "success", 05:05:55 DEBUG "metric": { 05:05:55 DEBUG "data": { 05:05:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:05:55 DEBUG "resultType": "vector", 05:05:55 DEBUG }, 05:05:55 DEBUG "result": [ 05:05:55 DEBUG "value": [ 05:05:55 DEBUG { 05:05:55 DEBUG 05:05:55 DEBUG "metric": { 05:05:55 DEBUG 05:05:55 DEBUG "pod": "ds-idrepo-1" 05:05:55 DEBUG 05:05:55 DEBUG }, 05:05:55 DEBUG [...] 05:05:55 DEBUG "value": [ 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:05:55 DEBUG [...] 05:05:55 DEBUG }, 05:05:55 DEBUG 05:05:55 DEBUG "value": [ 05:05:55 DEBUG 05:05:55 DEBUG 1713499555, 05:05:55 DEBUG 05:05:55 DEBUG "2169167872" 05:05:55 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:05:55 DEBUG ] 05:05:55 DEBUG }, 05:05:55 DEBUG } 05:05:55 DEBUG "value": [ 05:05:55 DEBUG ] 05:05:55 DEBUG 1713499555, 05:05:55 DEBUG } 05:05:55 DEBUG "0" 05:05:55 INFO [http_cmd]: http status code OK 05:05:55 DEBUG } 05:05:55 INFO [http_cmd]: http status code OK 05:05:55 DEBUG ] 05:05:55 DEBUG --- status code --- 05:05:55 DEBUG --- status code --- 05:05:55 DEBUG } 05:05:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:05:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:05:55 DEBUG ] 05:05:55 DEBUG --- http response --- 05:05:55 DEBUG --- http response --- 05:05:55 DEBUG } 05:05:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:05:55 DEBUG } 05:05:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:05:55 INFO [http_cmd]: http status code OK 05:05:55 INFO [http_cmd]: http status code OK 05:05:55 DEBUG ----- output ----- 05:05:55 DEBUG ----- output ----- 05:05:55 DEBUG --- status code --- 05:05:55 DEBUG --- status code --- 05:05:55 DEBUG { 05:05:55 DEBUG { 05:05:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:05:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:05:55 DEBUG "status": "success", 05:05:55 DEBUG "status": "success", 05:05:55 DEBUG --- http response --- 05:05:55 DEBUG --- http response --- 05:05:55 DEBUG "data": { 05:05:55 DEBUG "data": { 05:05:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:05:55 DEBUG ----- output ----- 05:05:55 DEBUG "resultType": "vector", 05:05:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:05:55 DEBUG "resultType": "vector", 05:05:55 INFO [http_cmd]: http status code OK 05:05:55 DEBUG { 05:05:55 DEBUG "result": [ 05:05:55 DEBUG ----- output ----- 05:05:55 DEBUG "result": [ 05:05:55 DEBUG --- status code --- 05:05:55 DEBUG "status": "success", 05:05:55 DEBUG { 05:05:55 DEBUG { 05:05:55 DEBUG { 05:05:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:05:55 DEBUG "data": { 05:05:55 DEBUG "metric": { 05:05:55 DEBUG "status": "success", 05:05:55 DEBUG "metric": { 05:05:55 DEBUG --- http response --- 05:05:55 DEBUG "resultType": "vector", 05:05:55 DEBUG "pod": "ds-idrepo-0" 05:05:55 DEBUG "data": { 05:05:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:05:55 DEBUG "result": [ 05:05:55 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:05:55 DEBUG }, 05:05:55 DEBUG "resultType": "vector", 05:05:55 DEBUG }, 05:05:55 DEBUG { 05:05:55 DEBUG "value": [ 05:05:55 DEBUG "result": [ 05:05:55 DEBUG "value": [ 05:05:55 DEBUG "metric": { 05:05:55 DEBUG 05:05:55 DEBUG { 05:05:55 DEBUG 05:05:55 DEBUG "pod": "ds-idrepo-0" 05:05:55 DEBUG 05:05:55 DEBUG "metric": { 05:05:55 DEBUG 05:05:55 DEBUG }, 05:05:55 DEBUG 05:05:55 DEBUG "pod": "ds-idrepo-1" 05:05:55 DEBUG 05:05:55 DEBUG "value": [ 05:05:55 DEBUG [...] 05:05:55 DEBUG }, 05:05:55 DEBUG [...] 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG "value": [ 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG [...] 05:05:55 DEBUG "pod": "ds-cts-2" 05:05:55 DEBUG 05:05:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:05:55 DEBUG 05:05:55 DEBUG }, 05:05:55 DEBUG [...] 05:05:55 DEBUG }, 05:05:55 DEBUG 05:05:55 DEBUG "value": [ 05:05:55 DEBUG 05:05:55 DEBUG "value": [ 05:05:55 DEBUG 05:05:55 DEBUG 1713499555, 05:05:55 DEBUG 05:05:55 DEBUG 1713499555, 05:05:55 DEBUG "pod": "ds-idrepo-2" 05:05:55 DEBUG "0" 05:05:55 DEBUG 05:05:55 DEBUG "2003.9333333333334" 05:05:55 DEBUG }, 05:05:55 DEBUG ] 05:05:55 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:05:55 DEBUG ] 05:05:55 DEBUG "value": [ 05:05:55 DEBUG } 05:05:55 DEBUG }, 05:05:55 DEBUG } 05:05:55 DEBUG 1713499555, 05:05:55 DEBUG ] 05:05:55 DEBUG "value": [ 05:05:55 DEBUG ] 05:05:55 DEBUG "0" 05:05:55 DEBUG } 05:05:55 DEBUG 1713499555, 05:05:55 DEBUG } 05:05:55 DEBUG ] 05:05:55 DEBUG } 05:05:55 DEBUG "0" 05:05:55 DEBUG } 05:05:55 DEBUG } 05:05:55 DEBUG ] 05:05:55 DEBUG ] 05:05:55 DEBUG } 05:05:55 DEBUG } 05:05:55 DEBUG ] 05:05:55 DEBUG } 05:05:55 DEBUG } 05:05:55 DEBUG } 05:05:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499555" 05:05:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499555" 05:05:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499555" 05:05:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713499555" 05:05:55 INFO [http_cmd]: http status code OK 05:05:55 INFO [http_cmd]: http status code OK 05:05:55 DEBUG --- status code --- 05:05:55 DEBUG --- status code --- 05:05:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:05:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:05:55 DEBUG --- http response --- 05:05:55 DEBUG --- http response --- 05:05:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:05:55 DEBUG ----- output ----- 05:05:55 DEBUG { 05:05:55 DEBUG "status": "success", 05:05:55 DEBUG "data": { 05:05:55 DEBUG "resultType": "vector", 05:05:55 DEBUG "result": [ 05:05:55 DEBUG { 05:05:55 DEBUG "metric": { 05:05:55 DEBUG "instance": "10.142.0.107:9100" 05:05:55 DEBUG }, 05:05:55 DEBUG "value": [ 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG [...] 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG "instance": "10.142.0.43:9100" 05:05:55 DEBUG }, 05:05:55 DEBUG "value": [ 05:05:55 DEBUG 1713499555, 05:05:55 DEBUG "434732004352" 05:05:55 DEBUG ] 05:05:55 DEBUG } 05:05:55 DEBUG ] 05:05:55 DEBUG } 05:05:55 DEBUG } 05:05:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:05:55 DEBUG ----- output ----- 05:05:55 DEBUG { 05:05:55 DEBUG "status": "success", 05:05:55 DEBUG "data": { 05:05:55 DEBUG "resultType": "vector", 05:05:55 DEBUG "result": [ 05:05:55 DEBUG { 05:05:55 DEBUG "metric": { 05:05:55 DEBUG "pod": "ds-idrepo-1" 05:05:55 DEBUG }, 05:05:55 DEBUG "value": [ 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG [...] 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:05:55 DEBUG }, 05:05:55 DEBUG "value": [ 05:05:55 DEBUG 1713499555, 05:05:55 DEBUG "39479.2454914524" 05:05:55 DEBUG ] 05:05:55 DEBUG } 05:05:55 DEBUG ] 05:05:55 DEBUG } 05:05:55 DEBUG } 05:05:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499555" 05:05:55 INFO [http_cmd]: http status code OK 05:05:55 DEBUG --- status code --- 05:05:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:05:55 DEBUG --- http response --- 05:05:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:05:55 DEBUG ----- output ----- 05:05:55 DEBUG { 05:05:55 DEBUG "status": "success", 05:05:55 DEBUG "data": { 05:05:55 DEBUG "resultType": "vector", 05:05:55 DEBUG "result": [ 05:05:55 DEBUG { 05:05:55 DEBUG "metric": { 05:05:55 DEBUG "pod": "ds-idrepo-1" 05:05:55 DEBUG }, 05:05:55 DEBUG "value": [ 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG [...] 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:05:55 DEBUG }, 05:05:55 DEBUG "value": [ 05:05:55 DEBUG 1713499555, 05:05:55 DEBUG "6.566715818746524" 05:05:55 DEBUG ] 05:05:55 DEBUG } 05:05:55 DEBUG ] 05:05:55 DEBUG } 05:05:55 DEBUG } 05:05:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499555" 05:05:55 INFO [http_cmd]: http status code OK 05:05:55 DEBUG --- status code --- 05:05:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:05:55 DEBUG --- http response --- 05:05:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:05:55 DEBUG ----- output ----- 05:05:55 DEBUG { 05:05:55 DEBUG "status": "success", 05:05:55 DEBUG "data": { 05:05:55 DEBUG "resultType": "vector", 05:05:55 DEBUG "result": [ 05:05:55 DEBUG { 05:05:55 DEBUG "metric": { 05:05:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:05:55 DEBUG }, 05:05:55 DEBUG "value": [ 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG [...] 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:05:55 DEBUG }, 05:05:55 DEBUG "value": [ 05:05:55 DEBUG 1713499555, 05:05:55 DEBUG "6.5898119886482105" 05:05:55 DEBUG ] 05:05:55 DEBUG } 05:05:55 DEBUG ] 05:05:55 DEBUG } 05:05:55 DEBUG } 05:05:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499555" 05:05:55 INFO [http_cmd]: http status code OK 05:05:55 DEBUG --- status code --- 05:05:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:05:55 DEBUG --- http response --- 05:05:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:05:55 DEBUG ----- output ----- 05:05:55 DEBUG { 05:05:55 DEBUG "status": "success", 05:05:55 DEBUG "data": { 05:05:55 DEBUG "resultType": "vector", 05:05:55 DEBUG "result": [ 05:05:55 DEBUG { 05:05:55 DEBUG "metric": { 05:05:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:05:55 DEBUG }, 05:05:55 DEBUG "value": [ 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG [...] 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:05:55 DEBUG }, 05:05:55 DEBUG "value": [ 05:05:55 DEBUG 1713499555, 05:05:55 DEBUG "2003.9666666666667" 05:05:55 DEBUG ] 05:05:55 DEBUG } 05:05:55 DEBUG ] 05:05:55 DEBUG } 05:05:55 DEBUG } 05:05:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499555" 05:05:55 INFO [http_cmd]: http status code OK 05:05:55 DEBUG --- status code --- 05:05:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:05:55 DEBUG --- http response --- 05:05:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:05:55 DEBUG ----- output ----- 05:05:55 DEBUG { 05:05:55 DEBUG "status": "success", 05:05:55 DEBUG "data": { 05:05:55 DEBUG "resultType": "vector", 05:05:55 DEBUG "result": [ 05:05:55 DEBUG { 05:05:55 DEBUG "metric": { 05:05:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:05:55 DEBUG }, 05:05:55 DEBUG "value": [ 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG [...] 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:05:55 DEBUG }, 05:05:55 DEBUG "value": [ 05:05:55 DEBUG 1713499555, 05:05:55 DEBUG "2501921" 05:05:55 DEBUG ] 05:05:55 DEBUG } 05:05:55 DEBUG ] 05:05:55 DEBUG } 05:05:55 DEBUG } 05:05:55 INFO [http_cmd]: http status code OK 05:05:55 DEBUG --- status code --- 05:05:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:05:55 DEBUG --- http response --- 05:05:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:05:55 DEBUG ----- output ----- 05:05:55 DEBUG { 05:05:55 DEBUG "status": "success", 05:05:55 DEBUG "data": { 05:05:55 DEBUG "resultType": "vector", 05:05:55 DEBUG "result": [ 05:05:55 DEBUG { 05:05:55 DEBUG "metric": { 05:05:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:05:55 DEBUG }, 05:05:55 DEBUG "value": [ 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG [...] 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:05:55 DEBUG }, 05:05:55 DEBUG "value": [ 05:05:55 DEBUG 1713499555, 05:05:55 DEBUG "0" 05:05:55 DEBUG ] 05:05:55 DEBUG } 05:05:55 DEBUG ] 05:05:55 DEBUG } 05:05:55 DEBUG } 05:05:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499555" 05:05:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499555" 05:05:55 INFO [http_cmd]: http status code OK 05:05:55 DEBUG --- status code --- 05:05:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:05:55 DEBUG --- http response --- 05:05:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499555" 05:05:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:05:55 DEBUG ----- output ----- 05:05:55 DEBUG { 05:05:55 DEBUG "status": "success", 05:05:55 DEBUG "data": { 05:05:55 DEBUG "resultType": "vector", 05:05:55 DEBUG "result": [ 05:05:55 DEBUG { 05:05:55 DEBUG "metric": { 05:05:55 DEBUG "pod": "ds-cts-0" 05:05:55 DEBUG }, 05:05:55 DEBUG "value": [ 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG [...] 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG "pod": "ds-cts-2" 05:05:55 DEBUG }, 05:05:55 DEBUG "value": [ 05:05:55 DEBUG 1713499555, 05:05:55 DEBUG "0" 05:05:55 DEBUG ] 05:05:55 DEBUG } 05:05:55 DEBUG ] 05:05:55 DEBUG } 05:05:55 DEBUG } 05:05:55 INFO [http_cmd]: http status code OK 05:05:55 INFO [http_cmd]: http status code OK 05:05:55 DEBUG --- status code --- 05:05:55 DEBUG --- status code --- 05:05:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:05:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:05:55 DEBUG --- http response --- 05:05:55 DEBUG --- http response --- 05:05:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:05:55 DEBUG ----- output ----- 05:05:55 DEBUG { 05:05:55 DEBUG "status": "success", 05:05:55 DEBUG "data": { 05:05:55 DEBUG "resultType": "vector", 05:05:55 DEBUG "result": [ 05:05:55 DEBUG { 05:05:55 DEBUG "metric": { 05:05:55 DEBUG "pod": "ds-cts-0" 05:05:55 DEBUG }, 05:05:55 DEBUG "value": [ 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG [...] 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG "pod": "ds-cts-2" 05:05:55 DEBUG }, 05:05:55 DEBUG "value": [ 05:05:55 DEBUG 1713499555, 05:05:55 DEBUG "0.000033333333333333335" 05:05:55 DEBUG ] 05:05:55 DEBUG } 05:05:55 DEBUG ] 05:05:55 DEBUG } 05:05:55 DEBUG } 05:05:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:05:55 DEBUG ----- output ----- 05:05:55 DEBUG { 05:05:55 DEBUG "status": "success", 05:05:55 DEBUG "data": { 05:05:55 DEBUG "resultType": "vector", 05:05:55 DEBUG "result": [ 05:05:55 DEBUG { 05:05:55 DEBUG "metric": { 05:05:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:05:55 DEBUG }, 05:05:55 DEBUG "value": [ 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG [...] 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:05:55 DEBUG }, 05:05:55 DEBUG "value": [ 05:05:55 DEBUG 1713499555, 05:05:55 DEBUG "2003.8999999999999" 05:05:55 DEBUG ] 05:05:55 DEBUG } 05:05:55 DEBUG ] 05:05:55 DEBUG } 05:05:55 DEBUG } 05:05:55 INFO [http_cmd]: http status code OK 05:05:55 DEBUG --- status code --- 05:05:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:05:55 DEBUG --- http response --- 05:05:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:05:55 DEBUG ----- output ----- 05:05:55 DEBUG { 05:05:55 DEBUG "status": "success", 05:05:55 DEBUG "data": { 05:05:55 DEBUG "resultType": "vector", 05:05:55 DEBUG "result": [ 05:05:55 DEBUG { 05:05:55 DEBUG "metric": { 05:05:55 DEBUG "instance": "10.142.0.107:9100" 05:05:55 DEBUG }, 05:05:55 DEBUG "value": [ 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG [...] 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG "instance": "10.142.0.43:9100" 05:05:55 DEBUG }, 05:05:55 DEBUG "value": [ 05:05:55 DEBUG 1713499555, 05:05:55 DEBUG "0.06130000000000469" 05:05:55 DEBUG ] 05:05:55 DEBUG } 05:05:55 DEBUG ] 05:05:55 DEBUG } 05:05:55 DEBUG } 05:05:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499555" 05:05:55 INFO [http_cmd]: http status code OK 05:05:55 DEBUG --- status code --- 05:05:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:05:55 DEBUG --- http response --- 05:05:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:05:55 DEBUG ----- output ----- 05:05:55 DEBUG { 05:05:55 DEBUG "status": "success", 05:05:55 DEBUG "data": { 05:05:55 DEBUG "resultType": "vector", 05:05:55 DEBUG "result": [ 05:05:55 DEBUG { 05:05:55 DEBUG "metric": { 05:05:55 DEBUG "instance": "10.142.0.107:9100" 05:05:55 DEBUG }, 05:05:55 DEBUG "value": [ 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG [...] 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG "instance": "10.142.0.43:9100" 05:05:55 DEBUG }, 05:05:55 DEBUG "value": [ 05:05:55 DEBUG 1713499555, 05:05:55 DEBUG "47843885.93333334" 05:05:55 DEBUG ] 05:05:55 DEBUG } 05:05:55 DEBUG ] 05:05:55 DEBUG } 05:05:55 DEBUG } 05:05:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499555" 05:05:55 INFO [http_cmd]: http status code OK 05:05:55 DEBUG --- status code --- 05:05:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:05:55 DEBUG --- http response --- 05:05:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:05:55 DEBUG ----- output ----- 05:05:55 DEBUG { 05:05:55 DEBUG "status": "success", 05:05:55 DEBUG "data": { 05:05:55 DEBUG "resultType": "vector", 05:05:55 DEBUG "result": [ 05:05:55 DEBUG { 05:05:55 DEBUG "metric": { 05:05:55 DEBUG "pod": "ds-idrepo-0" 05:05:55 DEBUG }, 05:05:55 DEBUG "value": [ 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG [...] 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG "pod": "ds-idrepo-2" 05:05:55 DEBUG }, 05:05:55 DEBUG "value": [ 05:05:55 DEBUG 1713499555, 05:05:55 DEBUG "0" 05:05:55 DEBUG ] 05:05:55 DEBUG } 05:05:55 DEBUG ] 05:05:55 DEBUG } 05:05:55 DEBUG } 05:05:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713499555" 05:05:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499555" 05:05:55 INFO [http_cmd]: http status code OK 05:05:55 INFO [http_cmd]: http status code OK 05:05:55 DEBUG --- status code --- 05:05:55 DEBUG --- status code --- 05:05:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:05:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:05:55 DEBUG --- http response --- 05:05:55 DEBUG --- http response --- 05:05:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:05:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:05:55 DEBUG ----- output ----- 05:05:55 DEBUG ----- output ----- 05:05:55 DEBUG { 05:05:55 DEBUG { 05:05:55 DEBUG "status": "success", 05:05:55 DEBUG "status": "success", 05:05:55 DEBUG "data": { 05:05:55 DEBUG "data": { 05:05:55 DEBUG "resultType": "vector", 05:05:55 DEBUG "resultType": "vector", 05:05:55 DEBUG "result": [ 05:05:55 DEBUG "result": [ 05:05:55 DEBUG { 05:05:55 DEBUG { 05:05:55 DEBUG "metric": { 05:05:55 DEBUG "metric": { 05:05:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:05:55 DEBUG "pod": "ds-idrepo-0" 05:05:55 DEBUG }, 05:05:55 DEBUG }, 05:05:55 DEBUG "value": [ 05:05:55 DEBUG "value": [ 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG [...] 05:05:55 DEBUG [...] 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:05:55 DEBUG "pod": "ds-idrepo-2" 05:05:55 DEBUG }, 05:05:55 DEBUG }, 05:05:55 DEBUG "value": [ 05:05:55 DEBUG "value": [ 05:05:55 DEBUG 1713499555, 05:05:55 DEBUG 1713499555, 05:05:55 DEBUG "55.171" 05:05:55 DEBUG "0" 05:05:55 DEBUG ] 05:05:55 DEBUG ] 05:05:55 DEBUG } 05:05:55 DEBUG } 05:05:55 DEBUG ] 05:05:55 DEBUG ] 05:05:55 DEBUG } 05:05:55 DEBUG } 05:05:55 DEBUG } 05:05:55 DEBUG } 05:05:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499555" 05:05:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499555" 05:05:55 INFO [http_cmd]: http status code OK 05:05:55 DEBUG --- status code --- 05:05:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:05:55 DEBUG --- http response --- 05:05:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:05:55 DEBUG ----- output ----- 05:05:55 DEBUG { 05:05:55 DEBUG "status": "success", 05:05:55 DEBUG "data": { 05:05:55 DEBUG "resultType": "vector", 05:05:55 DEBUG "result": [ 05:05:55 DEBUG { 05:05:55 DEBUG "metric": { 05:05:55 DEBUG "instance": "10.142.0.107:9100" 05:05:55 DEBUG }, 05:05:55 DEBUG "value": [ 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG [...] 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG "instance": "10.142.0.43:9100" 05:05:55 DEBUG }, 05:05:55 DEBUG "value": [ 05:05:55 DEBUG 1713499555, 05:05:55 DEBUG "0.04233333333333368" 05:05:55 DEBUG ] 05:05:55 DEBUG } 05:05:55 DEBUG ] 05:05:55 DEBUG } 05:05:55 DEBUG } 05:05:55 INFO [http_cmd]: http status code OK 05:05:55 DEBUG --- status code --- 05:05:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:05:55 DEBUG --- http response --- 05:05:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:05:55 DEBUG ----- output ----- 05:05:55 DEBUG { 05:05:55 DEBUG "status": "success", 05:05:55 DEBUG "data": { 05:05:55 DEBUG "resultType": "vector", 05:05:55 DEBUG "result": [ 05:05:55 DEBUG { 05:05:55 DEBUG "metric": { 05:05:55 DEBUG "instance": "10.142.0.107:9100" 05:05:55 DEBUG }, 05:05:55 DEBUG "value": [ 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG [...] 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG 05:05:55 DEBUG "instance": "10.142.0.43:9100" 05:05:55 DEBUG }, 05:05:55 DEBUG "value": [ 05:05:55 DEBUG 1713499555, 05:05:55 DEBUG "47840596.83333333" 05:05:55 DEBUG ] 05:05:55 DEBUG } 05:05:55 DEBUG ] 05:05:55 DEBUG } 05:05:55 DEBUG } 05:05:56 INFO 05:05:56 INFO [loop_until]: kubectl --namespace=xlou top pods 05:05:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:05:56 INFO [loop_until]: OK (rc = 0) 05:05:56 DEBUG --- stdout --- 05:05:56 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 9963m 9122Mi am-6fd554f8f7-hvvkw 9459m 8875Mi am-6fd554f8f7-pmj4b 10124m 8974Mi ds-cts-0 7566m 12248Mi ds-cts-1 7346m 12170Mi ds-cts-2 7783m 12207Mi ds-idrepo-0 4760m 3255Mi ds-idrepo-1 4557m 3207Mi ds-idrepo-2 4546m 3065Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 4m 1215Mi lodemon-6b89d774d5-rf7sk 9m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 6686m 1976Mi 05:05:56 DEBUG --- stderr --- 05:05:56 DEBUG 05:05:56 INFO 05:05:56 INFO [loop_until]: kubectl --namespace=xlou top node 05:05:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:05:56 INFO [loop_until]: OK (rc = 0) 05:05:56 DEBUG --- stdout --- 05:05:56 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 75m 0% 6264Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 4638m 15% 6703Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9444m 31% 11582Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 82m 0% 3253Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 7751m 25% 14784Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 153m 0% 5165Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 9618m 32% 11907Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 10293m 34% 2726Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 11083m 37% 11165Mi 10% gke-xlou-cdm-ds-0f601751-j488 4880m 16% 6159Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 7254m 24% 16183Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 5106m 17% 6881Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 7430m 24% 14184Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 6780m 42% 5724Mi 9% 05:05:56 DEBUG --- stderr --- 05:05:56 DEBUG 05:06:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499615" 05:06:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713499615" 05:06:55 ERROR [http_cmd]: ERROR 05:06:55 INFO [http_cmd]: http status code OK 05:06:55 ERROR --- status code --- 05:06:55 DEBUG --- status code --- 05:06:55 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:06:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:06:55 ERROR --- http response --- 05:06:55 DEBUG --- http response --- 05:06:55 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:06:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:06:55 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:06:55 DEBUG ----- output ----- 05:06:55 DEBUG { 05:06:55 DEBUG "status": "success", 05:06:55 DEBUG "data": { 05:06:55 DEBUG "resultType": "vector", 05:06:55 DEBUG "result": [ 05:06:55 DEBUG { 05:06:55 DEBUG "metric": { 05:06:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:06:55 DEBUG }, 05:06:55 DEBUG "value": [ 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG [...] 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:06:55 DEBUG }, 05:06:55 DEBUG "value": [ 05:06:55 DEBUG 1713499615, 05:06:55 DEBUG "0" 05:06:55 DEBUG ] 05:06:55 DEBUG } 05:06:55 DEBUG ] 05:06:55 DEBUG } 05:06:55 DEBUG } 05:06:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499615" 05:06:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713499615" 05:06:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499615" 05:06:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499615" 05:06:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499615" 05:06:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499615" 05:06:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713499615" 05:06:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499615" 05:06:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499615" 05:06:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499615" 05:06:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499615" 05:06:55 INFO [http_cmd]: http status code OK 05:06:55 DEBUG --- status code --- 05:06:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:06:55 INFO [http_cmd]: http status code OK 05:06:55 INFO [http_cmd]: http status code OK 05:06:55 DEBUG --- http response --- 05:06:55 INFO [http_cmd]: http status code OK 05:06:55 DEBUG --- status code --- 05:06:55 DEBUG --- status code --- 05:06:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:06:55 DEBUG --- status code --- 05:06:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:06:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:06:55 DEBUG ----- output ----- 05:06:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:06:55 DEBUG --- http response --- 05:06:55 DEBUG --- http response --- 05:06:55 INFO [http_cmd]: http status code OK 05:06:55 INFO [http_cmd]: http status code OK 05:06:55 INFO [http_cmd]: http status code OK 05:06:55 INFO [http_cmd]: http status code OK 05:06:55 DEBUG { 05:06:55 DEBUG --- http response --- 05:06:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 05:06:55 DEBUG --- status code --- 05:06:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:06:55 DEBUG --- status code --- 05:06:55 INFO [http_cmd]: http status code OK 05:06:55 INFO [http_cmd]: http status code OK 05:06:55 INFO [http_cmd]: http status code OK 05:06:55 DEBUG --- status code --- 05:06:55 DEBUG --- status code --- 05:06:55 DEBUG "status": "success", 05:06:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:06:55 DEBUG ----- output ----- 05:06:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:06:55 DEBUG ----- output ----- 05:06:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:06:55 DEBUG --- status code --- 05:06:55 DEBUG --- status code --- 05:06:55 DEBUG --- status code --- 05:06:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:06:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:06:55 DEBUG "data": { 05:06:55 DEBUG ----- output ----- 05:06:55 DEBUG { 05:06:55 DEBUG --- http response --- 05:06:55 DEBUG { 05:06:55 DEBUG --- http response --- 05:06:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:06:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:06:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:06:55 DEBUG --- http response --- 05:06:55 DEBUG --- http response --- 05:06:55 DEBUG "resultType": "vector", 05:06:55 DEBUG { 05:06:55 DEBUG "status": "success", 05:06:55 DEBUG "status": "success", 05:06:55 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:06:55 DEBUG --- http response --- 05:06:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 05:06:55 DEBUG --- http response --- 05:06:55 DEBUG --- http response --- 05:06:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:06:55 DEBUG "result": [ 05:06:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:06:55 DEBUG "status": "success", 05:06:55 DEBUG "data": { 05:06:55 DEBUG "data": { 05:06:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:06:55 DEBUG ----- output ----- 05:06:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:06:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 05:06:55 DEBUG ----- output ----- 05:06:55 DEBUG { 05:06:55 DEBUG ----- output ----- 05:06:55 DEBUG "data": { 05:06:55 DEBUG "resultType": "vector", 05:06:55 DEBUG "resultType": "vector", 05:06:55 DEBUG ----- output ----- 05:06:55 DEBUG { 05:06:55 DEBUG ----- output ----- 05:06:55 DEBUG ----- output ----- 05:06:55 DEBUG { 05:06:55 DEBUG "metric": { 05:06:55 DEBUG { 05:06:55 DEBUG "resultType": "vector", 05:06:55 DEBUG "result": [ 05:06:55 DEBUG "result": [ 05:06:55 DEBUG { 05:06:55 DEBUG "status": "success", 05:06:55 DEBUG { 05:06:55 DEBUG { 05:06:55 DEBUG "status": "success", 05:06:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:06:55 DEBUG "status": "success", 05:06:55 DEBUG "result": [ 05:06:55 DEBUG { 05:06:55 DEBUG { 05:06:55 DEBUG "status": "success", 05:06:55 DEBUG "data": { 05:06:55 DEBUG "status": "success", 05:06:55 DEBUG "status": "success", 05:06:55 DEBUG "data": { 05:06:55 DEBUG }, 05:06:55 DEBUG "data": { 05:06:55 DEBUG { 05:06:55 DEBUG "metric": { 05:06:55 DEBUG "metric": { 05:06:55 DEBUG "data": { 05:06:55 DEBUG "resultType": "vector", 05:06:55 DEBUG "data": { 05:06:55 DEBUG "data": { 05:06:55 DEBUG "resultType": "vector", 05:06:55 DEBUG "value": [ 05:06:55 DEBUG "resultType": "vector", 05:06:55 DEBUG "metric": { 05:06:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:06:55 DEBUG "pod": "ds-cts-0" 05:06:55 DEBUG "resultType": "vector", 05:06:55 DEBUG "result": [ 05:06:55 DEBUG "resultType": "vector", 05:06:55 DEBUG "resultType": "vector", 05:06:55 DEBUG "result": [ 05:06:55 DEBUG 05:06:55 DEBUG "result": [ 05:06:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:06:55 DEBUG }, 05:06:55 DEBUG }, 05:06:55 DEBUG "result": [ 05:06:55 DEBUG { 05:06:55 DEBUG "result": [ 05:06:55 DEBUG "result": [ 05:06:55 DEBUG { 05:06:55 DEBUG 05:06:55 DEBUG { 05:06:55 DEBUG }, 05:06:55 DEBUG "value": [ 05:06:55 DEBUG "value": [ 05:06:55 DEBUG { 05:06:55 DEBUG "metric": { 05:06:55 DEBUG { 05:06:55 DEBUG { 05:06:55 DEBUG "metric": { 05:06:55 DEBUG 05:06:55 DEBUG "metric": { 05:06:55 DEBUG "value": [ 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG "metric": { 05:06:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:06:55 DEBUG "metric": { 05:06:55 DEBUG "metric": { 05:06:55 DEBUG "pod": "ds-idrepo-0" 05:06:55 DEBUG [...] 05:06:55 DEBUG "pod": "ds-idrepo-0" 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:06:55 DEBUG }, 05:06:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:06:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:06:55 DEBUG }, 05:06:55 DEBUG 05:06:55 DEBUG }, 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG }, 05:06:55 DEBUG "value": [ 05:06:55 DEBUG }, 05:06:55 DEBUG }, 05:06:55 DEBUG "value": [ 05:06:55 DEBUG 05:06:55 DEBUG "value": [ 05:06:55 DEBUG 05:06:55 DEBUG [...] 05:06:55 DEBUG [...] 05:06:55 DEBUG "value": [ 05:06:55 DEBUG 05:06:55 DEBUG "value": [ 05:06:55 DEBUG "value": [ 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG [...] 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG }, 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG [...] 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG [...] 05:06:55 DEBUG "value": [ 05:06:55 DEBUG [...] 05:06:55 DEBUG 05:06:55 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:06:55 DEBUG "pod": "ds-cts-2" 05:06:55 DEBUG [...] 05:06:55 DEBUG 05:06:55 DEBUG [...] 05:06:55 DEBUG [...] 05:06:55 DEBUG 05:06:55 DEBUG 1713499615, 05:06:55 DEBUG 05:06:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:06:55 DEBUG }, 05:06:55 DEBUG }, 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG "2000.6666666666667" 05:06:55 DEBUG 05:06:55 DEBUG }, 05:06:55 DEBUG "value": [ 05:06:55 DEBUG "value": [ 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG ] 05:06:55 DEBUG 05:06:55 DEBUG "value": [ 05:06:55 DEBUG 1713499615, 05:06:55 DEBUG 1713499615, 05:06:55 DEBUG 05:06:55 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG "pod": "ds-cts-2" 05:06:55 DEBUG } 05:06:55 DEBUG "pod": "ds-idrepo-2" 05:06:55 DEBUG 1713499615, 05:06:55 DEBUG "21804.46877973794" 05:06:55 DEBUG "1.3" 05:06:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:06:55 DEBUG }, 05:06:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:06:55 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:06:55 DEBUG }, 05:06:55 DEBUG ] 05:06:55 DEBUG }, 05:06:55 DEBUG "2170290176" 05:06:55 DEBUG ] 05:06:55 DEBUG ] 05:06:55 DEBUG }, 05:06:55 DEBUG "value": [ 05:06:55 DEBUG }, 05:06:55 DEBUG }, 05:06:55 DEBUG "value": [ 05:06:55 DEBUG } 05:06:55 DEBUG "value": [ 05:06:55 DEBUG ] 05:06:55 DEBUG } 05:06:55 DEBUG } 05:06:55 DEBUG "value": [ 05:06:55 DEBUG 1713499615, 05:06:55 DEBUG "value": [ 05:06:55 DEBUG "value": [ 05:06:55 DEBUG 1713499615, 05:06:55 DEBUG } 05:06:55 DEBUG 1713499615, 05:06:55 DEBUG } 05:06:55 DEBUG ] 05:06:55 DEBUG ] 05:06:55 DEBUG 1713499615, 05:06:55 DEBUG "0" 05:06:55 DEBUG 1713499615, 05:06:55 DEBUG 1713499615, 05:06:55 DEBUG "0" 05:06:55 DEBUG "0" 05:06:55 DEBUG ] 05:06:55 DEBUG } 05:06:55 DEBUG } 05:06:55 DEBUG "0" 05:06:55 DEBUG ] 05:06:55 DEBUG } 05:06:55 DEBUG "0" 05:06:55 DEBUG ] 05:06:55 DEBUG ] 05:06:55 DEBUG } 05:06:55 DEBUG } 05:06:55 DEBUG } 05:06:55 DEBUG ] 05:06:55 DEBUG "2000.7666666666667" 05:06:55 DEBUG ] 05:06:55 DEBUG } 05:06:55 DEBUG } 05:06:55 DEBUG } 05:06:55 DEBUG } 05:06:55 DEBUG } 05:06:55 DEBUG ] 05:06:55 DEBUG ] 05:06:55 DEBUG } 05:06:55 DEBUG ] 05:06:55 DEBUG ] 05:06:55 DEBUG ] 05:06:55 DEBUG } 05:06:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499615" 05:06:55 DEBUG } 05:06:55 DEBUG } 05:06:55 DEBUG } 05:06:55 DEBUG } 05:06:55 DEBUG } 05:06:55 DEBUG ] 05:06:55 DEBUG } 05:06:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499615" 05:06:55 DEBUG } 05:06:55 DEBUG ] 05:06:55 DEBUG } 05:06:55 DEBUG } 05:06:55 DEBUG } 05:06:55 DEBUG } 05:06:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713499615" 05:06:55 INFO [http_cmd]: http status code OK 05:06:55 INFO [http_cmd]: http status code OK 05:06:55 DEBUG --- status code --- 05:06:55 DEBUG --- status code --- 05:06:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:06:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:06:55 DEBUG --- http response --- 05:06:55 DEBUG --- http response --- 05:06:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 05:06:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:06:55 DEBUG ----- output ----- 05:06:55 DEBUG ----- output ----- 05:06:55 DEBUG { 05:06:55 DEBUG { 05:06:55 DEBUG "status": "success", 05:06:55 DEBUG "status": "success", 05:06:55 DEBUG "data": { 05:06:55 DEBUG "data": { 05:06:55 DEBUG "resultType": "vector", 05:06:55 DEBUG "resultType": "vector", 05:06:55 DEBUG "result": [ 05:06:55 DEBUG "result": [ 05:06:55 DEBUG { 05:06:55 DEBUG { 05:06:55 DEBUG "metric": { 05:06:55 DEBUG "metric": { 05:06:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:06:55 DEBUG "instance": "10.142.0.107:9100" 05:06:55 DEBUG }, 05:06:55 DEBUG }, 05:06:55 DEBUG "value": [ 05:06:55 DEBUG "value": [ 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG [...] 05:06:55 DEBUG [...] 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:06:55 DEBUG }, 05:06:55 DEBUG "instance": "10.142.0.43:9100" 05:06:55 DEBUG "value": [ 05:06:55 DEBUG }, 05:06:55 DEBUG 1713499615, 05:06:55 DEBUG "value": [ 05:06:55 DEBUG "2.8567255255952" 05:06:55 DEBUG 1713499615, 05:06:55 DEBUG ] 05:06:55 DEBUG "437138617344" 05:06:55 DEBUG } 05:06:55 DEBUG ] 05:06:55 DEBUG ] 05:06:55 DEBUG } 05:06:55 DEBUG } 05:06:55 DEBUG ] 05:06:55 DEBUG } 05:06:55 DEBUG } 05:06:55 DEBUG } 05:06:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499615" 05:06:55 INFO [http_cmd]: http status code OK 05:06:55 DEBUG --- status code --- 05:06:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:06:55 DEBUG --- http response --- 05:06:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:06:55 DEBUG ----- output ----- 05:06:55 DEBUG { 05:06:55 DEBUG "status": "success", 05:06:55 DEBUG "data": { 05:06:55 DEBUG "resultType": "vector", 05:06:55 DEBUG "result": [ 05:06:55 DEBUG { 05:06:55 DEBUG "metric": { 05:06:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:06:55 DEBUG }, 05:06:55 DEBUG "value": [ 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG [...] 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:06:55 DEBUG }, 05:06:55 DEBUG "value": [ 05:06:55 DEBUG 1713499615, 05:06:55 DEBUG "6.555176021251007" 05:06:55 DEBUG ] 05:06:55 DEBUG } 05:06:55 DEBUG ] 05:06:55 DEBUG } 05:06:55 DEBUG } 05:06:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499615" 05:06:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499615" 05:06:55 INFO [http_cmd]: http status code OK 05:06:55 DEBUG --- status code --- 05:06:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:06:55 DEBUG --- http response --- 05:06:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:06:55 DEBUG ----- output ----- 05:06:55 DEBUG { 05:06:55 DEBUG "status": "success", 05:06:55 DEBUG "data": { 05:06:55 DEBUG "resultType": "vector", 05:06:55 DEBUG "result": [ 05:06:55 DEBUG { 05:06:55 DEBUG "metric": { 05:06:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:06:55 DEBUG }, 05:06:55 DEBUG "value": [ 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG [...] 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:06:55 DEBUG }, 05:06:55 DEBUG "value": [ 05:06:55 DEBUG 1713499615, 05:06:55 DEBUG "2622383" 05:06:55 DEBUG ] 05:06:55 DEBUG } 05:06:55 DEBUG ] 05:06:55 DEBUG } 05:06:55 DEBUG } 05:06:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499615" 05:06:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499615" 05:06:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499615" 05:06:55 INFO [http_cmd]: http status code OK 05:06:55 INFO [http_cmd]: http status code OK 05:06:55 DEBUG --- status code --- 05:06:55 DEBUG --- status code --- 05:06:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:06:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:06:55 DEBUG --- http response --- 05:06:55 DEBUG --- http response --- 05:06:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:06:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:06:55 DEBUG ----- output ----- 05:06:55 DEBUG ----- output ----- 05:06:55 DEBUG { 05:06:55 DEBUG { 05:06:55 DEBUG "status": "success", 05:06:55 DEBUG "status": "success", 05:06:55 DEBUG "data": { 05:06:55 DEBUG "data": { 05:06:55 DEBUG "resultType": "vector", 05:06:55 DEBUG "resultType": "vector", 05:06:55 DEBUG "result": [ 05:06:55 DEBUG "result": [ 05:06:55 DEBUG { 05:06:55 DEBUG { 05:06:55 DEBUG "metric": { 05:06:55 DEBUG "metric": { 05:06:55 DEBUG "pod": "ds-cts-0" 05:06:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:06:55 DEBUG }, 05:06:55 DEBUG }, 05:06:55 DEBUG "value": [ 05:06:55 DEBUG "value": [ 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG [...] 05:06:55 DEBUG 05:06:55 DEBUG [...] 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG "pod": "ds-cts-2" 05:06:55 DEBUG 05:06:55 DEBUG }, 05:06:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:06:55 DEBUG "value": [ 05:06:55 DEBUG }, 05:06:55 DEBUG 1713499615, 05:06:55 DEBUG "value": [ 05:06:55 DEBUG "0" 05:06:55 DEBUG 1713499615, 05:06:55 INFO [http_cmd]: http status code OK 05:06:55 DEBUG ] 05:06:55 DEBUG "2000.8333333333333" 05:06:55 DEBUG --- status code --- 05:06:55 DEBUG } 05:06:55 DEBUG ] 05:06:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:06:55 DEBUG ] 05:06:55 DEBUG } 05:06:55 DEBUG --- http response --- 05:06:55 DEBUG } 05:06:55 DEBUG ] 05:06:55 DEBUG } 05:06:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:06:55 DEBUG } 05:06:55 DEBUG ----- output ----- 05:06:55 DEBUG } 05:06:55 DEBUG { 05:06:55 DEBUG "status": "success", 05:06:55 DEBUG "data": { 05:06:55 DEBUG "resultType": "vector", 05:06:55 DEBUG "result": [ 05:06:55 DEBUG { 05:06:55 DEBUG "metric": { 05:06:55 DEBUG "pod": "ds-cts-0" 05:06:55 DEBUG }, 05:06:55 DEBUG "value": [ 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG [...] 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG "pod": "ds-cts-2" 05:06:55 DEBUG }, 05:06:55 DEBUG "value": [ 05:06:55 DEBUG 1713499615, 05:06:55 DEBUG "0.000017660000000000004" 05:06:55 DEBUG ] 05:06:55 DEBUG } 05:06:55 DEBUG ] 05:06:55 DEBUG } 05:06:55 DEBUG } 05:06:55 INFO [http_cmd]: http status code OK 05:06:55 DEBUG --- status code --- 05:06:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:06:55 DEBUG --- http response --- 05:06:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:06:55 DEBUG ----- output ----- 05:06:55 DEBUG { 05:06:55 DEBUG "status": "success", 05:06:55 DEBUG "data": { 05:06:55 DEBUG "resultType": "vector", 05:06:55 DEBUG "result": [ 05:06:55 DEBUG { 05:06:55 DEBUG "metric": { 05:06:55 DEBUG "pod": "ds-idrepo-0" 05:06:55 DEBUG }, 05:06:55 DEBUG "value": [ 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG [...] 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG "pod": "ds-idrepo-2" 05:06:55 DEBUG }, 05:06:55 DEBUG "value": [ 05:06:55 DEBUG 1713499615, 05:06:55 DEBUG "0" 05:06:55 DEBUG ] 05:06:55 DEBUG } 05:06:55 DEBUG ] 05:06:55 DEBUG } 05:06:55 DEBUG } 05:06:55 INFO [http_cmd]: http status code OK 05:06:55 DEBUG --- status code --- 05:06:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:06:55 DEBUG --- http response --- 05:06:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:06:55 DEBUG ----- output ----- 05:06:55 DEBUG { 05:06:55 DEBUG "status": "success", 05:06:55 DEBUG "data": { 05:06:55 DEBUG "resultType": "vector", 05:06:55 DEBUG "result": [ 05:06:55 DEBUG { 05:06:55 DEBUG "metric": { 05:06:55 DEBUG "instance": "10.142.0.107:9100" 05:06:55 DEBUG }, 05:06:55 DEBUG "value": [ 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG [...] 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG "instance": "10.142.0.43:9100" 05:06:55 DEBUG }, 05:06:55 DEBUG "value": [ 05:06:55 DEBUG 1713499615, 05:06:55 DEBUG "0.05843333333333476" 05:06:55 DEBUG ] 05:06:55 DEBUG } 05:06:55 DEBUG ] 05:06:55 DEBUG } 05:06:55 DEBUG } 05:06:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499615" 05:06:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499615" 05:06:55 INFO [http_cmd]: http status code OK 05:06:55 DEBUG --- status code --- 05:06:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:06:55 DEBUG --- http response --- 05:06:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:06:55 DEBUG ----- output ----- 05:06:55 DEBUG { 05:06:55 DEBUG "status": "success", 05:06:55 DEBUG "data": { 05:06:55 DEBUG "resultType": "vector", 05:06:55 DEBUG "result": [ 05:06:55 DEBUG { 05:06:55 DEBUG "metric": { 05:06:55 DEBUG "pod": "ds-idrepo-0" 05:06:55 DEBUG }, 05:06:55 DEBUG "value": [ 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG [...] 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG "pod": "ds-idrepo-2" 05:06:55 DEBUG }, 05:06:55 DEBUG "value": [ 05:06:55 DEBUG 1713499615, 05:06:55 DEBUG "0" 05:06:55 DEBUG ] 05:06:55 DEBUG } 05:06:55 DEBUG ] 05:06:55 DEBUG } 05:06:55 DEBUG } 05:06:55 INFO [http_cmd]: http status code OK 05:06:55 DEBUG --- status code --- 05:06:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:06:55 DEBUG --- http response --- 05:06:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:06:55 DEBUG ----- output ----- 05:06:55 DEBUG { 05:06:55 DEBUG "status": "success", 05:06:55 DEBUG "data": { 05:06:55 DEBUG "resultType": "vector", 05:06:55 DEBUG "result": [ 05:06:55 DEBUG { 05:06:55 DEBUG "metric": { 05:06:55 DEBUG "instance": "10.142.0.107:9100" 05:06:55 DEBUG }, 05:06:55 DEBUG "value": [ 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG [...] 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG "instance": "10.142.0.43:9100" 05:06:55 DEBUG }, 05:06:55 DEBUG "value": [ 05:06:55 DEBUG 1713499615, 05:06:55 DEBUG "47745250.46666667" 05:06:55 DEBUG ] 05:06:55 DEBUG } 05:06:55 DEBUG ] 05:06:55 DEBUG } 05:06:55 DEBUG } 05:06:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713499615" 05:06:55 INFO [http_cmd]: http status code OK 05:06:55 DEBUG --- status code --- 05:06:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:06:55 DEBUG --- http response --- 05:06:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:06:55 DEBUG ----- output ----- 05:06:55 DEBUG { 05:06:55 DEBUG "status": "success", 05:06:55 DEBUG "data": { 05:06:55 DEBUG "resultType": "vector", 05:06:55 DEBUG "result": [ 05:06:55 DEBUG { 05:06:55 DEBUG "metric": { 05:06:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:06:55 DEBUG }, 05:06:55 DEBUG "value": [ 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG [...] 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:06:55 DEBUG }, 05:06:55 DEBUG "value": [ 05:06:55 DEBUG 1713499615, 05:06:55 DEBUG "55.171" 05:06:55 DEBUG ] 05:06:55 DEBUG } 05:06:55 DEBUG ] 05:06:55 DEBUG } 05:06:55 DEBUG } 05:06:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499615" 05:06:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499615" 05:06:55 INFO [http_cmd]: http status code OK 05:06:55 DEBUG --- status code --- 05:06:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:06:55 DEBUG --- http response --- 05:06:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:06:55 DEBUG ----- output ----- 05:06:55 DEBUG { 05:06:55 DEBUG "status": "success", 05:06:55 DEBUG "data": { 05:06:55 DEBUG "resultType": "vector", 05:06:55 DEBUG "result": [ 05:06:55 DEBUG { 05:06:55 DEBUG "metric": { 05:06:55 DEBUG "instance": "10.142.0.107:9100" 05:06:55 DEBUG }, 05:06:55 DEBUG "value": [ 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG [...] 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG "instance": "10.142.0.43:9100" 05:06:55 DEBUG }, 05:06:55 DEBUG "value": [ 05:06:55 DEBUG 1713499615, 05:06:55 DEBUG "0.03933333333333449" 05:06:55 DEBUG ] 05:06:55 DEBUG } 05:06:55 DEBUG ] 05:06:55 DEBUG } 05:06:55 DEBUG } 05:06:55 INFO [http_cmd]: http status code OK 05:06:55 DEBUG --- status code --- 05:06:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:06:55 DEBUG --- http response --- 05:06:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:06:55 DEBUG ----- output ----- 05:06:55 DEBUG { 05:06:55 DEBUG "status": "success", 05:06:55 DEBUG "data": { 05:06:55 DEBUG "resultType": "vector", 05:06:55 DEBUG "result": [ 05:06:55 DEBUG { 05:06:55 DEBUG "metric": { 05:06:55 DEBUG "instance": "10.142.0.107:9100" 05:06:55 DEBUG }, 05:06:55 DEBUG "value": [ 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG [...] 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG 05:06:55 DEBUG "instance": "10.142.0.43:9100" 05:06:55 DEBUG }, 05:06:55 DEBUG "value": [ 05:06:55 DEBUG 1713499615, 05:06:55 DEBUG "47743543.666666664" 05:06:55 DEBUG ] 05:06:55 DEBUG } 05:06:55 DEBUG ] 05:06:55 DEBUG } 05:06:55 DEBUG } 05:06:56 INFO 05:06:56 INFO [loop_until]: kubectl --namespace=xlou top pods 05:06:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:06:56 INFO [loop_until]: OK (rc = 0) 05:06:56 DEBUG --- stdout --- 05:06:56 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 9968m 9121Mi am-6fd554f8f7-hvvkw 9494m 8875Mi am-6fd554f8f7-pmj4b 10080m 8974Mi ds-cts-0 7626m 12254Mi ds-cts-1 7426m 12176Mi ds-cts-2 7596m 12214Mi ds-idrepo-0 4715m 3255Mi ds-idrepo-1 4721m 3206Mi ds-idrepo-2 4685m 3065Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 12m 1215Mi lodemon-6b89d774d5-rf7sk 13m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 6632m 1978Mi 05:06:56 DEBUG --- stderr --- 05:06:56 DEBUG 05:06:56 INFO 05:06:56 INFO [loop_until]: kubectl --namespace=xlou top node 05:06:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:06:56 INFO [loop_until]: OK (rc = 0) 05:06:56 DEBUG --- stdout --- 05:06:56 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 81m 0% 6257Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 4493m 15% 6709Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9414m 31% 11588Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 3254Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 7778m 26% 14811Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 155m 0% 5167Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 9733m 32% 11910Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 10386m 34% 2730Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 11148m 37% 11177Mi 10% gke-xlou-cdm-ds-0f601751-j488 4906m 16% 6163Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 7227m 24% 16195Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 5143m 17% 6883Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 7524m 25% 14194Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 6679m 42% 5725Mi 9% 05:06:56 DEBUG --- stderr --- 05:06:56 DEBUG 05:07:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713499675" 05:07:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499675" 05:07:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713499675" 05:07:55 ERROR [http_cmd]: ERROR 05:07:55 INFO [http_cmd]: http status code OK 05:07:55 ERROR --- status code --- 05:07:55 DEBUG --- status code --- 05:07:55 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:07:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:07:55 ERROR --- http response --- 05:07:55 DEBUG --- http response --- 05:07:55 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:07:55 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:07:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:07:55 DEBUG ----- output ----- 05:07:55 DEBUG { 05:07:55 DEBUG "status": "success", 05:07:55 DEBUG "data": { 05:07:55 DEBUG "resultType": "vector", 05:07:55 DEBUG "result": [ 05:07:55 DEBUG { 05:07:55 DEBUG "metric": { 05:07:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:07:55 DEBUG }, 05:07:55 DEBUG "value": [ 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG [...] 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:07:55 DEBUG }, 05:07:55 DEBUG "value": [ 05:07:55 DEBUG 1713499675, 05:07:55 DEBUG "0" 05:07:55 DEBUG ] 05:07:55 DEBUG } 05:07:55 DEBUG ] 05:07:55 DEBUG } 05:07:55 DEBUG } 05:07:55 INFO [http_cmd]: http status code OK 05:07:55 DEBUG --- status code --- 05:07:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:07:55 DEBUG --- http response --- 05:07:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:07:55 DEBUG ----- output ----- 05:07:55 DEBUG { 05:07:55 DEBUG "status": "success", 05:07:55 DEBUG "data": { 05:07:55 DEBUG "resultType": "vector", 05:07:55 DEBUG "result": [ 05:07:55 DEBUG { 05:07:55 DEBUG "metric": { 05:07:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:07:55 DEBUG }, 05:07:55 DEBUG "value": [ 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG [...] 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:07:55 DEBUG }, 05:07:55 DEBUG "value": [ 05:07:55 DEBUG 1713499675, 05:07:55 DEBUG "6.53272909954911" 05:07:55 DEBUG ] 05:07:55 DEBUG } 05:07:55 DEBUG ] 05:07:55 DEBUG } 05:07:55 DEBUG } 05:07:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499675" 05:07:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499675" 05:07:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499675" 05:07:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499675" 05:07:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499675" 05:07:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499675" 05:07:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499675" 05:07:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713499675" 05:07:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499675" 05:07:55 INFO [http_cmd]: http status code OK 05:07:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713499675" 05:07:55 DEBUG --- status code --- 05:07:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499675" 05:07:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:07:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499675" 05:07:55 DEBUG --- http response --- 05:07:55 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:07:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499675" 05:07:55 INFO [http_cmd]: http status code OK 05:07:55 DEBUG --- status code --- 05:07:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:07:55 DEBUG --- http response --- 05:07:55 INFO [http_cmd]: http status code OK 05:07:55 INFO [http_cmd]: http status code OK 05:07:55 INFO [http_cmd]: http status code OK 05:07:55 INFO [http_cmd]: http status code OK 05:07:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:07:55 DEBUG --- status code --- 05:07:55 INFO [http_cmd]: http status code OK 05:07:55 DEBUG --- status code --- 05:07:55 DEBUG --- status code --- 05:07:55 DEBUG --- status code --- 05:07:55 DEBUG ----- output ----- 05:07:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:07:55 DEBUG --- status code --- 05:07:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:07:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:07:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:07:55 DEBUG { 05:07:55 DEBUG --- http response --- 05:07:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:07:55 DEBUG --- http response --- 05:07:55 DEBUG --- http response --- 05:07:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499675" 05:07:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499675" 05:07:55 DEBUG --- http response --- 05:07:55 DEBUG "status": "success", 05:07:55 DEBUG --- http response --- 05:07:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:07:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:07:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:07:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:07:55 DEBUG "data": { 05:07:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:07:55 DEBUG ----- output ----- 05:07:55 DEBUG ----- output ----- 05:07:55 DEBUG ----- output ----- 05:07:55 DEBUG ----- output ----- 05:07:55 DEBUG "resultType": "vector", 05:07:55 DEBUG ----- output ----- 05:07:55 DEBUG { 05:07:55 DEBUG { 05:07:55 DEBUG { 05:07:55 DEBUG { 05:07:55 DEBUG "result": [ 05:07:55 DEBUG { 05:07:55 DEBUG "status": "success", 05:07:55 DEBUG "status": "success", 05:07:55 DEBUG "status": "success", 05:07:55 DEBUG "status": "success", 05:07:55 DEBUG { 05:07:55 DEBUG "status": "success", 05:07:55 DEBUG "data": { 05:07:55 DEBUG "data": { 05:07:55 DEBUG "data": { 05:07:55 DEBUG "data": { 05:07:55 DEBUG "metric": { 05:07:55 DEBUG "data": { 05:07:55 DEBUG "resultType": "vector", 05:07:55 DEBUG "resultType": "vector", 05:07:55 DEBUG "resultType": "vector", 05:07:55 DEBUG "resultType": "vector", 05:07:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:07:55 DEBUG "resultType": "vector", 05:07:55 DEBUG "result": [ 05:07:55 DEBUG "result": [ 05:07:55 DEBUG "result": [ 05:07:55 DEBUG "result": [ 05:07:55 DEBUG }, 05:07:55 DEBUG "result": [ 05:07:55 DEBUG { 05:07:55 DEBUG { 05:07:55 DEBUG { 05:07:55 DEBUG { 05:07:55 DEBUG "value": [ 05:07:55 DEBUG { 05:07:55 DEBUG "metric": { 05:07:55 DEBUG "metric": { 05:07:55 DEBUG "metric": { 05:07:55 DEBUG "metric": { 05:07:55 DEBUG 05:07:55 DEBUG "metric": { 05:07:55 DEBUG "pod": "ds-cts-0" 05:07:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:07:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:07:55 DEBUG "pod": "ds-idrepo-0" 05:07:55 DEBUG 05:07:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:07:55 DEBUG }, 05:07:55 DEBUG }, 05:07:55 DEBUG }, 05:07:55 DEBUG }, 05:07:55 DEBUG 05:07:55 DEBUG }, 05:07:55 DEBUG "value": [ 05:07:55 DEBUG "value": [ 05:07:55 DEBUG "value": [ 05:07:55 DEBUG "value": [ 05:07:55 DEBUG [...] 05:07:55 DEBUG "value": [ 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG [...] 05:07:55 DEBUG [...] 05:07:55 DEBUG [...] 05:07:55 DEBUG [...] 05:07:55 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:07:55 DEBUG [...] 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG }, 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG "value": [ 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 1713499675, 05:07:55 DEBUG 05:07:55 DEBUG "pod": "ds-cts-2" 05:07:55 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:07:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:07:55 DEBUG "pod": "ds-cts-2" 05:07:55 DEBUG "0" 05:07:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:07:55 DEBUG }, 05:07:55 DEBUG }, 05:07:55 DEBUG }, 05:07:55 DEBUG }, 05:07:55 DEBUG ] 05:07:55 DEBUG }, 05:07:55 DEBUG "value": [ 05:07:55 DEBUG "value": [ 05:07:55 DEBUG "value": [ 05:07:55 DEBUG "value": [ 05:07:55 DEBUG } 05:07:55 DEBUG "value": [ 05:07:55 DEBUG 1713499675, 05:07:55 DEBUG 1713499675, 05:07:55 DEBUG 1713499675, 05:07:55 DEBUG 1713499675, 05:07:55 DEBUG ] 05:07:55 DEBUG 1713499675, 05:07:55 DEBUG "1.6666666666666667" 05:07:55 DEBUG "9915.028060784758" 05:07:55 DEBUG "1999.9" 05:07:55 DEBUG "0" 05:07:55 DEBUG } 05:07:55 DEBUG "0" 05:07:55 DEBUG ] 05:07:55 DEBUG ] 05:07:55 DEBUG ] 05:07:55 DEBUG ] 05:07:55 DEBUG } 05:07:55 DEBUG ] 05:07:55 DEBUG } 05:07:55 DEBUG } 05:07:55 DEBUG } 05:07:55 DEBUG } 05:07:55 DEBUG } 05:07:55 DEBUG ] 05:07:55 DEBUG ] 05:07:55 DEBUG ] 05:07:55 DEBUG ] 05:07:55 INFO [http_cmd]: http status code OK 05:07:55 DEBUG ] 05:07:55 DEBUG } 05:07:55 DEBUG } 05:07:55 DEBUG } 05:07:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499675" 05:07:55 INFO [http_cmd]: http status code OK 05:07:55 DEBUG } 05:07:55 INFO [http_cmd]: http status code OK 05:07:55 INFO [http_cmd]: http status code OK 05:07:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499675" 05:07:55 INFO [http_cmd]: http status code OK 05:07:55 INFO [http_cmd]: http status code OK 05:07:55 INFO [http_cmd]: http status code OK 05:07:55 DEBUG --- status code --- 05:07:55 INFO [http_cmd]: http status code OK 05:07:55 DEBUG } 05:07:55 DEBUG } 05:07:55 DEBUG } 05:07:55 DEBUG } 05:07:55 DEBUG --- status code --- 05:07:55 DEBUG } 05:07:55 DEBUG --- status code --- 05:07:55 DEBUG --- status code --- 05:07:55 DEBUG --- status code --- 05:07:55 DEBUG --- status code --- 05:07:55 DEBUG --- status code --- 05:07:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:07:55 DEBUG --- status code --- 05:07:55 DEBUG } 05:07:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:07:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:07:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:07:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:07:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:07:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:07:55 DEBUG --- http response --- 05:07:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:07:55 DEBUG --- http response --- 05:07:55 DEBUG --- http response --- 05:07:55 DEBUG --- http response --- 05:07:55 DEBUG --- http response --- 05:07:55 DEBUG --- http response --- 05:07:55 DEBUG --- http response --- 05:07:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:07:55 DEBUG --- http response --- 05:07:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:07:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:07:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:07:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:07:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:07:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:07:55 DEBUG ----- output ----- 05:07:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:07:55 DEBUG ----- output ----- 05:07:55 DEBUG ----- output ----- 05:07:55 DEBUG ----- output ----- 05:07:55 DEBUG ----- output ----- 05:07:55 DEBUG ----- output ----- 05:07:55 DEBUG ----- output ----- 05:07:55 DEBUG { 05:07:55 DEBUG ----- output ----- 05:07:55 DEBUG { 05:07:55 DEBUG { 05:07:55 DEBUG { 05:07:55 DEBUG { 05:07:55 DEBUG { 05:07:55 DEBUG { 05:07:55 DEBUG "status": "success", 05:07:55 DEBUG { 05:07:55 INFO [http_cmd]: http status code OK 05:07:55 DEBUG "status": "success", 05:07:55 INFO [http_cmd]: http status code OK 05:07:55 DEBUG "status": "success", 05:07:55 DEBUG "status": "success", 05:07:55 DEBUG "status": "success", 05:07:55 DEBUG "status": "success", 05:07:55 DEBUG "status": "success", 05:07:55 DEBUG "data": { 05:07:55 DEBUG "status": "success", 05:07:55 DEBUG --- status code --- 05:07:55 DEBUG "data": { 05:07:55 DEBUG --- status code --- 05:07:55 DEBUG "data": { 05:07:55 DEBUG "data": { 05:07:55 DEBUG "data": { 05:07:55 DEBUG "data": { 05:07:55 DEBUG "data": { 05:07:55 DEBUG "resultType": "vector", 05:07:55 DEBUG "data": { 05:07:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:07:55 DEBUG "resultType": "vector", 05:07:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:07:55 DEBUG "resultType": "vector", 05:07:55 DEBUG "resultType": "vector", 05:07:55 DEBUG "resultType": "vector", 05:07:55 DEBUG "resultType": "vector", 05:07:55 DEBUG "resultType": "vector", 05:07:55 DEBUG "result": [ 05:07:55 DEBUG "resultType": "vector", 05:07:55 DEBUG --- http response --- 05:07:55 DEBUG "result": [ 05:07:55 DEBUG --- http response --- 05:07:55 DEBUG "result": [ 05:07:55 DEBUG "result": [ 05:07:55 DEBUG "result": [ 05:07:55 DEBUG "result": [ 05:07:55 DEBUG "result": [ 05:07:55 DEBUG { 05:07:55 DEBUG "result": [ 05:07:55 DEBUG { 05:07:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:07:55 DEBUG { 05:07:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:07:55 DEBUG { 05:07:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499675" 05:07:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499675" 05:07:55 DEBUG { 05:07:55 DEBUG { 05:07:55 DEBUG { 05:07:55 DEBUG "metric": { 05:07:55 DEBUG { 05:07:55 DEBUG "metric": { 05:07:55 DEBUG ----- output ----- 05:07:55 DEBUG "metric": { 05:07:55 DEBUG ----- output ----- 05:07:55 DEBUG "metric": { 05:07:55 DEBUG "metric": { 05:07:55 DEBUG "metric": { 05:07:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499675" 05:07:55 DEBUG "metric": { 05:07:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:07:55 DEBUG "metric": { 05:07:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:07:55 DEBUG { 05:07:55 DEBUG "pod": "ds-idrepo-0" 05:07:55 DEBUG { 05:07:55 DEBUG "pod": "ds-idrepo-0" 05:07:55 DEBUG "instance": "10.142.0.107:9100" 05:07:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:07:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:07:55 DEBUG }, 05:07:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:07:55 DEBUG }, 05:07:55 DEBUG "status": "success", 05:07:55 DEBUG }, 05:07:55 DEBUG "status": "success", 05:07:55 DEBUG }, 05:07:55 DEBUG }, 05:07:55 DEBUG }, 05:07:55 DEBUG }, 05:07:55 DEBUG "value": [ 05:07:55 DEBUG }, 05:07:55 DEBUG "value": [ 05:07:55 DEBUG "data": { 05:07:55 DEBUG "value": [ 05:07:55 DEBUG "data": { 05:07:55 DEBUG "value": [ 05:07:55 DEBUG "value": [ 05:07:55 DEBUG "value": [ 05:07:55 DEBUG "value": [ 05:07:55 DEBUG 05:07:55 DEBUG "value": [ 05:07:55 DEBUG 05:07:55 DEBUG "resultType": "vector", 05:07:55 DEBUG 05:07:55 DEBUG "resultType": "vector", 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG "result": [ 05:07:55 DEBUG 05:07:55 DEBUG "result": [ 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG { 05:07:55 DEBUG 05:07:55 DEBUG { 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG [...] 05:07:55 DEBUG 05:07:55 DEBUG [...] 05:07:55 DEBUG "metric": { 05:07:55 DEBUG [...] 05:07:55 DEBUG "metric": { 05:07:55 DEBUG [...] 05:07:55 DEBUG [...] 05:07:55 DEBUG [...] 05:07:55 DEBUG [...] 05:07:55 DEBUG 05:07:55 DEBUG [...] 05:07:55 DEBUG 05:07:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:07:55 DEBUG 05:07:55 DEBUG "pod": "ds-idrepo-0" 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG }, 05:07:55 DEBUG 05:07:55 DEBUG }, 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG "value": [ 05:07:55 DEBUG 05:07:55 DEBUG "value": [ 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:07:55 DEBUG 05:07:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:07:55 DEBUG 05:07:55 DEBUG "pod": "ds-idrepo-2" 05:07:55 DEBUG 05:07:55 DEBUG "pod": "ds-idrepo-2" 05:07:55 DEBUG "instance": "10.142.0.43:9100" 05:07:55 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:07:55 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:07:55 DEBUG }, 05:07:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:07:55 DEBUG }, 05:07:55 DEBUG 05:07:55 DEBUG }, 05:07:55 DEBUG 05:07:55 DEBUG }, 05:07:55 DEBUG }, 05:07:55 DEBUG }, 05:07:55 DEBUG }, 05:07:55 DEBUG "value": [ 05:07:55 DEBUG }, 05:07:55 DEBUG "value": [ 05:07:55 DEBUG 05:07:55 DEBUG "value": [ 05:07:55 DEBUG 05:07:55 DEBUG "value": [ 05:07:55 DEBUG "value": [ 05:07:55 INFO [http_cmd]: http status code OK 05:07:55 DEBUG "value": [ 05:07:55 DEBUG "value": [ 05:07:55 DEBUG 1713499675, 05:07:55 DEBUG "value": [ 05:07:55 DEBUG 1713499675, 05:07:55 DEBUG [...] 05:07:55 DEBUG 1713499675, 05:07:55 DEBUG [...] 05:07:55 DEBUG 1713499675, 05:07:55 DEBUG 1713499675, 05:07:55 DEBUG --- status code --- 05:07:55 DEBUG 1713499675, 05:07:55 DEBUG 1713499675, 05:07:55 DEBUG "4.280981820068896" 05:07:55 DEBUG 1713499675, 05:07:55 DEBUG "2172198912" 05:07:55 DEBUG 05:07:55 DEBUG "0" 05:07:55 DEBUG 05:07:55 DEBUG "0" 05:07:55 DEBUG "439533794304" 05:07:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:07:55 DEBUG "0" 05:07:55 DEBUG "2000.0333333333333" 05:07:55 DEBUG ] 05:07:55 DEBUG "1999.9666666666667" 05:07:55 DEBUG ] 05:07:55 DEBUG 05:07:55 DEBUG ] 05:07:55 DEBUG 05:07:55 DEBUG ] 05:07:55 DEBUG ] 05:07:55 DEBUG --- http response --- 05:07:55 DEBUG ] 05:07:55 DEBUG ] 05:07:55 DEBUG } 05:07:55 DEBUG ] 05:07:55 DEBUG } 05:07:55 DEBUG 05:07:55 DEBUG } 05:07:55 DEBUG 05:07:55 DEBUG } 05:07:55 DEBUG } 05:07:55 DEBUG } 05:07:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:07:55 DEBUG } 05:07:55 DEBUG ] 05:07:55 DEBUG } 05:07:55 DEBUG ] 05:07:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:07:55 DEBUG ] 05:07:55 DEBUG "pod": "ds-idrepo-2" 05:07:55 DEBUG ] 05:07:55 DEBUG ] 05:07:55 DEBUG ] 05:07:55 DEBUG ----- output ----- 05:07:55 DEBUG ] 05:07:55 DEBUG } 05:07:55 DEBUG ] 05:07:55 DEBUG } 05:07:55 INFO [http_cmd]: http status code OK 05:07:55 DEBUG }, 05:07:55 INFO [http_cmd]: http status code OK 05:07:55 DEBUG } 05:07:55 DEBUG }, 05:07:55 DEBUG } 05:07:55 DEBUG } 05:07:55 DEBUG } 05:07:55 DEBUG { 05:07:55 DEBUG } 05:07:55 DEBUG } 05:07:55 DEBUG } 05:07:55 DEBUG } 05:07:55 DEBUG --- status code --- 05:07:55 DEBUG "value": [ 05:07:55 DEBUG --- status code --- 05:07:55 DEBUG } 05:07:55 DEBUG "value": [ 05:07:55 DEBUG } 05:07:55 DEBUG } 05:07:55 DEBUG } 05:07:55 DEBUG "status": "success", 05:07:55 DEBUG } 05:07:55 DEBUG } 05:07:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:07:55 DEBUG 1713499675, 05:07:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:07:55 DEBUG 1713499675, 05:07:55 DEBUG "data": { 05:07:55 DEBUG --- http response --- 05:07:55 DEBUG "2742407.5" 05:07:55 DEBUG --- http response --- 05:07:55 DEBUG "0" 05:07:55 DEBUG "resultType": "vector", 05:07:55 DEBUG ] 05:07:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:07:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:07:55 DEBUG ] 05:07:55 DEBUG "result": [ 05:07:55 DEBUG } 05:07:55 DEBUG ----- output ----- 05:07:55 DEBUG ----- output ----- 05:07:55 DEBUG } 05:07:55 DEBUG { 05:07:55 DEBUG ] 05:07:55 DEBUG { 05:07:55 DEBUG { 05:07:55 DEBUG ] 05:07:55 DEBUG "metric": { 05:07:55 DEBUG } 05:07:55 DEBUG "status": "success", 05:07:55 DEBUG "status": "success", 05:07:55 DEBUG } 05:07:55 DEBUG "pod": "ds-cts-0" 05:07:55 DEBUG } 05:07:55 DEBUG "data": { 05:07:55 DEBUG "data": { 05:07:55 DEBUG } 05:07:55 DEBUG }, 05:07:55 DEBUG "resultType": "vector", 05:07:55 DEBUG "resultType": "vector", 05:07:55 DEBUG "value": [ 05:07:55 DEBUG "result": [ 05:07:55 DEBUG "result": [ 05:07:55 DEBUG 05:07:55 DEBUG { 05:07:55 DEBUG { 05:07:55 DEBUG 05:07:55 DEBUG "metric": { 05:07:55 DEBUG "metric": { 05:07:55 DEBUG 05:07:55 DEBUG "pod": "ds-cts-0" 05:07:55 DEBUG "instance": "10.142.0.107:9100" 05:07:55 DEBUG [...] 05:07:55 DEBUG }, 05:07:55 DEBUG }, 05:07:55 DEBUG 05:07:55 DEBUG "value": [ 05:07:55 DEBUG "value": [ 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG "pod": "ds-cts-2" 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG }, 05:07:55 DEBUG [...] 05:07:55 DEBUG [...] 05:07:55 DEBUG "value": [ 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 1713499675, 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG "0" 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG ] 05:07:55 DEBUG "pod": "ds-cts-2" 05:07:55 DEBUG "instance": "10.142.0.43:9100" 05:07:55 DEBUG } 05:07:55 DEBUG }, 05:07:55 DEBUG }, 05:07:55 DEBUG ] 05:07:55 DEBUG "value": [ 05:07:55 DEBUG "value": [ 05:07:55 DEBUG } 05:07:55 DEBUG 1713499675, 05:07:55 DEBUG 1713499675, 05:07:55 DEBUG } 05:07:55 DEBUG "0" 05:07:55 DEBUG "0.05543333333332327" 05:07:55 DEBUG ] 05:07:55 DEBUG ] 05:07:55 DEBUG } 05:07:55 DEBUG } 05:07:55 DEBUG ] 05:07:55 DEBUG ] 05:07:55 DEBUG } 05:07:55 DEBUG } 05:07:55 DEBUG } 05:07:55 DEBUG } 05:07:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499675" 05:07:55 INFO [http_cmd]: http status code OK 05:07:55 DEBUG --- status code --- 05:07:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:07:55 DEBUG --- http response --- 05:07:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:07:55 DEBUG ----- output ----- 05:07:55 DEBUG { 05:07:55 DEBUG "status": "success", 05:07:55 DEBUG "data": { 05:07:55 DEBUG "resultType": "vector", 05:07:55 DEBUG "result": [ 05:07:55 DEBUG { 05:07:55 DEBUG "metric": { 05:07:55 DEBUG "instance": "10.142.0.107:9100" 05:07:55 DEBUG }, 05:07:55 DEBUG "value": [ 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG [...] 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG "instance": "10.142.0.43:9100" 05:07:55 DEBUG }, 05:07:55 DEBUG "value": [ 05:07:55 DEBUG 1713499675, 05:07:55 DEBUG "47862272.13333333" 05:07:55 DEBUG ] 05:07:55 DEBUG } 05:07:55 DEBUG ] 05:07:55 DEBUG } 05:07:55 DEBUG } 05:07:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713499675" 05:07:55 INFO [http_cmd]: http status code OK 05:07:55 DEBUG --- status code --- 05:07:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:07:55 DEBUG --- http response --- 05:07:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:07:55 DEBUG ----- output ----- 05:07:55 DEBUG { 05:07:55 DEBUG "status": "success", 05:07:55 DEBUG "data": { 05:07:55 DEBUG "resultType": "vector", 05:07:55 DEBUG "result": [ 05:07:55 DEBUG { 05:07:55 DEBUG "metric": { 05:07:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:07:55 DEBUG }, 05:07:55 DEBUG "value": [ 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG [...] 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:07:55 DEBUG }, 05:07:55 DEBUG "value": [ 05:07:55 DEBUG 1713499675, 05:07:55 DEBUG "55.171" 05:07:55 DEBUG ] 05:07:55 DEBUG } 05:07:55 DEBUG ] 05:07:55 DEBUG } 05:07:55 DEBUG } 05:07:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499675" 05:07:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499675" 05:07:55 INFO [http_cmd]: http status code OK 05:07:55 DEBUG --- status code --- 05:07:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:07:55 DEBUG --- http response --- 05:07:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:07:55 DEBUG ----- output ----- 05:07:55 DEBUG { 05:07:55 DEBUG "status": "success", 05:07:55 DEBUG "data": { 05:07:55 DEBUG "resultType": "vector", 05:07:55 DEBUG "result": [ 05:07:55 DEBUG { 05:07:55 DEBUG "metric": { 05:07:55 DEBUG "instance": "10.142.0.107:9100" 05:07:55 DEBUG }, 05:07:55 DEBUG "value": [ 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG [...] 05:07:55 INFO [http_cmd]: http status code OK 05:07:55 DEBUG 05:07:55 DEBUG --- status code --- 05:07:55 DEBUG 05:07:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:07:55 DEBUG 05:07:55 DEBUG --- http response --- 05:07:55 DEBUG "instance": "10.142.0.43:9100" 05:07:55 DEBUG }, 05:07:55 DEBUG "value": [ 05:07:55 DEBUG 1713499675, 05:07:55 DEBUG "0.03799999999999954" 05:07:55 DEBUG ] 05:07:55 DEBUG } 05:07:55 DEBUG ] 05:07:55 DEBUG } 05:07:55 DEBUG } 05:07:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:07:55 DEBUG ----- output ----- 05:07:55 DEBUG { 05:07:55 DEBUG "status": "success", 05:07:55 DEBUG "data": { 05:07:55 DEBUG "resultType": "vector", 05:07:55 DEBUG "result": [ 05:07:55 DEBUG { 05:07:55 DEBUG "metric": { 05:07:55 DEBUG "instance": "10.142.0.107:9100" 05:07:55 DEBUG }, 05:07:55 DEBUG "value": [ 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG [...] 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG 05:07:55 DEBUG "instance": "10.142.0.43:9100" 05:07:55 DEBUG }, 05:07:55 DEBUG "value": [ 05:07:55 DEBUG 1713499675, 05:07:55 DEBUG "47860733.4" 05:07:55 DEBUG ] 05:07:55 DEBUG } 05:07:55 DEBUG ] 05:07:55 DEBUG } 05:07:55 DEBUG } 05:07:56 INFO 05:07:56 INFO [loop_until]: kubectl --namespace=xlou top pods 05:07:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:07:56 INFO [loop_until]: OK (rc = 0) 05:07:56 DEBUG --- stdout --- 05:07:56 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 10171m 9122Mi am-6fd554f8f7-hvvkw 9560m 8875Mi am-6fd554f8f7-pmj4b 10189m 8973Mi ds-cts-0 7697m 12259Mi ds-cts-1 7262m 12181Mi ds-cts-2 7667m 12220Mi ds-idrepo-0 4664m 3255Mi ds-idrepo-1 4489m 3207Mi ds-idrepo-2 4499m 3065Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 4m 1215Mi lodemon-6b89d774d5-rf7sk 8m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 6530m 1979Mi 05:07:56 DEBUG --- stderr --- 05:07:56 DEBUG 05:07:56 INFO 05:07:56 INFO [loop_until]: kubectl --namespace=xlou top node 05:07:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:07:56 INFO [loop_until]: OK (rc = 0) 05:07:56 DEBUG --- stdout --- 05:07:56 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 79m 0% 6253Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 4669m 15% 6698Mi 5% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9641m 32% 11589Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 3266Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 7682m 25% 14804Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 138m 0% 5168Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 9965m 33% 11898Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 10313m 34% 2719Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 11079m 37% 11170Mi 10% gke-xlou-cdm-ds-0f601751-j488 4857m 16% 6149Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 7313m 24% 16197Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 5161m 17% 6874Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 7416m 24% 14197Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 6468m 40% 5725Mi 9% 05:07:56 DEBUG --- stderr --- 05:07:56 DEBUG 05:08:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713499735" 05:08:55 ERROR [http_cmd]: ERROR 05:08:55 ERROR --- status code --- 05:08:55 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:08:55 ERROR --- http response --- 05:08:55 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:08:55 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:08:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713499735" 05:08:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499735" 05:08:55 INFO [http_cmd]: http status code OK 05:08:55 DEBUG --- status code --- 05:08:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:08:55 DEBUG --- http response --- 05:08:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:08:55 DEBUG ----- output ----- 05:08:55 DEBUG { 05:08:55 DEBUG "status": "success", 05:08:55 DEBUG "data": { 05:08:55 DEBUG "resultType": "vector", 05:08:55 DEBUG "result": [ 05:08:55 DEBUG { 05:08:55 DEBUG "metric": { 05:08:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:08:55 DEBUG }, 05:08:55 DEBUG "value": [ 05:08:55 INFO [http_cmd]: http status code OK 05:08:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713499735" 05:08:55 DEBUG 05:08:55 DEBUG --- status code --- 05:08:55 DEBUG 05:08:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:08:55 DEBUG 05:08:55 DEBUG --- http response --- 05:08:55 DEBUG [...] 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:08:55 DEBUG }, 05:08:55 DEBUG "value": [ 05:08:55 DEBUG 1713499735, 05:08:55 DEBUG "0" 05:08:55 DEBUG ] 05:08:55 DEBUG } 05:08:55 DEBUG ] 05:08:55 DEBUG } 05:08:55 DEBUG } 05:08:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:08:55 DEBUG ----- output ----- 05:08:55 DEBUG { 05:08:55 DEBUG "status": "success", 05:08:55 DEBUG "data": { 05:08:55 DEBUG "resultType": "vector", 05:08:55 DEBUG "result": [ 05:08:55 DEBUG { 05:08:55 DEBUG "metric": { 05:08:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:08:55 DEBUG }, 05:08:55 DEBUG "value": [ 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG [...] 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:08:55 DEBUG }, 05:08:55 DEBUG "value": [ 05:08:55 DEBUG 1713499735, 05:08:55 DEBUG "6.520716913436173" 05:08:55 DEBUG ] 05:08:55 DEBUG } 05:08:55 DEBUG ] 05:08:55 DEBUG } 05:08:55 DEBUG } 05:08:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499735" 05:08:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499735" 05:08:55 INFO [http_cmd]: http status code OK 05:08:55 DEBUG --- status code --- 05:08:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:08:55 DEBUG --- http response --- 05:08:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:08:55 DEBUG ----- output ----- 05:08:55 DEBUG { 05:08:55 DEBUG "status": "success", 05:08:55 DEBUG "data": { 05:08:55 DEBUG "resultType": "vector", 05:08:55 DEBUG "result": [ 05:08:55 DEBUG { 05:08:55 DEBUG "metric": { 05:08:55 DEBUG "pod": "ds-idrepo-0" 05:08:55 DEBUG }, 05:08:55 DEBUG "value": [ 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG [...] 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG "pod": "ds-cts-2" 05:08:55 DEBUG }, 05:08:55 DEBUG "value": [ 05:08:55 DEBUG 1713499735, 05:08:55 DEBUG "0" 05:08:55 DEBUG ] 05:08:55 DEBUG } 05:08:55 DEBUG ] 05:08:55 DEBUG } 05:08:55 DEBUG } 05:08:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499735" 05:08:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499735" 05:08:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499735" 05:08:55 INFO [http_cmd]: http status code OK 05:08:55 DEBUG --- status code --- 05:08:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:08:55 DEBUG --- http response --- 05:08:55 INFO [http_cmd]: http status code OK 05:08:55 DEBUG --- status code --- 05:08:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:08:55 DEBUG --- http response --- 05:08:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499735" 05:08:55 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:08:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:08:55 DEBUG ----- output ----- 05:08:55 DEBUG { 05:08:55 DEBUG "status": "success", 05:08:55 DEBUG "data": { 05:08:55 DEBUG "resultType": "vector", 05:08:55 DEBUG "result": [ 05:08:55 DEBUG { 05:08:55 DEBUG "metric": { 05:08:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:08:55 DEBUG }, 05:08:55 DEBUG "value": [ 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG [...] 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:08:55 DEBUG }, 05:08:55 DEBUG "value": [ 05:08:55 DEBUG 1713499735, 05:08:55 DEBUG "2862293.5" 05:08:55 DEBUG ] 05:08:55 DEBUG } 05:08:55 DEBUG ] 05:08:55 DEBUG } 05:08:55 DEBUG } 05:08:55 INFO [http_cmd]: http status code OK 05:08:55 DEBUG --- status code --- 05:08:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:08:55 DEBUG --- http response --- 05:08:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:08:55 DEBUG ----- output ----- 05:08:55 DEBUG { 05:08:55 DEBUG "status": "success", 05:08:55 DEBUG "data": { 05:08:55 DEBUG "resultType": "vector", 05:08:55 DEBUG "result": [ 05:08:55 DEBUG { 05:08:55 DEBUG "metric": { 05:08:55 DEBUG "pod": "ds-cts-0" 05:08:55 DEBUG }, 05:08:55 DEBUG "value": [ 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG [...] 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG "pod": "ds-cts-2" 05:08:55 DEBUG }, 05:08:55 DEBUG "value": [ 05:08:55 DEBUG 1713499735, 05:08:55 DEBUG "2.1333333333333333" 05:08:55 DEBUG ] 05:08:55 DEBUG } 05:08:55 DEBUG ] 05:08:55 DEBUG } 05:08:55 DEBUG } 05:08:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499735" 05:08:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499735" 05:08:55 INFO [http_cmd]: http status code OK 05:08:55 INFO [http_cmd]: http status code OK 05:08:55 DEBUG --- status code --- 05:08:55 DEBUG --- status code --- 05:08:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499735" 05:08:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499735" 05:08:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:08:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:08:55 DEBUG --- http response --- 05:08:55 DEBUG --- http response --- 05:08:55 INFO [http_cmd]: http status code OK 05:08:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:08:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:08:55 DEBUG --- status code --- 05:08:55 DEBUG ----- output ----- 05:08:55 DEBUG ----- output ----- 05:08:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:08:55 DEBUG { 05:08:55 DEBUG { 05:08:55 DEBUG --- http response --- 05:08:55 DEBUG "status": "success", 05:08:55 DEBUG "status": "success", 05:08:55 DEBUG "data": { 05:08:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:08:55 DEBUG "data": { 05:08:55 DEBUG "resultType": "vector", 05:08:55 DEBUG ----- output ----- 05:08:55 DEBUG "resultType": "vector", 05:08:55 DEBUG "result": [ 05:08:55 DEBUG { 05:08:55 DEBUG "result": [ 05:08:55 DEBUG { 05:08:55 DEBUG "status": "success", 05:08:55 DEBUG { 05:08:55 DEBUG "metric": { 05:08:55 DEBUG "data": { 05:08:55 DEBUG "metric": { 05:08:55 DEBUG "pod": "ds-cts-0" 05:08:55 DEBUG "resultType": "vector", 05:08:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:08:55 DEBUG }, 05:08:55 DEBUG "result": [ 05:08:55 DEBUG }, 05:08:55 DEBUG "value": [ 05:08:55 DEBUG { 05:08:55 DEBUG "value": [ 05:08:55 DEBUG 05:08:55 DEBUG "metric": { 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG }, 05:08:55 DEBUG 05:08:55 DEBUG [...] 05:08:55 DEBUG "value": [ 05:08:55 DEBUG [...] 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG "pod": "ds-cts-2" 05:08:55 DEBUG [...] 05:08:55 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:08:55 DEBUG }, 05:08:55 DEBUG 05:08:55 DEBUG }, 05:08:55 DEBUG "value": [ 05:08:55 DEBUG 05:08:55 DEBUG "value": [ 05:08:55 DEBUG 1713499735, 05:08:55 DEBUG 05:08:55 DEBUG 1713499735, 05:08:55 DEBUG "0" 05:08:55 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:08:55 DEBUG "16719.942223757338" 05:08:55 DEBUG ] 05:08:55 DEBUG }, 05:08:55 DEBUG ] 05:08:55 DEBUG } 05:08:55 DEBUG "value": [ 05:08:55 DEBUG } 05:08:55 DEBUG ] 05:08:55 DEBUG 1713499735, 05:08:55 DEBUG ] 05:08:55 DEBUG } 05:08:55 DEBUG "0" 05:08:55 DEBUG } 05:08:55 DEBUG } 05:08:55 DEBUG ] 05:08:55 DEBUG } 05:08:55 DEBUG } 05:08:55 DEBUG ] 05:08:55 INFO [http_cmd]: http status code OK 05:08:55 INFO [http_cmd]: http status code OK 05:08:55 DEBUG } 05:08:55 INFO [http_cmd]: http status code OK 05:08:55 DEBUG --- status code --- 05:08:55 DEBUG --- status code --- 05:08:55 DEBUG } 05:08:55 DEBUG --- status code --- 05:08:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:08:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:08:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:08:55 DEBUG --- http response --- 05:08:55 DEBUG --- http response --- 05:08:55 DEBUG --- http response --- 05:08:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:08:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:08:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:08:55 DEBUG ----- output ----- 05:08:55 DEBUG ----- output ----- 05:08:55 DEBUG ----- output ----- 05:08:55 DEBUG { 05:08:55 DEBUG { 05:08:55 DEBUG { 05:08:55 INFO [http_cmd]: http status code OK 05:08:55 DEBUG "status": "success", 05:08:55 DEBUG "status": "success", 05:08:55 DEBUG "status": "success", 05:08:55 DEBUG --- status code --- 05:08:55 DEBUG "data": { 05:08:55 DEBUG "data": { 05:08:55 DEBUG "data": { 05:08:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:08:55 DEBUG "resultType": "vector", 05:08:55 DEBUG "resultType": "vector", 05:08:55 DEBUG "resultType": "vector", 05:08:55 DEBUG --- http response --- 05:08:55 DEBUG "result": [ 05:08:55 DEBUG "result": [ 05:08:55 DEBUG "result": [ 05:08:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:08:55 DEBUG { 05:08:55 DEBUG { 05:08:55 DEBUG { 05:08:55 DEBUG ----- output ----- 05:08:55 DEBUG "metric": { 05:08:55 DEBUG "metric": { 05:08:55 DEBUG "metric": { 05:08:55 DEBUG { 05:08:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:08:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:08:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:08:55 DEBUG "status": "success", 05:08:55 DEBUG }, 05:08:55 DEBUG }, 05:08:55 DEBUG }, 05:08:55 DEBUG "data": { 05:08:55 DEBUG "value": [ 05:08:55 DEBUG "value": [ 05:08:55 DEBUG "value": [ 05:08:55 DEBUG "resultType": "vector", 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG "result": [ 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG { 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG "metric": { 05:08:55 DEBUG [...] 05:08:55 DEBUG [...] 05:08:55 DEBUG [...] 05:08:55 DEBUG "instance": "10.142.0.107:9100" 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG }, 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG "value": [ 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:08:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:08:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:08:55 DEBUG 05:08:55 DEBUG }, 05:08:55 DEBUG }, 05:08:55 DEBUG }, 05:08:55 DEBUG 05:08:55 DEBUG "value": [ 05:08:55 DEBUG "value": [ 05:08:55 DEBUG "value": [ 05:08:55 DEBUG [...] 05:08:55 DEBUG 1713499735, 05:08:55 DEBUG 1713499735, 05:08:55 DEBUG 1713499735, 05:08:55 DEBUG 05:08:55 DEBUG "2001.6666666666667" 05:08:55 DEBUG "0" 05:08:55 DEBUG "2001.5333333333333" 05:08:55 DEBUG 05:08:55 DEBUG ] 05:08:55 DEBUG ] 05:08:55 DEBUG ] 05:08:55 DEBUG 05:08:55 DEBUG } 05:08:55 DEBUG } 05:08:55 DEBUG } 05:08:55 DEBUG "instance": "10.142.0.43:9100" 05:08:55 DEBUG ] 05:08:55 DEBUG ] 05:08:55 DEBUG ] 05:08:55 DEBUG }, 05:08:55 DEBUG } 05:08:55 DEBUG } 05:08:55 DEBUG } 05:08:55 DEBUG "value": [ 05:08:55 DEBUG } 05:08:55 DEBUG } 05:08:55 DEBUG } 05:08:55 DEBUG 1713499735, 05:08:55 DEBUG "441883323392" 05:08:55 DEBUG ] 05:08:55 DEBUG } 05:08:55 DEBUG ] 05:08:55 DEBUG } 05:08:55 DEBUG } 05:08:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499735" 05:08:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713499735" 05:08:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499735" 05:08:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499735" 05:08:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499735" 05:08:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499735" 05:08:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499735" 05:08:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499735" 05:08:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499735" 05:08:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499735" 05:08:55 INFO [http_cmd]: http status code OK 05:08:55 INFO [http_cmd]: http status code OK 05:08:55 INFO [http_cmd]: http status code OK 05:08:55 DEBUG --- status code --- 05:08:55 INFO [http_cmd]: http status code OK 05:08:55 DEBUG --- status code --- 05:08:55 INFO [http_cmd]: http status code OK 05:08:55 DEBUG --- status code --- 05:08:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:08:55 DEBUG --- status code --- 05:08:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:08:55 DEBUG --- status code --- 05:08:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:08:55 DEBUG --- http response --- 05:08:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:08:55 DEBUG --- http response --- 05:08:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:08:55 DEBUG --- http response --- 05:08:55 INFO [http_cmd]: http status code OK 05:08:55 DEBUG --- http response --- 05:08:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:08:55 INFO [http_cmd]: http status code OK 05:08:55 DEBUG --- http response --- 05:08:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:08:55 INFO [http_cmd]: http status code OK 05:08:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:08:55 DEBUG --- status code --- 05:08:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:08:55 DEBUG ----- output ----- 05:08:55 DEBUG --- status code --- 05:08:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:08:55 DEBUG ----- output ----- 05:08:55 DEBUG --- status code --- 05:08:55 DEBUG ----- output ----- 05:08:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:08:55 DEBUG ----- output ----- 05:08:55 DEBUG { 05:08:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:08:55 DEBUG ----- output ----- 05:08:55 DEBUG { 05:08:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:08:55 DEBUG { 05:08:55 DEBUG --- http response --- 05:08:55 DEBUG { 05:08:55 DEBUG "status": "success", 05:08:55 DEBUG --- http response --- 05:08:55 DEBUG { 05:08:55 DEBUG "status": "success", 05:08:55 DEBUG --- http response --- 05:08:55 DEBUG "status": "success", 05:08:55 DEBUG "status": "success", 05:08:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:08:55 DEBUG "data": { 05:08:55 DEBUG "status": "success", 05:08:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:08:55 DEBUG "data": { 05:08:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:08:55 DEBUG "data": { 05:08:55 INFO [http_cmd]: http status code OK 05:08:55 DEBUG "data": { 05:08:55 DEBUG ----- output ----- 05:08:55 DEBUG "resultType": "vector", 05:08:55 DEBUG "data": { 05:08:55 DEBUG ----- output ----- 05:08:55 DEBUG "resultType": "vector", 05:08:55 DEBUG ----- output ----- 05:08:55 DEBUG "resultType": "vector", 05:08:55 DEBUG --- status code --- 05:08:55 DEBUG "resultType": "vector", 05:08:55 DEBUG { 05:08:55 DEBUG "result": [ 05:08:55 DEBUG "resultType": "vector", 05:08:55 DEBUG { 05:08:55 DEBUG "result": [ 05:08:55 DEBUG { 05:08:55 DEBUG "result": [ 05:08:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:08:55 DEBUG "result": [ 05:08:55 DEBUG "status": "success", 05:08:55 DEBUG { 05:08:55 INFO [http_cmd]: http status code OK 05:08:55 DEBUG "result": [ 05:08:55 DEBUG "status": "success", 05:08:55 DEBUG { 05:08:55 DEBUG "status": "success", 05:08:55 DEBUG { 05:08:55 DEBUG --- http response --- 05:08:55 DEBUG { 05:08:55 DEBUG "data": { 05:08:55 DEBUG "metric": { 05:08:55 DEBUG --- status code --- 05:08:55 DEBUG { 05:08:55 DEBUG "data": { 05:08:55 DEBUG "metric": { 05:08:55 DEBUG "data": { 05:08:55 DEBUG "metric": { 05:08:55 DEBUG "metric": { 05:08:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:08:55 DEBUG "resultType": "vector", 05:08:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:08:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:08:55 DEBUG "metric": { 05:08:55 DEBUG "resultType": "vector", 05:08:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:08:55 DEBUG "resultType": "vector", 05:08:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:08:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:08:55 DEBUG ----- output ----- 05:08:55 DEBUG "result": [ 05:08:55 DEBUG }, 05:08:55 DEBUG --- http response --- 05:08:55 DEBUG "pod": "ds-idrepo-0" 05:08:55 DEBUG "result": [ 05:08:55 DEBUG }, 05:08:55 DEBUG "result": [ 05:08:55 DEBUG }, 05:08:55 DEBUG }, 05:08:55 DEBUG { 05:08:55 DEBUG { 05:08:55 DEBUG "value": [ 05:08:55 DEBUG }, 05:08:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:08:55 DEBUG { 05:08:55 DEBUG "value": [ 05:08:55 DEBUG { 05:08:55 DEBUG "value": [ 05:08:55 DEBUG "value": [ 05:08:55 DEBUG "status": "success", 05:08:55 DEBUG "metric": { 05:08:55 DEBUG 05:08:55 DEBUG "value": [ 05:08:55 DEBUG ----- output ----- 05:08:55 DEBUG "metric": { 05:08:55 DEBUG 05:08:55 DEBUG "metric": { 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG "data": { 05:08:55 DEBUG "pod": "ds-idrepo-0" 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG { 05:08:55 DEBUG "pod": "ds-idrepo-0" 05:08:55 DEBUG 05:08:55 DEBUG "instance": "10.142.0.107:9100" 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG "resultType": "vector", 05:08:55 DEBUG }, 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG "status": "success", 05:08:55 DEBUG }, 05:08:55 DEBUG 05:08:55 DEBUG }, 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG "result": [ 05:08:55 DEBUG "value": [ 05:08:55 DEBUG [...] 05:08:55 DEBUG 05:08:55 DEBUG "data": { 05:08:55 DEBUG "value": [ 05:08:55 DEBUG [...] 05:08:55 DEBUG "value": [ 05:08:55 DEBUG [...] 05:08:55 DEBUG [...] 05:08:55 DEBUG { 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG [...] 05:08:55 DEBUG "resultType": "vector", 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG "metric": { 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG "result": [ 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG "instance": "10.142.0.107:9100" 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG { 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG }, 05:08:55 DEBUG [...] 05:08:55 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:08:55 DEBUG 05:08:55 DEBUG "metric": { 05:08:55 DEBUG [...] 05:08:55 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:08:55 DEBUG [...] 05:08:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:08:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:08:55 DEBUG "value": [ 05:08:55 DEBUG 05:08:55 DEBUG }, 05:08:55 DEBUG "pod": "ds-idrepo-2" 05:08:55 DEBUG "pod": "ds-cts-0" 05:08:55 DEBUG 05:08:55 DEBUG }, 05:08:55 DEBUG 05:08:55 DEBUG }, 05:08:55 DEBUG }, 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG "value": [ 05:08:55 DEBUG }, 05:08:55 DEBUG }, 05:08:55 DEBUG 05:08:55 DEBUG "value": [ 05:08:55 DEBUG 05:08:55 DEBUG "value": [ 05:08:55 DEBUG "value": [ 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG 1713499735, 05:08:55 DEBUG "value": [ 05:08:55 DEBUG "value": [ 05:08:55 DEBUG 05:08:55 DEBUG 1713499735, 05:08:55 DEBUG 05:08:55 DEBUG 1713499735, 05:08:55 DEBUG 1713499735, 05:08:55 DEBUG 05:08:55 DEBUG "pod": "ds-idrepo-2" 05:08:55 DEBUG "6.482209825415436" 05:08:55 DEBUG 1713499735, 05:08:55 DEBUG 05:08:55 DEBUG "pod": "ds-idrepo-2" 05:08:55 DEBUG "0" 05:08:55 DEBUG "instance": "10.142.0.43:9100" 05:08:55 DEBUG "2174291968" 05:08:55 DEBUG "2001.5333333333333" 05:08:55 DEBUG [...] 05:08:55 DEBUG }, 05:08:55 DEBUG ] 05:08:55 DEBUG "0" 05:08:55 DEBUG 05:08:55 DEBUG }, 05:08:55 DEBUG ] 05:08:55 DEBUG }, 05:08:55 DEBUG ] 05:08:55 DEBUG ] 05:08:55 DEBUG 05:08:55 DEBUG "value": [ 05:08:55 DEBUG } 05:08:55 DEBUG ] 05:08:55 DEBUG 05:08:55 DEBUG "value": [ 05:08:55 DEBUG } 05:08:55 DEBUG "value": [ 05:08:55 DEBUG } 05:08:55 DEBUG } 05:08:55 DEBUG 05:08:55 DEBUG 1713499735, 05:08:55 DEBUG ] 05:08:55 DEBUG } 05:08:55 DEBUG [...] 05:08:55 DEBUG 1713499735, 05:08:55 DEBUG ] 05:08:55 DEBUG 1713499735, 05:08:55 DEBUG ] 05:08:55 DEBUG ] 05:08:55 DEBUG 05:08:55 DEBUG "0" 05:08:55 DEBUG } 05:08:55 DEBUG ] 05:08:55 DEBUG 05:08:55 DEBUG "0" 05:08:55 DEBUG } 05:08:55 DEBUG "0.058766666666668736" 05:08:55 DEBUG } 05:08:55 DEBUG } 05:08:55 DEBUG "instance": "10.142.0.43:9100" 05:08:55 DEBUG ] 05:08:55 DEBUG } 05:08:55 DEBUG } 05:08:55 DEBUG 05:08:55 DEBUG ] 05:08:55 DEBUG } 05:08:55 DEBUG ] 05:08:55 DEBUG } 05:08:55 DEBUG } 05:08:55 DEBUG }, 05:08:55 DEBUG } 05:08:55 DEBUG } 05:08:55 DEBUG 05:08:55 DEBUG } 05:08:55 DEBUG } 05:08:55 DEBUG "value": [ 05:08:55 DEBUG ] 05:08:55 DEBUG "pod": "ds-cts-2" 05:08:55 DEBUG ] 05:08:55 DEBUG ] 05:08:55 DEBUG 1713499735, 05:08:55 DEBUG } 05:08:55 DEBUG }, 05:08:55 DEBUG } 05:08:55 DEBUG } 05:08:55 DEBUG "47734091.86666667" 05:08:55 DEBUG } 05:08:55 DEBUG "value": [ 05:08:55 DEBUG } 05:08:55 DEBUG } 05:08:55 DEBUG ] 05:08:55 DEBUG 1713499735, 05:08:55 DEBUG } 05:08:55 DEBUG "0" 05:08:55 DEBUG ] 05:08:55 DEBUG ] 05:08:55 DEBUG } 05:08:55 DEBUG } 05:08:55 DEBUG } 05:08:55 DEBUG ] 05:08:55 DEBUG } 05:08:55 DEBUG } 05:08:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713499735" 05:08:55 INFO [http_cmd]: http status code OK 05:08:55 DEBUG --- status code --- 05:08:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:08:55 DEBUG --- http response --- 05:08:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:08:55 DEBUG ----- output ----- 05:08:55 DEBUG { 05:08:55 DEBUG "status": "success", 05:08:55 DEBUG "data": { 05:08:55 DEBUG "resultType": "vector", 05:08:55 DEBUG "result": [ 05:08:55 DEBUG { 05:08:55 DEBUG "metric": { 05:08:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:08:55 DEBUG }, 05:08:55 DEBUG "value": [ 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG [...] 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:08:55 DEBUG }, 05:08:55 DEBUG "value": [ 05:08:55 DEBUG 1713499735, 05:08:55 DEBUG "55.171" 05:08:55 DEBUG ] 05:08:55 DEBUG } 05:08:55 DEBUG ] 05:08:55 DEBUG } 05:08:55 DEBUG } 05:08:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499735" 05:08:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499735" 05:08:55 INFO [http_cmd]: http status code OK 05:08:55 DEBUG --- status code --- 05:08:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:08:55 DEBUG --- http response --- 05:08:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:08:55 DEBUG ----- output ----- 05:08:55 DEBUG { 05:08:55 DEBUG "status": "success", 05:08:55 DEBUG "data": { 05:08:55 DEBUG "resultType": "vector", 05:08:55 DEBUG "result": [ 05:08:55 DEBUG { 05:08:55 DEBUG "metric": { 05:08:55 DEBUG "instance": "10.142.0.107:9100" 05:08:55 DEBUG }, 05:08:55 DEBUG "value": [ 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG [...] 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG "instance": "10.142.0.43:9100" 05:08:55 DEBUG }, 05:08:55 DEBUG "value": [ 05:08:55 DEBUG 1713499735, 05:08:55 DEBUG "47730632.96666667" 05:08:55 DEBUG ] 05:08:55 DEBUG } 05:08:55 DEBUG ] 05:08:55 DEBUG } 05:08:55 DEBUG } 05:08:55 INFO [http_cmd]: http status code OK 05:08:55 DEBUG --- status code --- 05:08:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:08:55 DEBUG --- http response --- 05:08:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:08:55 DEBUG ----- output ----- 05:08:55 DEBUG { 05:08:55 DEBUG "status": "success", 05:08:55 DEBUG "data": { 05:08:55 DEBUG "resultType": "vector", 05:08:55 DEBUG "result": [ 05:08:55 DEBUG { 05:08:55 DEBUG "metric": { 05:08:55 DEBUG "instance": "10.142.0.107:9100" 05:08:55 DEBUG }, 05:08:55 DEBUG "value": [ 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG [...] 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG 05:08:55 DEBUG "instance": "10.142.0.43:9100" 05:08:55 DEBUG }, 05:08:55 DEBUG "value": [ 05:08:55 DEBUG 1713499735, 05:08:55 DEBUG "0.0383333333333321" 05:08:55 DEBUG ] 05:08:55 DEBUG } 05:08:55 DEBUG ] 05:08:55 DEBUG } 05:08:55 DEBUG } 05:08:56 INFO 05:08:56 INFO [loop_until]: kubectl --namespace=xlou top pods 05:08:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:08:56 INFO [loop_until]: OK (rc = 0) 05:08:56 DEBUG --- stdout --- 05:08:56 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 9735m 9122Mi am-6fd554f8f7-hvvkw 9304m 8875Mi am-6fd554f8f7-pmj4b 10265m 8974Mi ds-cts-0 7396m 12265Mi ds-cts-1 7347m 12188Mi ds-cts-2 7816m 12226Mi ds-idrepo-0 4691m 3255Mi ds-idrepo-1 4608m 3207Mi ds-idrepo-2 4687m 3065Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 3m 1216Mi lodemon-6b89d774d5-rf7sk 10m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 6238m 1981Mi 05:08:56 DEBUG --- stderr --- 05:08:56 DEBUG 05:08:56 INFO 05:08:56 INFO [loop_until]: kubectl --namespace=xlou top node 05:08:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:08:56 INFO [loop_until]: OK (rc = 0) 05:08:56 DEBUG --- stdout --- 05:08:56 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 76m 0% 6250Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 4630m 15% 6706Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9602m 32% 11593Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 88m 0% 3250Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 7449m 24% 14804Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 156m 0% 5181Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 9948m 33% 11904Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 10292m 34% 2717Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 11120m 37% 11178Mi 10% gke-xlou-cdm-ds-0f601751-j488 4853m 16% 6150Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 7340m 24% 16203Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 5156m 17% 6873Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 7561m 25% 14199Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 6674m 42% 5724Mi 9% 05:08:56 DEBUG --- stderr --- 05:08:56 DEBUG 05:09:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713499795" 05:09:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499795" 05:09:55 ERROR [http_cmd]: ERROR 05:09:55 ERROR --- status code --- 05:09:55 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:09:55 ERROR --- http response --- 05:09:55 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:09:55 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:09:55 INFO [http_cmd]: http status code OK 05:09:55 DEBUG --- status code --- 05:09:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:09:55 DEBUG --- http response --- 05:09:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:09:55 DEBUG ----- output ----- 05:09:55 DEBUG { 05:09:55 DEBUG "status": "success", 05:09:55 DEBUG "data": { 05:09:55 DEBUG "resultType": "vector", 05:09:55 DEBUG "result": [ 05:09:55 DEBUG { 05:09:55 DEBUG "metric": { 05:09:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:09:55 DEBUG }, 05:09:55 DEBUG "value": [ 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG [...] 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:09:55 DEBUG }, 05:09:55 DEBUG "value": [ 05:09:55 DEBUG 1713499795, 05:09:55 DEBUG "2982397" 05:09:55 DEBUG ] 05:09:55 DEBUG } 05:09:55 DEBUG ] 05:09:55 DEBUG } 05:09:55 DEBUG } 05:09:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499795" 05:09:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713499795" 05:09:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499795" 05:09:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499795" 05:09:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499795" 05:09:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499795" 05:09:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499795" 05:09:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713499795" 05:09:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499795" 05:09:55 INFO [http_cmd]: http status code OK 05:09:55 DEBUG --- status code --- 05:09:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:09:55 DEBUG --- http response --- 05:09:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:09:55 DEBUG ----- output ----- 05:09:55 DEBUG { 05:09:55 DEBUG "status": "success", 05:09:55 DEBUG "data": { 05:09:55 DEBUG "resultType": "vector", 05:09:55 DEBUG "result": [ 05:09:55 DEBUG { 05:09:55 DEBUG "metric": { 05:09:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:09:55 DEBUG }, 05:09:55 DEBUG "value": [ 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG [...] 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:09:55 DEBUG }, 05:09:55 DEBUG "value": [ 05:09:55 DEBUG 1713499795, 05:09:55 DEBUG "0" 05:09:55 DEBUG ] 05:09:55 DEBUG } 05:09:55 DEBUG ] 05:09:55 DEBUG } 05:09:55 DEBUG } 05:09:55 INFO [http_cmd]: http status code OK 05:09:55 DEBUG --- status code --- 05:09:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:09:55 DEBUG --- http response --- 05:09:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:09:55 DEBUG ----- output ----- 05:09:55 DEBUG { 05:09:55 DEBUG "status": "success", 05:09:55 DEBUG "data": { 05:09:55 DEBUG "resultType": "vector", 05:09:55 DEBUG "result": [ 05:09:55 DEBUG { 05:09:55 DEBUG "metric": { 05:09:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:09:55 DEBUG }, 05:09:55 DEBUG "value": [ 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG [...] 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:09:55 DEBUG }, 05:09:55 DEBUG "value": [ 05:09:55 DEBUG 1713499795, 05:09:55 DEBUG "6.538035073055029" 05:09:55 DEBUG ] 05:09:55 DEBUG } 05:09:55 DEBUG ] 05:09:55 DEBUG } 05:09:55 DEBUG } 05:09:55 INFO [http_cmd]: http status code OK 05:09:55 DEBUG --- status code --- 05:09:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:09:55 DEBUG --- http response --- 05:09:55 INFO [http_cmd]: http status code OK 05:09:55 INFO [http_cmd]: http status code OK 05:09:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:09:55 INFO [http_cmd]: http status code OK 05:09:55 DEBUG --- status code --- 05:09:55 INFO [http_cmd]: http status code OK 05:09:55 INFO [http_cmd]: http status code OK 05:09:55 DEBUG --- status code --- 05:09:55 DEBUG ----- output ----- 05:09:55 DEBUG --- status code --- 05:09:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:09:55 DEBUG --- status code --- 05:09:55 DEBUG --- status code --- 05:09:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:09:55 DEBUG { 05:09:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:09:55 DEBUG --- http response --- 05:09:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:09:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:09:55 DEBUG --- http response --- 05:09:55 DEBUG "status": "success", 05:09:55 DEBUG --- http response --- 05:09:55 DEBUG --- http response --- 05:09:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:09:55 DEBUG --- http response --- 05:09:55 DEBUG "data": { 05:09:55 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:09:55 INFO [http_cmd]: http status code OK 05:09:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:09:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:09:55 DEBUG ----- output ----- 05:09:55 DEBUG "resultType": "vector", 05:09:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:09:55 DEBUG --- status code --- 05:09:55 DEBUG ----- output ----- 05:09:55 DEBUG ----- output ----- 05:09:55 DEBUG { 05:09:55 DEBUG "result": [ 05:09:55 DEBUG ----- output ----- 05:09:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:09:55 DEBUG { 05:09:55 DEBUG { 05:09:55 DEBUG "status": "success", 05:09:55 DEBUG { 05:09:55 DEBUG { 05:09:55 DEBUG --- http response --- 05:09:55 DEBUG "status": "success", 05:09:55 DEBUG "status": "success", 05:09:55 DEBUG "data": { 05:09:55 DEBUG "metric": { 05:09:55 DEBUG "status": "success", 05:09:55 DEBUG "data": { 05:09:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:09:55 DEBUG "data": { 05:09:55 DEBUG "resultType": "vector", 05:09:55 DEBUG "pod": "ds-cts-0" 05:09:55 DEBUG "data": { 05:09:55 DEBUG "resultType": "vector", 05:09:55 DEBUG ----- output ----- 05:09:55 DEBUG "resultType": "vector", 05:09:55 DEBUG "result": [ 05:09:55 DEBUG }, 05:09:55 DEBUG "resultType": "vector", 05:09:55 DEBUG "result": [ 05:09:55 DEBUG { 05:09:55 DEBUG "result": [ 05:09:55 DEBUG { 05:09:55 DEBUG "value": [ 05:09:55 DEBUG "result": [ 05:09:55 DEBUG { 05:09:55 DEBUG "status": "success", 05:09:55 DEBUG { 05:09:55 DEBUG "metric": { 05:09:55 DEBUG 05:09:55 DEBUG { 05:09:55 DEBUG "metric": { 05:09:55 DEBUG "data": { 05:09:55 DEBUG "metric": { 05:09:55 DEBUG "pod": "ds-idrepo-1" 05:09:55 DEBUG 05:09:55 DEBUG "metric": { 05:09:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:09:55 DEBUG "resultType": "vector", 05:09:55 DEBUG "pod": "ds-idrepo-0" 05:09:55 DEBUG }, 05:09:55 DEBUG 05:09:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:09:55 DEBUG }, 05:09:55 DEBUG "result": [ 05:09:55 DEBUG }, 05:09:55 DEBUG "value": [ 05:09:55 DEBUG [...] 05:09:55 DEBUG }, 05:09:55 DEBUG "value": [ 05:09:55 DEBUG { 05:09:55 DEBUG "value": [ 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG "value": [ 05:09:55 DEBUG 05:09:55 DEBUG "metric": { 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG "pod": "ds-cts-0" 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG }, 05:09:55 DEBUG 05:09:55 DEBUG [...] 05:09:55 DEBUG "pod": "ds-cts-2" 05:09:55 DEBUG 05:09:55 DEBUG [...] 05:09:55 DEBUG "value": [ 05:09:55 DEBUG [...] 05:09:55 DEBUG 05:09:55 DEBUG }, 05:09:55 DEBUG [...] 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG "value": [ 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG 1713499795, 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG "pod": "ds-cts-0" 05:09:55 DEBUG "2.5" 05:09:55 DEBUG 05:09:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:09:55 DEBUG [...] 05:09:55 DEBUG "pod": "ds-cts-2" 05:09:55 DEBUG }, 05:09:55 DEBUG ] 05:09:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:09:55 DEBUG }, 05:09:55 DEBUG 05:09:55 DEBUG }, 05:09:55 DEBUG "value": [ 05:09:55 DEBUG } 05:09:55 DEBUG }, 05:09:55 DEBUG "value": [ 05:09:55 DEBUG 05:09:55 DEBUG "value": [ 05:09:55 DEBUG 1713499795, 05:09:55 DEBUG ] 05:09:55 DEBUG "value": [ 05:09:55 DEBUG 1713499795, 05:09:55 DEBUG 05:09:55 DEBUG 1713499795, 05:09:55 DEBUG "83093.52980233642" 05:09:55 DEBUG } 05:09:55 DEBUG 1713499795, 05:09:55 DEBUG "2005.8666666666666" 05:09:55 DEBUG "pod": "ds-cts-2" 05:09:55 DEBUG "0" 05:09:55 DEBUG ] 05:09:55 DEBUG } 05:09:55 DEBUG "0" 05:09:55 DEBUG ] 05:09:55 DEBUG }, 05:09:55 DEBUG ] 05:09:55 DEBUG } 05:09:55 DEBUG ] 05:09:55 DEBUG } 05:09:55 DEBUG "value": [ 05:09:55 DEBUG } 05:09:55 DEBUG ] 05:09:55 DEBUG } 05:09:55 DEBUG ] 05:09:55 DEBUG 1713499795, 05:09:55 DEBUG ] 05:09:55 DEBUG } 05:09:55 DEBUG ] 05:09:55 DEBUG } 05:09:55 DEBUG "0" 05:09:55 DEBUG } 05:09:55 DEBUG } 05:09:55 DEBUG } 05:09:55 DEBUG } 05:09:55 DEBUG ] 05:09:55 DEBUG } 05:09:55 DEBUG } 05:09:55 DEBUG } 05:09:55 DEBUG ] 05:09:55 DEBUG } 05:09:55 DEBUG } 05:09:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499795" 05:09:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499795" 05:09:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499795" 05:09:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499795" 05:09:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499795" 05:09:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499795" 05:09:55 INFO [http_cmd]: http status code OK 05:09:55 DEBUG --- status code --- 05:09:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:09:55 DEBUG --- http response --- 05:09:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:09:55 DEBUG ----- output ----- 05:09:55 DEBUG { 05:09:55 DEBUG "status": "success", 05:09:55 DEBUG "data": { 05:09:55 DEBUG "resultType": "vector", 05:09:55 DEBUG "result": [ 05:09:55 DEBUG { 05:09:55 DEBUG "metric": { 05:09:55 DEBUG "pod": "ds-idrepo-1" 05:09:55 DEBUG }, 05:09:55 DEBUG "value": [ 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG [...] 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG "pod": "ds-cts-0" 05:09:55 DEBUG }, 05:09:55 DEBUG "value": [ 05:09:55 DEBUG 1713499795, 05:09:55 DEBUG "27307781.77138809" 05:09:55 DEBUG ] 05:09:55 DEBUG } 05:09:55 DEBUG ] 05:09:55 DEBUG } 05:09:55 DEBUG } 05:09:55 INFO [http_cmd]: http status code OK 05:09:55 INFO [http_cmd]: http status code OK 05:09:55 DEBUG --- status code --- 05:09:55 DEBUG --- status code --- 05:09:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:09:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:09:55 DEBUG --- http response --- 05:09:55 DEBUG --- http response --- 05:09:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:09:55 DEBUG ----- output ----- 05:09:55 DEBUG { 05:09:55 DEBUG "status": "success", 05:09:55 DEBUG "data": { 05:09:55 DEBUG "resultType": "vector", 05:09:55 DEBUG "result": [ 05:09:55 DEBUG { 05:09:55 DEBUG "metric": { 05:09:55 DEBUG "pod": "ds-idrepo-0" 05:09:55 DEBUG }, 05:09:55 DEBUG "value": [ 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG [...] 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG "pod": "ds-idrepo-2" 05:09:55 DEBUG }, 05:09:55 DEBUG "value": [ 05:09:55 DEBUG 1713499795, 05:09:55 DEBUG "0" 05:09:55 DEBUG ] 05:09:55 DEBUG } 05:09:55 DEBUG ] 05:09:55 DEBUG } 05:09:55 DEBUG } 05:09:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:09:55 DEBUG ----- output ----- 05:09:55 DEBUG { 05:09:55 DEBUG "status": "success", 05:09:55 DEBUG "data": { 05:09:55 DEBUG "resultType": "vector", 05:09:55 DEBUG "result": [ 05:09:55 DEBUG { 05:09:55 DEBUG "metric": { 05:09:55 DEBUG "pod": "ds-idrepo-0" 05:09:55 DEBUG }, 05:09:55 DEBUG "value": [ 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG [...] 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG "pod": "ds-idrepo-2" 05:09:55 DEBUG }, 05:09:55 DEBUG "value": [ 05:09:55 DEBUG 1713499795, 05:09:55 DEBUG "0" 05:09:55 DEBUG ] 05:09:55 DEBUG } 05:09:55 DEBUG ] 05:09:55 DEBUG } 05:09:55 DEBUG } 05:09:55 INFO [http_cmd]: http status code OK 05:09:55 DEBUG --- status code --- 05:09:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:09:55 DEBUG --- http response --- 05:09:55 INFO [http_cmd]: http status code OK 05:09:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:09:55 DEBUG --- status code --- 05:09:55 DEBUG ----- output ----- 05:09:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:09:55 DEBUG { 05:09:55 DEBUG --- http response --- 05:09:55 DEBUG "status": "success", 05:09:55 DEBUG "data": { 05:09:55 DEBUG "resultType": "vector", 05:09:55 DEBUG "result": [ 05:09:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:09:55 DEBUG { 05:09:55 DEBUG ----- output ----- 05:09:55 DEBUG "metric": { 05:09:55 DEBUG { 05:09:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:09:55 DEBUG "status": "success", 05:09:55 DEBUG }, 05:09:55 DEBUG "data": { 05:09:55 DEBUG "value": [ 05:09:55 DEBUG "resultType": "vector", 05:09:55 DEBUG 05:09:55 DEBUG "result": [ 05:09:55 DEBUG 05:09:55 DEBUG { 05:09:55 DEBUG 05:09:55 DEBUG "metric": { 05:09:55 DEBUG [...] 05:09:55 DEBUG "pod": "ds-idrepo-1" 05:09:55 DEBUG 05:09:55 DEBUG }, 05:09:55 DEBUG 05:09:55 DEBUG "value": [ 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:09:55 DEBUG 05:09:55 DEBUG }, 05:09:55 DEBUG 05:09:55 DEBUG "value": [ 05:09:55 DEBUG [...] 05:09:55 DEBUG 1713499795, 05:09:55 DEBUG 05:09:55 DEBUG "2005.8999999999999" 05:09:55 DEBUG 05:09:55 DEBUG ] 05:09:55 DEBUG 05:09:55 DEBUG } 05:09:55 DEBUG "pod": "ds-cts-0" 05:09:55 DEBUG ] 05:09:55 DEBUG }, 05:09:55 DEBUG } 05:09:55 DEBUG "value": [ 05:09:55 DEBUG } 05:09:55 DEBUG 1713499795, 05:09:55 DEBUG "5.283954612956957" 05:09:55 DEBUG ] 05:09:55 DEBUG } 05:09:55 DEBUG ] 05:09:55 DEBUG } 05:09:55 DEBUG } 05:09:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499795" 05:09:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499795" 05:09:55 INFO [http_cmd]: http status code OK 05:09:55 DEBUG --- status code --- 05:09:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:09:55 DEBUG --- http response --- 05:09:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:09:55 DEBUG ----- output ----- 05:09:55 DEBUG { 05:09:55 DEBUG "status": "success", 05:09:55 DEBUG "data": { 05:09:55 DEBUG "resultType": "vector", 05:09:55 DEBUG "result": [ 05:09:55 DEBUG { 05:09:55 DEBUG "metric": { 05:09:55 DEBUG "instance": "10.142.0.107:9100" 05:09:55 DEBUG }, 05:09:55 DEBUG "value": [ 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG [...] 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG "instance": "10.142.0.43:9100" 05:09:55 DEBUG }, 05:09:55 DEBUG "value": [ 05:09:55 DEBUG 1713499795, 05:09:55 DEBUG "444261254144" 05:09:55 DEBUG ] 05:09:55 DEBUG } 05:09:55 DEBUG ] 05:09:55 DEBUG } 05:09:55 DEBUG } 05:09:55 INFO [http_cmd]: http status code OK 05:09:55 DEBUG --- status code --- 05:09:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:09:55 DEBUG --- http response --- 05:09:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:09:55 DEBUG ----- output ----- 05:09:55 DEBUG { 05:09:55 DEBUG "status": "success", 05:09:55 DEBUG "data": { 05:09:55 DEBUG "resultType": "vector", 05:09:55 DEBUG "result": [ 05:09:55 DEBUG { 05:09:55 DEBUG "metric": { 05:09:55 DEBUG "instance": "10.142.0.107:9100" 05:09:55 DEBUG }, 05:09:55 DEBUG "value": [ 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG [...] 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG "instance": "10.142.0.43:9100" 05:09:55 DEBUG }, 05:09:55 DEBUG "value": [ 05:09:55 DEBUG 1713499795, 05:09:55 DEBUG "0.060466666666666426" 05:09:55 DEBUG ] 05:09:55 DEBUG } 05:09:55 DEBUG ] 05:09:55 DEBUG } 05:09:55 DEBUG } 05:09:55 INFO [http_cmd]: http status code OK 05:09:55 DEBUG --- status code --- 05:09:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:09:55 DEBUG --- http response --- 05:09:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:09:55 DEBUG ----- output ----- 05:09:55 DEBUG { 05:09:55 DEBUG "status": "success", 05:09:55 DEBUG "data": { 05:09:55 DEBUG "resultType": "vector", 05:09:55 DEBUG "result": [ 05:09:55 DEBUG { 05:09:55 DEBUG "metric": { 05:09:55 DEBUG "instance": "10.142.0.107:9100" 05:09:55 DEBUG }, 05:09:55 DEBUG "value": [ 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG [...] 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG "instance": "10.142.0.43:9100" 05:09:55 DEBUG }, 05:09:55 DEBUG "value": [ 05:09:55 DEBUG 1713499795, 05:09:55 DEBUG "47964871.66666667" 05:09:55 DEBUG ] 05:09:55 DEBUG } 05:09:55 DEBUG ] 05:09:55 DEBUG } 05:09:55 DEBUG } 05:09:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499795" 05:09:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499795" 05:09:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713499795" 05:09:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499795" 05:09:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499795" 05:09:55 INFO [http_cmd]: http status code OK 05:09:55 DEBUG --- status code --- 05:09:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:09:55 DEBUG --- http response --- 05:09:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:09:55 INFO [http_cmd]: http status code OK 05:09:55 DEBUG ----- output ----- 05:09:55 INFO [http_cmd]: http status code OK 05:09:55 DEBUG --- status code --- 05:09:55 DEBUG { 05:09:55 DEBUG --- status code --- 05:09:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:09:55 DEBUG "status": "success", 05:09:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:09:55 DEBUG --- http response --- 05:09:55 DEBUG "data": { 05:09:55 DEBUG --- http response --- 05:09:55 DEBUG "resultType": "vector", 05:09:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:09:55 DEBUG "result": [ 05:09:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:09:55 DEBUG ----- output ----- 05:09:55 DEBUG { 05:09:55 DEBUG ----- output ----- 05:09:55 DEBUG { 05:09:55 DEBUG "metric": { 05:09:55 DEBUG { 05:09:55 DEBUG "status": "success", 05:09:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:09:55 DEBUG "status": "success", 05:09:55 DEBUG "data": { 05:09:55 DEBUG }, 05:09:55 DEBUG "data": { 05:09:55 DEBUG "resultType": "vector", 05:09:55 DEBUG "value": [ 05:09:55 DEBUG "resultType": "vector", 05:09:55 DEBUG "result": [ 05:09:55 DEBUG 05:09:55 DEBUG "result": [ 05:09:55 DEBUG { 05:09:55 DEBUG 05:09:55 DEBUG { 05:09:55 DEBUG "metric": { 05:09:55 DEBUG 05:09:55 DEBUG "metric": { 05:09:55 DEBUG "pod": "ds-idrepo-1" 05:09:55 DEBUG [...] 05:09:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:09:55 DEBUG }, 05:09:55 DEBUG 05:09:55 DEBUG }, 05:09:55 DEBUG "value": [ 05:09:55 DEBUG 05:09:55 DEBUG "value": [ 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 INFO [http_cmd]: http status code OK 05:09:55 DEBUG 05:09:55 INFO [http_cmd]: http status code OK 05:09:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:09:55 DEBUG 05:09:55 DEBUG --- status code --- 05:09:55 DEBUG 05:09:55 DEBUG --- status code --- 05:09:55 DEBUG }, 05:09:55 DEBUG 05:09:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:09:55 DEBUG [...] 05:09:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:09:55 DEBUG "value": [ 05:09:55 DEBUG [...] 05:09:55 DEBUG --- http response --- 05:09:55 DEBUG 05:09:55 DEBUG --- http response --- 05:09:55 DEBUG 1713499795, 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:09:55 DEBUG "2006" 05:09:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG ----- output ----- 05:09:55 DEBUG ] 05:09:55 DEBUG ----- output ----- 05:09:55 DEBUG 05:09:55 DEBUG "pod": "ds-cts-0" 05:09:55 DEBUG { 05:09:55 DEBUG } 05:09:55 DEBUG { 05:09:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:09:55 DEBUG }, 05:09:55 DEBUG "status": "success", 05:09:55 DEBUG ] 05:09:55 DEBUG "status": "success", 05:09:55 DEBUG }, 05:09:55 DEBUG "value": [ 05:09:55 DEBUG "data": { 05:09:55 DEBUG } 05:09:55 DEBUG "data": { 05:09:55 DEBUG "value": [ 05:09:55 DEBUG 1713499795, 05:09:55 DEBUG "resultType": "vector", 05:09:55 DEBUG } 05:09:55 DEBUG "resultType": "vector", 05:09:55 DEBUG 1713499795, 05:09:55 DEBUG "38168.49728438277" 05:09:55 DEBUG "result": [ 05:09:55 DEBUG "result": [ 05:09:55 DEBUG "2175995904" 05:09:55 DEBUG ] 05:09:55 DEBUG { 05:09:55 DEBUG { 05:09:55 DEBUG ] 05:09:55 DEBUG } 05:09:55 DEBUG "metric": { 05:09:55 DEBUG "metric": { 05:09:55 DEBUG } 05:09:55 DEBUG ] 05:09:55 DEBUG "pod": "ds-cts-0" 05:09:55 DEBUG "pod": "ds-idrepo-0" 05:09:55 DEBUG ] 05:09:55 DEBUG } 05:09:55 DEBUG }, 05:09:55 DEBUG }, 05:09:55 DEBUG } 05:09:55 DEBUG } 05:09:55 DEBUG "value": [ 05:09:55 DEBUG "value": [ 05:09:55 DEBUG } 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG [...] 05:09:55 DEBUG [...] 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG "pod": "ds-cts-2" 05:09:55 DEBUG "pod": "ds-idrepo-2" 05:09:55 DEBUG }, 05:09:55 DEBUG }, 05:09:55 DEBUG "value": [ 05:09:55 DEBUG "value": [ 05:09:55 DEBUG 1713499795, 05:09:55 DEBUG 1713499795, 05:09:55 DEBUG "0" 05:09:55 DEBUG "0" 05:09:55 DEBUG ] 05:09:55 DEBUG ] 05:09:55 DEBUG } 05:09:55 DEBUG } 05:09:55 DEBUG ] 05:09:55 DEBUG ] 05:09:55 DEBUG } 05:09:55 DEBUG } 05:09:55 DEBUG } 05:09:55 DEBUG } 05:09:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713499795" 05:09:55 INFO [http_cmd]: http status code OK 05:09:55 DEBUG --- status code --- 05:09:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:09:55 DEBUG --- http response --- 05:09:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:09:55 DEBUG ----- output ----- 05:09:55 DEBUG { 05:09:55 DEBUG "status": "success", 05:09:55 DEBUG "data": { 05:09:55 DEBUG "resultType": "vector", 05:09:55 DEBUG "result": [ 05:09:55 DEBUG { 05:09:55 DEBUG "metric": { 05:09:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:09:55 DEBUG }, 05:09:55 DEBUG "value": [ 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG [...] 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:09:55 DEBUG }, 05:09:55 DEBUG "value": [ 05:09:55 DEBUG 1713499795, 05:09:55 DEBUG "55.171" 05:09:55 DEBUG ] 05:09:55 DEBUG } 05:09:55 DEBUG ] 05:09:55 DEBUG } 05:09:55 DEBUG } 05:09:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499795" 05:09:55 INFO [http_cmd]: http status code OK 05:09:55 DEBUG --- status code --- 05:09:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:09:55 DEBUG --- http response --- 05:09:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:09:55 DEBUG ----- output ----- 05:09:55 DEBUG { 05:09:55 DEBUG "status": "success", 05:09:55 DEBUG "data": { 05:09:55 DEBUG "resultType": "vector", 05:09:55 DEBUG "result": [ 05:09:55 DEBUG { 05:09:55 DEBUG "metric": { 05:09:55 DEBUG "instance": "10.142.0.107:9100" 05:09:55 DEBUG }, 05:09:55 DEBUG "value": [ 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG [...] 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG "instance": "10.142.0.43:9100" 05:09:55 DEBUG }, 05:09:55 DEBUG "value": [ 05:09:55 DEBUG 1713499795, 05:09:55 DEBUG "47961217.5" 05:09:55 DEBUG ] 05:09:55 DEBUG } 05:09:55 DEBUG ] 05:09:55 DEBUG } 05:09:55 DEBUG } 05:09:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499795" 05:09:55 INFO [http_cmd]: http status code OK 05:09:55 DEBUG --- status code --- 05:09:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:09:55 DEBUG --- http response --- 05:09:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:09:55 DEBUG ----- output ----- 05:09:55 DEBUG { 05:09:55 DEBUG "status": "success", 05:09:55 DEBUG "data": { 05:09:55 DEBUG "resultType": "vector", 05:09:55 DEBUG "result": [ 05:09:55 DEBUG { 05:09:55 DEBUG "metric": { 05:09:55 DEBUG "instance": "10.142.0.107:9100" 05:09:55 DEBUG }, 05:09:55 DEBUG "value": [ 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG [...] 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG 05:09:55 DEBUG "instance": "10.142.0.43:9100" 05:09:55 DEBUG }, 05:09:55 DEBUG "value": [ 05:09:55 DEBUG 1713499795, 05:09:55 DEBUG "0.03766666666666745" 05:09:55 DEBUG ] 05:09:55 DEBUG } 05:09:55 DEBUG ] 05:09:55 DEBUG } 05:09:55 DEBUG } 05:09:56 INFO 05:09:56 INFO [loop_until]: kubectl --namespace=xlou top pods 05:09:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:09:56 INFO [loop_until]: OK (rc = 0) 05:09:56 DEBUG --- stdout --- 05:09:56 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 9913m 9121Mi am-6fd554f8f7-hvvkw 9583m 8868Mi am-6fd554f8f7-pmj4b 10140m 8974Mi ds-cts-0 7675m 12271Mi ds-cts-1 7538m 12193Mi ds-cts-2 7573m 12232Mi ds-idrepo-0 4654m 3255Mi ds-idrepo-1 4711m 3206Mi ds-idrepo-2 4544m 3066Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 3m 1215Mi lodemon-6b89d774d5-rf7sk 9m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 6531m 1982Mi 05:09:56 DEBUG --- stderr --- 05:09:56 DEBUG 05:09:56 INFO 05:09:56 INFO [loop_until]: kubectl --namespace=xlou top node 05:09:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:09:56 INFO [loop_until]: OK (rc = 0) 05:09:56 DEBUG --- stdout --- 05:09:56 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 76m 0% 6262Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 4643m 15% 6696Mi 5% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9383m 31% 11595Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 81m 0% 3246Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 7517m 25% 14818Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 157m 0% 5157Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 9866m 33% 11906Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 10266m 34% 2727Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 10857m 36% 11174Mi 10% gke-xlou-cdm-ds-0f601751-j488 4859m 16% 6161Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 7425m 24% 16211Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 5172m 17% 6878Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 7594m 25% 14207Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 6676m 42% 5723Mi 9% 05:09:56 DEBUG --- stderr --- 05:09:56 DEBUG 05:10:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713499855" 05:10:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713499855" 05:10:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499855" 05:10:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713499855" 05:10:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499855" 05:10:55 ERROR [http_cmd]: ERROR 05:10:55 ERROR --- status code --- 05:10:55 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:10:55 ERROR --- http response --- 05:10:55 INFO [http_cmd]: http status code OK 05:10:55 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:10:55 INFO [http_cmd]: http status code OK 05:10:55 INFO [http_cmd]: http status code OK 05:10:55 DEBUG --- status code --- 05:10:55 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:10:55 DEBUG --- status code --- 05:10:55 DEBUG --- status code --- 05:10:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:10:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:10:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:10:55 DEBUG --- http response --- 05:10:55 DEBUG --- http response --- 05:10:55 DEBUG --- http response --- 05:10:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:10:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:10:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:10:55 DEBUG ----- output ----- 05:10:55 DEBUG ----- output ----- 05:10:55 DEBUG ----- output ----- 05:10:55 DEBUG { 05:10:55 DEBUG { 05:10:55 DEBUG { 05:10:55 DEBUG "status": "success", 05:10:55 DEBUG "status": "success", 05:10:55 DEBUG "status": "success", 05:10:55 DEBUG "data": { 05:10:55 DEBUG "data": { 05:10:55 DEBUG "data": { 05:10:55 DEBUG "resultType": "vector", 05:10:55 DEBUG "resultType": "vector", 05:10:55 DEBUG "resultType": "vector", 05:10:55 DEBUG "result": [ 05:10:55 DEBUG "result": [ 05:10:55 DEBUG "result": [ 05:10:55 DEBUG { 05:10:55 DEBUG { 05:10:55 DEBUG { 05:10:55 DEBUG "metric": { 05:10:55 DEBUG "metric": { 05:10:55 DEBUG "metric": { 05:10:55 DEBUG "pod": "ds-idrepo-0" 05:10:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:10:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:10:55 DEBUG }, 05:10:55 DEBUG }, 05:10:55 DEBUG }, 05:10:55 DEBUG "value": [ 05:10:55 DEBUG "value": [ 05:10:55 DEBUG "value": [ 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG [...] 05:10:55 DEBUG [...] 05:10:55 DEBUG [...] 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG "pod": "ds-cts-2" 05:10:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:10:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:10:55 DEBUG }, 05:10:55 DEBUG }, 05:10:55 DEBUG }, 05:10:55 DEBUG "value": [ 05:10:55 DEBUG "value": [ 05:10:55 DEBUG "value": [ 05:10:55 DEBUG 1713499855, 05:10:55 DEBUG 1713499855, 05:10:55 DEBUG 1713499855, 05:10:55 DEBUG "0" 05:10:55 DEBUG "3102816" 05:10:55 DEBUG "6.600271604828367" 05:10:55 DEBUG ] 05:10:55 DEBUG ] 05:10:55 DEBUG ] 05:10:55 DEBUG } 05:10:55 DEBUG } 05:10:55 DEBUG } 05:10:55 DEBUG ] 05:10:55 DEBUG ] 05:10:55 DEBUG ] 05:10:55 DEBUG } 05:10:55 DEBUG } 05:10:55 DEBUG } 05:10:55 DEBUG } 05:10:55 DEBUG } 05:10:55 DEBUG } 05:10:55 INFO [http_cmd]: http status code OK 05:10:55 DEBUG --- status code --- 05:10:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:10:55 DEBUG --- http response --- 05:10:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:10:55 DEBUG ----- output ----- 05:10:55 DEBUG { 05:10:55 DEBUG "status": "success", 05:10:55 DEBUG "data": { 05:10:55 DEBUG "resultType": "vector", 05:10:55 DEBUG "result": [ 05:10:55 DEBUG { 05:10:55 DEBUG "metric": { 05:10:55 DEBUG "pod": "ds-idrepo-1" 05:10:55 DEBUG }, 05:10:55 DEBUG "value": [ 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG [...] 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:10:55 DEBUG }, 05:10:55 DEBUG "value": [ 05:10:55 DEBUG 1713499855, 05:10:55 DEBUG "2210.24586017335" 05:10:55 DEBUG ] 05:10:55 DEBUG } 05:10:55 DEBUG ] 05:10:55 DEBUG } 05:10:55 DEBUG } 05:10:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499855" 05:10:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499855" 05:10:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499855" 05:10:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499855" 05:10:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499855" 05:10:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499855" 05:10:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499855" 05:10:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499855" 05:10:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499855" 05:10:55 INFO [http_cmd]: http status code OK 05:10:55 INFO [http_cmd]: http status code OK 05:10:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499855" 05:10:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499855" 05:10:55 DEBUG --- status code --- 05:10:55 INFO [http_cmd]: http status code OK 05:10:55 DEBUG --- status code --- 05:10:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:10:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499855" 05:10:55 DEBUG --- status code --- 05:10:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:10:55 DEBUG --- http response --- 05:10:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:10:55 DEBUG --- http response --- 05:10:55 INFO [http_cmd]: http status code OK 05:10:55 INFO [http_cmd]: http status code OK 05:10:55 INFO [http_cmd]: http status code OK 05:10:55 DEBUG --- http response --- 05:10:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:10:55 INFO [http_cmd]: http status code OK 05:10:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:10:55 DEBUG --- status code --- 05:10:55 DEBUG --- status code --- 05:10:55 DEBUG --- status code --- 05:10:55 DEBUG ----- output ----- 05:10:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:10:55 DEBUG --- status code --- 05:10:55 DEBUG ----- output ----- 05:10:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499855" 05:10:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:10:55 INFO [http_cmd]: http status code OK 05:10:55 INFO [http_cmd]: http status code OK 05:10:55 INFO [http_cmd]: http status code OK 05:10:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:10:55 INFO [http_cmd]: http status code OK 05:10:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:10:55 DEBUG { 05:10:55 DEBUG ----- output ----- 05:10:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:10:55 DEBUG { 05:10:55 DEBUG --- http response --- 05:10:55 DEBUG --- status code --- 05:10:55 INFO [http_cmd]: http status code OK 05:10:55 DEBUG --- status code --- 05:10:55 DEBUG --- status code --- 05:10:55 DEBUG --- http response --- 05:10:55 DEBUG --- status code --- 05:10:55 DEBUG --- http response --- 05:10:55 DEBUG "status": "success", 05:10:55 DEBUG { 05:10:55 DEBUG --- http response --- 05:10:55 DEBUG "status": "success", 05:10:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:10:55 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:10:55 DEBUG --- status code --- 05:10:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:10:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:10:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:10:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:10:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:10:55 DEBUG "data": { 05:10:55 DEBUG "status": "success", 05:10:55 DEBUG "data": { 05:10:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:10:55 DEBUG --- http response --- 05:10:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:10:55 DEBUG --- http response --- 05:10:55 DEBUG --- http response --- 05:10:55 DEBUG --- http response --- 05:10:55 DEBUG ----- output ----- 05:10:55 DEBUG ----- output ----- 05:10:55 DEBUG "resultType": "vector", 05:10:55 DEBUG "data": { 05:10:55 DEBUG "resultType": "vector", 05:10:55 DEBUG ----- output ----- 05:10:55 DEBUG --- http response --- 05:10:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:10:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:10:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:10:55 DEBUG { 05:10:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:10:55 DEBUG { 05:10:55 DEBUG "result": [ 05:10:55 DEBUG "resultType": "vector", 05:10:55 DEBUG "result": [ 05:10:55 DEBUG { 05:10:55 DEBUG ----- output ----- 05:10:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:10:55 DEBUG ----- output ----- 05:10:55 DEBUG ----- output ----- 05:10:55 DEBUG "status": "success", 05:10:55 DEBUG ----- output ----- 05:10:55 DEBUG "status": "success", 05:10:55 DEBUG { 05:10:55 DEBUG "result": [ 05:10:55 DEBUG { 05:10:55 DEBUG "status": "success", 05:10:55 DEBUG { 05:10:55 DEBUG ----- output ----- 05:10:55 DEBUG { 05:10:55 DEBUG { 05:10:55 DEBUG "data": { 05:10:55 DEBUG { 05:10:55 DEBUG "data": { 05:10:55 DEBUG "metric": { 05:10:55 DEBUG { 05:10:55 DEBUG "metric": { 05:10:55 DEBUG "data": { 05:10:55 DEBUG "status": "success", 05:10:55 DEBUG { 05:10:55 DEBUG "status": "success", 05:10:55 DEBUG "status": "success", 05:10:55 DEBUG "resultType": "vector", 05:10:55 DEBUG "status": "success", 05:10:55 DEBUG "resultType": "vector", 05:10:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:10:55 DEBUG "metric": { 05:10:55 DEBUG "pod": "ds-idrepo-1" 05:10:55 DEBUG "resultType": "vector", 05:10:55 DEBUG "data": { 05:10:55 DEBUG "status": "success", 05:10:55 DEBUG "data": { 05:10:55 DEBUG "data": { 05:10:55 DEBUG "result": [ 05:10:55 DEBUG "data": { 05:10:55 DEBUG "result": [ 05:10:55 DEBUG }, 05:10:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:10:55 DEBUG }, 05:10:55 DEBUG "result": [ 05:10:55 DEBUG "resultType": "vector", 05:10:55 DEBUG "data": { 05:10:55 DEBUG "resultType": "vector", 05:10:55 DEBUG "resultType": "vector", 05:10:55 DEBUG { 05:10:55 DEBUG "resultType": "vector", 05:10:55 DEBUG { 05:10:55 DEBUG "value": [ 05:10:55 DEBUG }, 05:10:55 DEBUG "value": [ 05:10:55 DEBUG { 05:10:55 DEBUG "result": [ 05:10:55 DEBUG "resultType": "vector", 05:10:55 DEBUG "result": [ 05:10:55 DEBUG "result": [ 05:10:55 DEBUG "metric": { 05:10:55 DEBUG "result": [ 05:10:55 DEBUG "metric": { 05:10:55 DEBUG 05:10:55 DEBUG "value": [ 05:10:55 DEBUG 05:10:55 DEBUG "metric": { 05:10:55 DEBUG { 05:10:55 DEBUG "result": [ 05:10:55 DEBUG { 05:10:55 DEBUG { 05:10:55 DEBUG "instance": "10.142.0.107:9100" 05:10:55 DEBUG { 05:10:55 DEBUG "pod": "ds-cts-0" 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG "pod": "ds-idrepo-1" 05:10:55 DEBUG "metric": { 05:10:55 DEBUG { 05:10:55 DEBUG "metric": { 05:10:55 DEBUG "metric": { 05:10:55 DEBUG }, 05:10:55 DEBUG "metric": { 05:10:55 DEBUG }, 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG }, 05:10:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:10:55 DEBUG "metric": { 05:10:55 DEBUG "pod": "ds-idrepo-0" 05:10:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:10:55 DEBUG "value": [ 05:10:55 DEBUG "pod": "ds-cts-0" 05:10:55 DEBUG "value": [ 05:10:55 DEBUG [...] 05:10:55 INFO [http_cmd]: http status code OK 05:10:55 DEBUG 05:10:55 DEBUG [...] 05:10:55 DEBUG "value": [ 05:10:55 DEBUG }, 05:10:55 DEBUG "instance": "10.142.0.107:9100" 05:10:55 DEBUG }, 05:10:55 DEBUG }, 05:10:55 DEBUG 05:10:55 DEBUG }, 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG --- status code --- 05:10:55 DEBUG [...] 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG "value": [ 05:10:55 DEBUG }, 05:10:55 DEBUG "value": [ 05:10:55 DEBUG "value": [ 05:10:55 DEBUG 05:10:55 DEBUG "value": [ 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG "value": [ 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG --- http response --- 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG [...] 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:10:55 DEBUG 05:10:55 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:10:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:10:55 DEBUG [...] 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG [...] 05:10:55 DEBUG [...] 05:10:55 DEBUG }, 05:10:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:10:55 DEBUG }, 05:10:55 DEBUG ----- output ----- 05:10:55 DEBUG 05:10:55 DEBUG [...] 05:10:55 DEBUG 05:10:55 DEBUG [...] 05:10:55 DEBUG [...] 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG "value": [ 05:10:55 DEBUG { 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG [...] 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG "value": [ 05:10:55 DEBUG }, 05:10:55 DEBUG "pod": "ds-cts-2" 05:10:55 DEBUG 1713499855, 05:10:55 DEBUG "status": "success", 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG "instance": "10.142.0.43:9100" 05:10:55 DEBUG 05:10:55 DEBUG 1713499855, 05:10:55 DEBUG "value": [ 05:10:55 DEBUG }, 05:10:55 DEBUG "6.601192520117332" 05:10:55 DEBUG "data": { 05:10:55 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG }, 05:10:55 DEBUG "pod": "ds-cts-2" 05:10:55 DEBUG "2002.8666666666666" 05:10:55 DEBUG 1713499855, 05:10:55 DEBUG "value": [ 05:10:55 DEBUG ] 05:10:55 DEBUG "resultType": "vector", 05:10:55 DEBUG }, 05:10:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:10:55 DEBUG 05:10:55 DEBUG "pod": "ds-idrepo-2" 05:10:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:10:55 DEBUG "value": [ 05:10:55 DEBUG }, 05:10:55 DEBUG ] 05:10:55 DEBUG "0" 05:10:55 DEBUG 1713499855, 05:10:55 DEBUG } 05:10:55 DEBUG "result": [ 05:10:55 DEBUG "value": [ 05:10:55 DEBUG }, 05:10:55 DEBUG "instance": "10.142.0.43:9100" 05:10:55 DEBUG }, 05:10:55 DEBUG }, 05:10:55 DEBUG 1713499855, 05:10:55 DEBUG "value": [ 05:10:55 DEBUG } 05:10:55 DEBUG ] 05:10:55 DEBUG } 05:10:55 DEBUG ] 05:10:55 DEBUG { 05:10:55 DEBUG 1713499855, 05:10:55 DEBUG "value": [ 05:10:55 DEBUG }, 05:10:55 DEBUG "value": [ 05:10:55 DEBUG "value": [ 05:10:55 DEBUG "0.06113333333333249" 05:10:55 DEBUG 1713499855, 05:10:55 DEBUG ] 05:10:55 DEBUG "0" 05:10:55 DEBUG ] 05:10:55 DEBUG } 05:10:55 DEBUG "metric": { 05:10:55 DEBUG "0" 05:10:55 DEBUG 1713499855, 05:10:55 DEBUG "value": [ 05:10:55 DEBUG 1713499855, 05:10:55 DEBUG 1713499855, 05:10:55 DEBUG ] 05:10:55 DEBUG "2.9" 05:10:55 DEBUG } 05:10:55 DEBUG ] 05:10:55 DEBUG } 05:10:55 DEBUG } 05:10:55 DEBUG "instance": "10.142.0.107:9100" 05:10:55 DEBUG ] 05:10:55 DEBUG "0.03333333333333333" 05:10:55 DEBUG 1713499855, 05:10:55 DEBUG "0" 05:10:55 DEBUG "2002.9" 05:10:55 DEBUG } 05:10:55 DEBUG ] 05:10:55 DEBUG } 05:10:55 DEBUG } 05:10:55 DEBUG ] 05:10:55 DEBUG }, 05:10:55 DEBUG } 05:10:55 DEBUG ] 05:10:55 DEBUG "446686553088" 05:10:55 DEBUG ] 05:10:55 DEBUG ] 05:10:55 DEBUG ] 05:10:55 DEBUG } 05:10:55 DEBUG } 05:10:55 DEBUG } 05:10:55 DEBUG "value": [ 05:10:55 DEBUG ] 05:10:55 DEBUG } 05:10:55 DEBUG ] 05:10:55 DEBUG } 05:10:55 DEBUG } 05:10:55 DEBUG } 05:10:55 DEBUG ] 05:10:55 DEBUG } 05:10:55 DEBUG 05:10:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499855" 05:10:55 DEBUG } 05:10:55 DEBUG ] 05:10:55 DEBUG } 05:10:55 DEBUG ] 05:10:55 DEBUG ] 05:10:55 DEBUG } 05:10:55 DEBUG } 05:10:55 DEBUG 05:10:55 DEBUG } 05:10:55 DEBUG } 05:10:55 DEBUG ] 05:10:55 DEBUG } 05:10:55 DEBUG } 05:10:55 DEBUG } 05:10:55 DEBUG 05:10:55 DEBUG } 05:10:55 DEBUG } 05:10:55 DEBUG } 05:10:55 DEBUG } 05:10:55 DEBUG [...] 05:10:55 DEBUG } 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG "instance": "10.142.0.43:9100" 05:10:55 DEBUG }, 05:10:55 DEBUG "value": [ 05:10:55 DEBUG 1713499855, 05:10:55 DEBUG "47935365.1" 05:10:55 DEBUG ] 05:10:55 DEBUG } 05:10:55 DEBUG ] 05:10:55 DEBUG } 05:10:55 DEBUG } 05:10:55 INFO [http_cmd]: http status code OK 05:10:55 DEBUG --- status code --- 05:10:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:10:55 DEBUG --- http response --- 05:10:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:10:55 DEBUG ----- output ----- 05:10:55 DEBUG { 05:10:55 DEBUG "status": "success", 05:10:55 DEBUG "data": { 05:10:55 DEBUG "resultType": "vector", 05:10:55 DEBUG "result": [ 05:10:55 DEBUG { 05:10:55 DEBUG "metric": { 05:10:55 DEBUG "pod": "ds-idrepo-0" 05:10:55 DEBUG }, 05:10:55 DEBUG "value": [ 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG [...] 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG "pod": "ds-idrepo-2" 05:10:55 DEBUG }, 05:10:55 DEBUG "value": [ 05:10:55 DEBUG 1713499855, 05:10:55 DEBUG "0" 05:10:55 DEBUG ] 05:10:55 DEBUG } 05:10:55 DEBUG ] 05:10:55 DEBUG } 05:10:55 DEBUG } 05:10:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499855" 05:10:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499855" 05:10:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713499855" 05:10:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499855" 05:10:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499855" 05:10:55 INFO [http_cmd]: http status code OK 05:10:55 INFO [http_cmd]: http status code OK 05:10:55 DEBUG --- status code --- 05:10:55 DEBUG --- status code --- 05:10:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:10:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:10:55 DEBUG --- http response --- 05:10:55 DEBUG --- http response --- 05:10:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:10:55 DEBUG ----- output ----- 05:10:55 DEBUG { 05:10:55 DEBUG "status": "success", 05:10:55 DEBUG "data": { 05:10:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:10:55 DEBUG "resultType": "vector", 05:10:55 DEBUG ----- output ----- 05:10:55 DEBUG "result": [ 05:10:55 DEBUG { 05:10:55 DEBUG { 05:10:55 DEBUG "status": "success", 05:10:55 DEBUG "metric": { 05:10:55 DEBUG "data": { 05:10:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:10:55 DEBUG "resultType": "vector", 05:10:55 DEBUG }, 05:10:55 DEBUG "result": [ 05:10:55 DEBUG "value": [ 05:10:55 DEBUG { 05:10:55 DEBUG 05:10:55 DEBUG "metric": { 05:10:55 DEBUG 05:10:55 DEBUG "pod": "ds-idrepo-1" 05:10:55 DEBUG 05:10:55 DEBUG }, 05:10:55 DEBUG [...] 05:10:55 DEBUG "value": [ 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:10:55 DEBUG [...] 05:10:55 DEBUG }, 05:10:55 DEBUG 05:10:55 DEBUG "value": [ 05:10:55 DEBUG 05:10:55 DEBUG 1713499855, 05:10:55 DEBUG 05:10:55 INFO [http_cmd]: http status code OK 05:10:55 DEBUG "2002.8666666666666" 05:10:55 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:10:55 DEBUG --- status code --- 05:10:55 DEBUG ] 05:10:55 DEBUG }, 05:10:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:10:55 DEBUG } 05:10:55 DEBUG "value": [ 05:10:55 DEBUG --- http response --- 05:10:55 DEBUG ] 05:10:55 DEBUG 1713499855, 05:10:55 DEBUG } 05:10:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:10:55 DEBUG "0" 05:10:55 DEBUG } 05:10:55 INFO [http_cmd]: http status code OK 05:10:55 DEBUG ----- output ----- 05:10:55 DEBUG ] 05:10:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713499855" 05:10:55 DEBUG --- status code --- 05:10:55 DEBUG { 05:10:55 DEBUG } 05:10:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:10:55 DEBUG "status": "success", 05:10:55 DEBUG ] 05:10:55 DEBUG --- http response --- 05:10:55 DEBUG "data": { 05:10:55 DEBUG } 05:10:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:10:55 DEBUG "resultType": "vector", 05:10:55 DEBUG } 05:10:55 DEBUG ----- output ----- 05:10:55 DEBUG "result": [ 05:10:55 DEBUG { 05:10:55 INFO [http_cmd]: http status code OK 05:10:55 DEBUG { 05:10:55 DEBUG "status": "success", 05:10:55 DEBUG --- status code --- 05:10:55 DEBUG "metric": { 05:10:55 DEBUG "data": { 05:10:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:10:55 DEBUG "pod": "ds-cts-0" 05:10:55 DEBUG "resultType": "vector", 05:10:55 DEBUG --- http response --- 05:10:55 DEBUG }, 05:10:55 DEBUG "result": [ 05:10:55 DEBUG "value": [ 05:10:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:10:55 DEBUG { 05:10:55 DEBUG 05:10:55 DEBUG ----- output ----- 05:10:55 DEBUG "metric": { 05:10:55 DEBUG 05:10:55 DEBUG { 05:10:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:10:55 DEBUG 05:10:55 DEBUG "status": "success", 05:10:55 DEBUG }, 05:10:55 DEBUG [...] 05:10:55 DEBUG "data": { 05:10:55 DEBUG "value": [ 05:10:55 DEBUG 05:10:55 DEBUG "resultType": "vector", 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG "result": [ 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG { 05:10:55 DEBUG 05:10:55 DEBUG "pod": "ds-cts-2" 05:10:55 DEBUG "metric": { 05:10:55 DEBUG [...] 05:10:55 DEBUG }, 05:10:55 DEBUG "pod": "ds-idrepo-0" 05:10:55 DEBUG 05:10:55 DEBUG "value": [ 05:10:55 DEBUG }, 05:10:55 DEBUG 05:10:55 DEBUG 1713499855, 05:10:55 DEBUG "value": [ 05:10:55 DEBUG 05:10:55 DEBUG "0" 05:10:55 DEBUG 05:10:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:10:55 DEBUG ] 05:10:55 DEBUG 05:10:55 DEBUG }, 05:10:55 DEBUG } 05:10:55 DEBUG 05:10:55 DEBUG "value": [ 05:10:55 DEBUG ] 05:10:55 DEBUG [...] 05:10:55 DEBUG 1713499855, 05:10:55 DEBUG } 05:10:55 DEBUG 05:10:55 DEBUG "2177548288" 05:10:55 DEBUG } 05:10:55 DEBUG 05:10:55 DEBUG ] 05:10:55 DEBUG 05:10:55 DEBUG } 05:10:55 DEBUG "pod": "ds-idrepo-2" 05:10:55 DEBUG ] 05:10:55 DEBUG }, 05:10:55 DEBUG } 05:10:55 DEBUG "value": [ 05:10:55 DEBUG } 05:10:55 DEBUG 1713499855, 05:10:55 DEBUG "0" 05:10:55 DEBUG ] 05:10:55 DEBUG } 05:10:55 DEBUG ] 05:10:55 DEBUG } 05:10:55 DEBUG } 05:10:55 INFO [http_cmd]: http status code OK 05:10:55 DEBUG --- status code --- 05:10:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:10:55 DEBUG --- http response --- 05:10:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:10:55 DEBUG ----- output ----- 05:10:55 DEBUG { 05:10:55 DEBUG "status": "success", 05:10:55 DEBUG "data": { 05:10:55 DEBUG "resultType": "vector", 05:10:55 DEBUG "result": [ 05:10:55 DEBUG { 05:10:55 DEBUG "metric": { 05:10:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:10:55 DEBUG }, 05:10:55 DEBUG "value": [ 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG [...] 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:10:55 DEBUG }, 05:10:55 DEBUG "value": [ 05:10:55 DEBUG 1713499855, 05:10:55 DEBUG "55.171" 05:10:55 DEBUG ] 05:10:55 DEBUG } 05:10:55 DEBUG ] 05:10:55 DEBUG } 05:10:55 DEBUG } 05:10:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499855" 05:10:55 INFO [http_cmd]: http status code OK 05:10:55 DEBUG --- status code --- 05:10:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:10:55 DEBUG --- http response --- 05:10:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:10:55 DEBUG ----- output ----- 05:10:55 DEBUG { 05:10:55 DEBUG "status": "success", 05:10:55 DEBUG "data": { 05:10:55 DEBUG "resultType": "vector", 05:10:55 DEBUG "result": [ 05:10:55 DEBUG { 05:10:55 DEBUG "metric": { 05:10:55 DEBUG "instance": "10.142.0.107:9100" 05:10:55 DEBUG }, 05:10:55 DEBUG "value": [ 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG [...] 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG "instance": "10.142.0.43:9100" 05:10:55 DEBUG }, 05:10:55 DEBUG "value": [ 05:10:55 DEBUG 1713499855, 05:10:55 DEBUG "47932183.83333333" 05:10:55 DEBUG ] 05:10:55 DEBUG } 05:10:55 DEBUG ] 05:10:55 DEBUG } 05:10:55 DEBUG } 05:10:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499855" 05:10:55 INFO [http_cmd]: http status code OK 05:10:55 DEBUG --- status code --- 05:10:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:10:55 DEBUG --- http response --- 05:10:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:10:55 DEBUG ----- output ----- 05:10:55 DEBUG { 05:10:55 DEBUG "status": "success", 05:10:55 DEBUG "data": { 05:10:55 DEBUG "resultType": "vector", 05:10:55 DEBUG "result": [ 05:10:55 DEBUG { 05:10:55 DEBUG "metric": { 05:10:55 DEBUG "instance": "10.142.0.107:9100" 05:10:55 DEBUG }, 05:10:55 DEBUG "value": [ 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG [...] 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG 05:10:55 DEBUG "instance": "10.142.0.43:9100" 05:10:55 DEBUG }, 05:10:55 DEBUG "value": [ 05:10:55 DEBUG 1713499855, 05:10:55 DEBUG "0.04033333333333502" 05:10:55 DEBUG ] 05:10:55 DEBUG } 05:10:55 DEBUG ] 05:10:55 DEBUG } 05:10:55 DEBUG } 05:10:56 INFO 05:10:56 INFO [loop_until]: kubectl --namespace=xlou top pods 05:10:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:10:56 INFO [loop_until]: OK (rc = 0) 05:10:56 DEBUG --- stdout --- 05:10:56 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 9919m 9122Mi am-6fd554f8f7-hvvkw 9261m 8875Mi am-6fd554f8f7-pmj4b 10187m 8975Mi ds-cts-0 7619m 12276Mi ds-cts-1 7056m 12199Mi ds-cts-2 7751m 12238Mi ds-idrepo-0 4697m 3256Mi ds-idrepo-1 4701m 3207Mi ds-idrepo-2 4589m 3065Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1172Mi idm-b85f9d978-wnhxb 3m 1215Mi lodemon-6b89d774d5-rf7sk 10m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 6551m 1984Mi 05:10:56 DEBUG --- stderr --- 05:10:56 DEBUG 05:10:57 INFO 05:10:57 INFO [loop_until]: kubectl --namespace=xlou top node 05:10:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:10:57 INFO [loop_until]: OK (rc = 0) 05:10:57 DEBUG --- stdout --- 05:10:57 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 80m 0% 6254Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 4656m 15% 6704Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9463m 31% 11595Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 87m 0% 3256Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 7330m 24% 14834Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 159m 0% 5156Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 9969m 33% 11913Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 10334m 34% 2721Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 11132m 37% 11176Mi 10% gke-xlou-cdm-ds-0f601751-j488 4896m 16% 6168Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 7507m 25% 16221Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 5152m 17% 6874Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 7591m 25% 14206Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 6708m 42% 5723Mi 9% 05:10:57 DEBUG --- stderr --- 05:10:57 DEBUG 05:11:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713499915" 05:11:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499915" 05:11:55 INFO [http_cmd]: http status code OK 05:11:55 DEBUG --- status code --- 05:11:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:11:55 DEBUG --- http response --- 05:11:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:11:55 DEBUG ----- output ----- 05:11:55 DEBUG { 05:11:55 DEBUG "status": "success", 05:11:55 DEBUG "data": { 05:11:55 DEBUG "resultType": "vector", 05:11:55 DEBUG "result": [ 05:11:55 DEBUG { 05:11:55 DEBUG "metric": { 05:11:55 DEBUG "pod": "ds-idrepo-0" 05:11:55 DEBUG }, 05:11:55 DEBUG "value": [ 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG [...] 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG "pod": "ds-cts-2" 05:11:55 DEBUG }, 05:11:55 DEBUG "value": [ 05:11:55 DEBUG 1713499915, 05:11:55 DEBUG "0" 05:11:55 DEBUG ] 05:11:55 DEBUG } 05:11:55 DEBUG ] 05:11:55 DEBUG } 05:11:55 DEBUG } 05:11:55 INFO [http_cmd]: http status code OK 05:11:55 DEBUG --- status code --- 05:11:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:11:55 DEBUG --- http response --- 05:11:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:11:55 DEBUG ----- output ----- 05:11:55 DEBUG { 05:11:55 DEBUG "status": "success", 05:11:55 DEBUG "data": { 05:11:55 DEBUG "resultType": "vector", 05:11:55 DEBUG "result": [ 05:11:55 DEBUG { 05:11:55 DEBUG "metric": { 05:11:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:11:55 DEBUG }, 05:11:55 DEBUG "value": [ 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG [...] 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:11:55 DEBUG }, 05:11:55 DEBUG "value": [ 05:11:55 DEBUG 1713499915, 05:11:55 DEBUG "3222906.5" 05:11:55 DEBUG ] 05:11:55 DEBUG } 05:11:55 DEBUG ] 05:11:55 DEBUG } 05:11:55 DEBUG } 05:11:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713499915" 05:11:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713499915" 05:11:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499915" 05:11:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499915" 05:11:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499915" 05:11:55 INFO [http_cmd]: http status code OK 05:11:55 DEBUG --- status code --- 05:11:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:11:55 DEBUG --- http response --- 05:11:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:11:55 DEBUG ----- output ----- 05:11:55 DEBUG { 05:11:55 DEBUG "status": "success", 05:11:55 DEBUG "data": { 05:11:55 DEBUG "resultType": "vector", 05:11:55 DEBUG "result": [ 05:11:55 DEBUG { 05:11:55 DEBUG "metric": { 05:11:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:11:55 DEBUG }, 05:11:55 DEBUG "value": [ 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG [...] 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG "pod": "ds-idrepo-1" 05:11:55 DEBUG }, 05:11:55 DEBUG "value": [ 05:11:55 DEBUG 1713499915, 05:11:55 DEBUG "140636845.10103583" 05:11:55 DEBUG ] 05:11:55 DEBUG } 05:11:55 DEBUG ] 05:11:55 DEBUG } 05:11:55 DEBUG } 05:11:55 ERROR [http_cmd]: ERROR 05:11:55 INFO [http_cmd]: http status code OK 05:11:55 ERROR --- status code --- 05:11:55 DEBUG --- status code --- 05:11:55 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:11:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:11:55 ERROR --- http response --- 05:11:55 DEBUG --- http response --- 05:11:55 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:11:55 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:11:55 INFO [http_cmd]: http status code OK 05:11:55 DEBUG --- status code --- 05:11:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:11:55 DEBUG --- http response --- 05:11:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:11:55 DEBUG ----- output ----- 05:11:55 DEBUG { 05:11:55 DEBUG "status": "success", 05:11:55 DEBUG "data": { 05:11:55 DEBUG "resultType": "vector", 05:11:55 DEBUG "result": [ 05:11:55 DEBUG { 05:11:55 DEBUG "metric": { 05:11:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:11:55 DEBUG }, 05:11:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:11:55 DEBUG "value": [ 05:11:55 DEBUG ----- output ----- 05:11:55 DEBUG 05:11:55 DEBUG { 05:11:55 DEBUG 05:11:55 DEBUG "status": "success", 05:11:55 DEBUG 05:11:55 DEBUG "data": { 05:11:55 DEBUG [...] 05:11:55 DEBUG "resultType": "vector", 05:11:55 DEBUG 05:11:55 DEBUG "result": [ 05:11:55 DEBUG 05:11:55 DEBUG { 05:11:55 DEBUG 05:11:55 DEBUG "metric": { 05:11:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:11:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:11:55 DEBUG }, 05:11:55 DEBUG }, 05:11:55 DEBUG "value": [ 05:11:55 DEBUG "value": [ 05:11:55 INFO [http_cmd]: http status code OK 05:11:55 DEBUG 1713499915, 05:11:55 DEBUG 05:11:55 DEBUG --- status code --- 05:11:55 DEBUG "0" 05:11:55 DEBUG 05:11:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:11:55 DEBUG ] 05:11:55 DEBUG 05:11:55 DEBUG --- http response --- 05:11:55 DEBUG } 05:11:55 DEBUG [...] 05:11:55 DEBUG ] 05:11:55 DEBUG 05:11:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:11:55 DEBUG } 05:11:55 DEBUG 05:11:55 DEBUG ----- output ----- 05:11:55 DEBUG } 05:11:55 DEBUG 05:11:55 DEBUG { 05:11:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:11:55 DEBUG "status": "success", 05:11:55 DEBUG }, 05:11:55 DEBUG "data": { 05:11:55 DEBUG "value": [ 05:11:55 DEBUG "resultType": "vector", 05:11:55 DEBUG 1713499915, 05:11:55 DEBUG "result": [ 05:11:55 DEBUG "6.535187068414914" 05:11:55 DEBUG ] 05:11:55 DEBUG { 05:11:55 DEBUG } 05:11:55 DEBUG "metric": { 05:11:55 DEBUG ] 05:11:55 DEBUG "instance": "10.142.0.107:9100" 05:11:55 DEBUG } 05:11:55 DEBUG }, 05:11:55 DEBUG } 05:11:55 DEBUG "value": [ 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG [...] 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG "instance": "10.142.0.43:9100" 05:11:55 DEBUG }, 05:11:55 DEBUG "value": [ 05:11:55 DEBUG 1713499915, 05:11:55 DEBUG "0.0629000000000038" 05:11:55 DEBUG ] 05:11:55 DEBUG } 05:11:55 DEBUG ] 05:11:55 DEBUG } 05:11:55 DEBUG } 05:11:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499915" 05:11:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499915" 05:11:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499915" 05:11:55 INFO [http_cmd]: http status code OK 05:11:55 DEBUG --- status code --- 05:11:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:11:55 DEBUG --- http response --- 05:11:55 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:11:55 INFO [http_cmd]: http status code OK 05:11:55 DEBUG --- status code --- 05:11:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:11:55 DEBUG --- http response --- 05:11:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:11:55 DEBUG ----- output ----- 05:11:55 DEBUG { 05:11:55 DEBUG "status": "success", 05:11:55 DEBUG "data": { 05:11:55 DEBUG "resultType": "vector", 05:11:55 DEBUG "result": [ 05:11:55 DEBUG { 05:11:55 DEBUG "metric": { 05:11:55 DEBUG "pod": "ds-idrepo-0" 05:11:55 DEBUG }, 05:11:55 DEBUG "value": [ 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG [...] 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG "pod": "ds-idrepo-2" 05:11:55 DEBUG }, 05:11:55 DEBUG "value": [ 05:11:55 DEBUG 1713499915, 05:11:55 DEBUG "0" 05:11:55 DEBUG ] 05:11:55 DEBUG } 05:11:55 DEBUG ] 05:11:55 DEBUG } 05:11:55 DEBUG } 05:11:55 INFO [http_cmd]: http status code OK 05:11:55 DEBUG --- status code --- 05:11:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:11:55 DEBUG --- http response --- 05:11:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499915" 05:11:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:11:55 DEBUG ----- output ----- 05:11:55 DEBUG { 05:11:55 DEBUG "status": "success", 05:11:55 DEBUG "data": { 05:11:55 DEBUG "resultType": "vector", 05:11:55 DEBUG "result": [ 05:11:55 DEBUG { 05:11:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499915" 05:11:55 DEBUG "metric": { 05:11:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:11:55 DEBUG }, 05:11:55 DEBUG "value": [ 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG [...] 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG "pod": "ds-idrepo-1" 05:11:55 DEBUG }, 05:11:55 DEBUG "value": [ 05:11:55 DEBUG 1713499915, 05:11:55 DEBUG "4.702957412124403" 05:11:55 DEBUG ] 05:11:55 DEBUG } 05:11:55 DEBUG ] 05:11:55 DEBUG } 05:11:55 DEBUG } 05:11:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499915" 05:11:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499915" 05:11:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499915" 05:11:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499915" 05:11:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499915" 05:11:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499915" 05:11:55 INFO [http_cmd]: http status code OK 05:11:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499915" 05:11:55 DEBUG --- status code --- 05:11:55 INFO [http_cmd]: http status code OK 05:11:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499915" 05:11:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:11:55 INFO [http_cmd]: http status code OK 05:11:55 INFO [http_cmd]: http status code OK 05:11:55 DEBUG --- status code --- 05:11:55 DEBUG --- http response --- 05:11:55 INFO [http_cmd]: http status code OK 05:11:55 DEBUG --- status code --- 05:11:55 DEBUG --- status code --- 05:11:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:11:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:11:55 DEBUG --- status code --- 05:11:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:11:55 DEBUG --- http response --- 05:11:55 DEBUG --- http response --- 05:11:55 DEBUG ----- output ----- 05:11:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:11:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:11:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:11:55 DEBUG { 05:11:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:11:55 DEBUG --- http response --- 05:11:55 DEBUG --- http response --- 05:11:55 DEBUG ----- output ----- 05:11:55 DEBUG "status": "success", 05:11:55 DEBUG ----- output ----- 05:11:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:11:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:11:55 DEBUG { 05:11:55 DEBUG "data": { 05:11:55 DEBUG { 05:11:55 DEBUG ----- output ----- 05:11:55 DEBUG ----- output ----- 05:11:55 DEBUG "status": "success", 05:11:55 DEBUG "resultType": "vector", 05:11:55 DEBUG "status": "success", 05:11:55 DEBUG { 05:11:55 DEBUG { 05:11:55 DEBUG "data": { 05:11:55 DEBUG "result": [ 05:11:55 DEBUG "data": { 05:11:55 DEBUG "status": "success", 05:11:55 DEBUG "status": "success", 05:11:55 DEBUG "resultType": "vector", 05:11:55 DEBUG { 05:11:55 INFO [http_cmd]: http status code OK 05:11:55 DEBUG "resultType": "vector", 05:11:55 DEBUG "data": { 05:11:55 DEBUG "data": { 05:11:55 DEBUG "result": [ 05:11:55 DEBUG "metric": { 05:11:55 DEBUG --- status code --- 05:11:55 DEBUG "result": [ 05:11:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499915" 05:11:55 DEBUG "resultType": "vector", 05:11:55 INFO [http_cmd]: http status code OK 05:11:55 DEBUG "resultType": "vector", 05:11:55 INFO [http_cmd]: http status code OK 05:11:55 INFO [http_cmd]: http status code OK 05:11:55 DEBUG { 05:11:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:11:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:11:55 DEBUG { 05:11:55 DEBUG "result": [ 05:11:55 DEBUG --- status code --- 05:11:55 DEBUG "result": [ 05:11:55 DEBUG --- status code --- 05:11:55 DEBUG --- status code --- 05:11:55 DEBUG "metric": { 05:11:55 DEBUG }, 05:11:55 DEBUG --- http response --- 05:11:55 DEBUG "metric": { 05:11:55 DEBUG { 05:11:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:11:55 DEBUG { 05:11:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:11:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:11:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:11:55 DEBUG "value": [ 05:11:55 INFO [http_cmd]: http status code OK 05:11:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:11:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:11:55 DEBUG "metric": { 05:11:55 DEBUG --- http response --- 05:11:55 DEBUG "metric": { 05:11:55 DEBUG --- http response --- 05:11:55 DEBUG --- http response --- 05:11:55 DEBUG }, 05:11:55 DEBUG 05:11:55 DEBUG --- status code --- 05:11:55 DEBUG }, 05:11:55 DEBUG ----- output ----- 05:11:55 DEBUG "pod": "ds-cts-0" 05:11:55 DEBUG "pod": "ds-cts-0" 05:11:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:11:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:11:55 DEBUG "value": [ 05:11:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:11:55 DEBUG 05:11:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:11:55 DEBUG "value": [ 05:11:55 DEBUG { 05:11:55 DEBUG }, 05:11:55 DEBUG }, 05:11:55 DEBUG ----- output ----- 05:11:55 DEBUG ----- output ----- 05:11:55 DEBUG 05:11:55 DEBUG ----- output ----- 05:11:55 DEBUG 05:11:55 DEBUG --- http response --- 05:11:55 DEBUG 05:11:55 DEBUG "status": "success", 05:11:55 DEBUG "value": [ 05:11:55 DEBUG "value": [ 05:11:55 DEBUG { 05:11:55 DEBUG { 05:11:55 DEBUG 05:11:55 DEBUG { 05:11:55 DEBUG [...] 05:11:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:11:55 DEBUG 05:11:55 DEBUG "data": { 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG "status": "success", 05:11:55 DEBUG "status": "success", 05:11:55 DEBUG 05:11:55 DEBUG "status": "success", 05:11:55 DEBUG 05:11:55 DEBUG ----- output ----- 05:11:55 DEBUG 05:11:55 DEBUG "resultType": "vector", 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG "data": { 05:11:55 DEBUG "data": { 05:11:55 DEBUG [...] 05:11:55 DEBUG "data": { 05:11:55 DEBUG 05:11:55 DEBUG { 05:11:55 DEBUG [...] 05:11:55 DEBUG "result": [ 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG "resultType": "vector", 05:11:55 DEBUG "resultType": "vector", 05:11:55 DEBUG 05:11:55 DEBUG "resultType": "vector", 05:11:55 DEBUG 05:11:55 DEBUG "status": "success", 05:11:55 DEBUG 05:11:55 DEBUG { 05:11:55 DEBUG [...] 05:11:55 DEBUG [...] 05:11:55 DEBUG "result": [ 05:11:55 DEBUG "result": [ 05:11:55 DEBUG 05:11:55 DEBUG "result": [ 05:11:55 DEBUG "pod": "ds-idrepo-1" 05:11:55 DEBUG "data": { 05:11:55 DEBUG 05:11:55 DEBUG "metric": { 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG { 05:11:55 DEBUG { 05:11:55 DEBUG 05:11:55 DEBUG { 05:11:55 DEBUG }, 05:11:55 DEBUG "resultType": "vector", 05:11:55 DEBUG 05:11:55 DEBUG "pod": "ds-idrepo-0" 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG "metric": { 05:11:55 DEBUG "metric": { 05:11:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:11:55 DEBUG "metric": { 05:11:55 DEBUG "value": [ 05:11:55 DEBUG "result": [ 05:11:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:11:55 DEBUG }, 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG "instance": "10.142.0.107:9100" 05:11:55 DEBUG "instance": "10.142.0.107:9100" 05:11:55 DEBUG }, 05:11:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:11:55 DEBUG 1713499915, 05:11:55 DEBUG { 05:11:55 DEBUG }, 05:11:55 DEBUG "value": [ 05:11:55 DEBUG "pod": "ds-cts-2" 05:11:55 DEBUG "pod": "ds-cts-2" 05:11:55 DEBUG }, 05:11:55 DEBUG }, 05:11:55 DEBUG "value": [ 05:11:55 DEBUG }, 05:11:55 DEBUG "5425.165562913908" 05:11:55 DEBUG "metric": { 05:11:55 DEBUG "value": [ 05:11:55 DEBUG 05:11:55 DEBUG }, 05:11:55 DEBUG }, 05:11:55 DEBUG "value": [ 05:11:55 DEBUG "value": [ 05:11:55 DEBUG 1713499915, 05:11:55 DEBUG "value": [ 05:11:55 DEBUG ] 05:11:55 DEBUG "pod": "ds-cts-0" 05:11:55 DEBUG 1713499915, 05:11:55 DEBUG 05:11:55 DEBUG "value": [ 05:11:55 DEBUG "value": [ 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG "0" 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG }, 05:11:55 INFO [http_cmd]: http status code OK 05:11:55 DEBUG "1997.8999999999999" 05:11:55 DEBUG 05:11:55 DEBUG 1713499915, 05:11:55 DEBUG 1713499915, 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG ] 05:11:55 DEBUG } 05:11:55 DEBUG 05:11:55 DEBUG "value": [ 05:11:55 DEBUG --- status code --- 05:11:55 DEBUG ] 05:11:55 DEBUG [...] 05:11:55 DEBUG "2.533333333333333" 05:11:55 DEBUG "0" 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG } 05:11:55 DEBUG ] 05:11:55 DEBUG [...] 05:11:55 DEBUG 05:11:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:11:55 DEBUG } 05:11:55 DEBUG 05:11:55 DEBUG ] 05:11:55 DEBUG ] 05:11:55 DEBUG [...] 05:11:55 DEBUG [...] 05:11:55 DEBUG ] 05:11:55 DEBUG } 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG --- http response --- 05:11:55 DEBUG ] 05:11:55 DEBUG 05:11:55 DEBUG } 05:11:55 DEBUG } 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG } 05:11:55 DEBUG } 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG } 05:11:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:11:55 DEBUG 05:11:55 DEBUG ] 05:11:55 DEBUG ] 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG } 05:11:55 DEBUG 05:11:55 DEBUG [...] 05:11:55 DEBUG } 05:11:55 DEBUG ----- output ----- 05:11:55 DEBUG "pod": "ds-idrepo-2" 05:11:55 DEBUG } 05:11:55 DEBUG } 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:11:55 DEBUG 05:11:55 DEBUG { 05:11:55 DEBUG }, 05:11:55 DEBUG } 05:11:55 DEBUG } 05:11:55 DEBUG "instance": "10.142.0.43:9100" 05:11:55 DEBUG "instance": "10.142.0.43:9100" 05:11:55 DEBUG }, 05:11:55 DEBUG 05:11:55 DEBUG "status": "success", 05:11:55 DEBUG "value": [ 05:11:55 DEBUG }, 05:11:55 DEBUG }, 05:11:55 DEBUG "value": [ 05:11:55 DEBUG 05:11:55 DEBUG "data": { 05:11:55 DEBUG 1713499915, 05:11:55 DEBUG "value": [ 05:11:55 DEBUG "value": [ 05:11:55 DEBUG 1713499915, 05:11:55 DEBUG "pod": "ds-cts-2" 05:11:55 DEBUG "resultType": "vector", 05:11:55 DEBUG "0" 05:11:55 DEBUG 1713499915, 05:11:55 DEBUG 1713499915, 05:11:55 DEBUG "1997.8" 05:11:55 DEBUG }, 05:11:55 DEBUG "result": [ 05:11:55 DEBUG ] 05:11:55 DEBUG "449098116096" 05:11:55 DEBUG "47725706.03333333" 05:11:55 DEBUG ] 05:11:55 DEBUG "value": [ 05:11:55 DEBUG { 05:11:55 DEBUG } 05:11:55 DEBUG ] 05:11:55 DEBUG ] 05:11:55 DEBUG } 05:11:55 DEBUG 1713499915, 05:11:55 DEBUG "metric": { 05:11:55 DEBUG ] 05:11:55 DEBUG } 05:11:55 DEBUG } 05:11:55 DEBUG ] 05:11:55 DEBUG "0.000017660000000000004" 05:11:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:11:55 DEBUG } 05:11:55 DEBUG ] 05:11:55 DEBUG ] 05:11:55 DEBUG } 05:11:55 DEBUG ] 05:11:55 DEBUG }, 05:11:55 DEBUG } 05:11:55 DEBUG } 05:11:55 DEBUG } 05:11:55 DEBUG } 05:11:55 DEBUG } 05:11:55 DEBUG "value": [ 05:11:55 DEBUG } 05:11:55 DEBUG } 05:11:55 DEBUG ] 05:11:55 DEBUG 05:11:55 DEBUG } 05:11:55 DEBUG 05:11:55 DEBUG } 05:11:55 DEBUG 05:11:55 DEBUG [...] 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:11:55 DEBUG }, 05:11:55 DEBUG "value": [ 05:11:55 DEBUG 1713499915, 05:11:55 DEBUG "1998.1666666666667" 05:11:55 DEBUG ] 05:11:55 DEBUG } 05:11:55 DEBUG ] 05:11:55 DEBUG } 05:11:55 DEBUG } 05:11:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499915" 05:11:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713499915" 05:11:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499915" 05:11:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713499915" 05:11:55 INFO [http_cmd]: http status code OK 05:11:55 DEBUG --- status code --- 05:11:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:11:55 DEBUG --- http response --- 05:11:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:11:55 DEBUG ----- output ----- 05:11:55 DEBUG { 05:11:55 DEBUG "status": "success", 05:11:55 DEBUG "data": { 05:11:55 DEBUG "resultType": "vector", 05:11:55 DEBUG "result": [ 05:11:55 DEBUG { 05:11:55 DEBUG "metric": { 05:11:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:11:55 DEBUG }, 05:11:55 DEBUG "value": [ 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG [...] 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG "pod": "ds-idrepo-1" 05:11:55 DEBUG }, 05:11:55 DEBUG "value": [ 05:11:55 DEBUG 1713499915, 05:11:55 DEBUG "0" 05:11:55 DEBUG ] 05:11:55 DEBUG } 05:11:55 DEBUG ] 05:11:55 DEBUG } 05:11:55 DEBUG } 05:11:55 INFO [http_cmd]: http status code OK 05:11:55 DEBUG --- status code --- 05:11:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:11:55 DEBUG --- http response --- 05:11:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:11:55 DEBUG ----- output ----- 05:11:55 DEBUG { 05:11:55 DEBUG "status": "success", 05:11:55 DEBUG "data": { 05:11:55 DEBUG "resultType": "vector", 05:11:55 DEBUG "result": [ 05:11:55 DEBUG { 05:11:55 DEBUG "metric": { 05:11:55 DEBUG "pod": "ds-idrepo-0" 05:11:55 DEBUG }, 05:11:55 DEBUG "value": [ 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG [...] 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG "pod": "ds-idrepo-2" 05:11:55 DEBUG }, 05:11:55 DEBUG "value": [ 05:11:55 DEBUG 1713499915, 05:11:55 DEBUG "0" 05:11:55 DEBUG ] 05:11:55 DEBUG } 05:11:55 DEBUG ] 05:11:55 DEBUG } 05:11:55 DEBUG } 05:11:55 INFO [http_cmd]: http status code OK 05:11:55 DEBUG --- status code --- 05:11:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:11:55 DEBUG --- http response --- 05:11:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:11:55 DEBUG ----- output ----- 05:11:55 DEBUG { 05:11:55 DEBUG "status": "success", 05:11:55 DEBUG "data": { 05:11:55 DEBUG "resultType": "vector", 05:11:55 DEBUG "result": [ 05:11:55 DEBUG { 05:11:55 DEBUG "metric": { 05:11:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:11:55 DEBUG }, 05:11:55 DEBUG "value": [ 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG [...] 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:11:55 DEBUG }, 05:11:55 DEBUG "value": [ 05:11:55 DEBUG 1713499915, 05:11:55 DEBUG "2177359872" 05:11:55 DEBUG ] 05:11:55 DEBUG } 05:11:55 DEBUG ] 05:11:55 DEBUG } 05:11:55 DEBUG } 05:11:55 INFO [http_cmd]: http status code OK 05:11:55 DEBUG --- status code --- 05:11:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:11:55 DEBUG --- http response --- 05:11:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:11:55 DEBUG ----- output ----- 05:11:55 DEBUG { 05:11:55 DEBUG "status": "success", 05:11:55 DEBUG "data": { 05:11:55 DEBUG "resultType": "vector", 05:11:55 DEBUG "result": [ 05:11:55 DEBUG { 05:11:55 DEBUG "metric": { 05:11:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:11:55 DEBUG }, 05:11:55 DEBUG "value": [ 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG [...] 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:11:55 DEBUG }, 05:11:55 DEBUG "value": [ 05:11:55 DEBUG 1713499915, 05:11:55 DEBUG "55.171" 05:11:55 DEBUG ] 05:11:55 DEBUG } 05:11:55 DEBUG ] 05:11:55 DEBUG } 05:11:55 DEBUG } 05:11:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499915" 05:11:55 INFO [http_cmd]: http status code OK 05:11:55 DEBUG --- status code --- 05:11:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:11:55 DEBUG --- http response --- 05:11:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:11:55 DEBUG ----- output ----- 05:11:55 DEBUG { 05:11:55 DEBUG "status": "success", 05:11:55 DEBUG "data": { 05:11:55 DEBUG "resultType": "vector", 05:11:55 DEBUG "result": [ 05:11:55 DEBUG { 05:11:55 DEBUG "metric": { 05:11:55 DEBUG "instance": "10.142.0.107:9100" 05:11:55 DEBUG }, 05:11:55 DEBUG "value": [ 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG [...] 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG "instance": "10.142.0.43:9100" 05:11:55 DEBUG }, 05:11:55 DEBUG "value": [ 05:11:55 DEBUG 1713499915, 05:11:55 DEBUG "47724131.599999994" 05:11:55 DEBUG ] 05:11:55 DEBUG } 05:11:55 DEBUG ] 05:11:55 DEBUG } 05:11:55 DEBUG } 05:11:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499915" 05:11:55 INFO [http_cmd]: http status code OK 05:11:55 DEBUG --- status code --- 05:11:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:11:55 DEBUG --- http response --- 05:11:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:11:55 DEBUG ----- output ----- 05:11:55 DEBUG { 05:11:55 DEBUG "status": "success", 05:11:55 DEBUG "data": { 05:11:55 DEBUG "resultType": "vector", 05:11:55 DEBUG "result": [ 05:11:55 DEBUG { 05:11:55 DEBUG "metric": { 05:11:55 DEBUG "instance": "10.142.0.107:9100" 05:11:55 DEBUG }, 05:11:55 DEBUG "value": [ 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG [...] 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG 05:11:55 DEBUG "instance": "10.142.0.43:9100" 05:11:55 DEBUG }, 05:11:55 DEBUG "value": [ 05:11:55 DEBUG 1713499915, 05:11:55 DEBUG "0.040666666666668516" 05:11:55 DEBUG ] 05:11:55 DEBUG } 05:11:55 DEBUG ] 05:11:55 DEBUG } 05:11:55 DEBUG } 05:11:56 INFO 05:11:56 INFO [loop_until]: kubectl --namespace=xlou top pods 05:11:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:11:56 INFO [loop_until]: OK (rc = 0) 05:11:56 DEBUG --- stdout --- 05:11:56 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 10004m 9122Mi am-6fd554f8f7-hvvkw 9246m 8877Mi am-6fd554f8f7-pmj4b 10157m 8975Mi ds-cts-0 7872m 12284Mi ds-cts-1 7524m 12206Mi ds-cts-2 7715m 12246Mi ds-idrepo-0 4593m 3257Mi ds-idrepo-1 4681m 3207Mi ds-idrepo-2 4541m 3065Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 4m 1215Mi lodemon-6b89d774d5-rf7sk 10m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 6499m 1985Mi 05:11:56 DEBUG --- stderr --- 05:11:56 DEBUG 05:11:57 INFO 05:11:57 INFO [loop_until]: kubectl --namespace=xlou top node 05:11:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:11:57 INFO [loop_until]: OK (rc = 0) 05:11:57 DEBUG --- stdout --- 05:11:57 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 70m 0% 6252Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 4631m 15% 6694Mi 5% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9629m 32% 11598Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 3239Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 7773m 26% 14836Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 146m 0% 5162Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 9927m 33% 11904Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 10282m 34% 2730Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 11097m 37% 11171Mi 10% gke-xlou-cdm-ds-0f601751-j488 4836m 16% 6169Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 7442m 24% 16238Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 5170m 17% 6879Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 7937m 26% 14231Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 6461m 40% 5724Mi 9% 05:11:57 DEBUG --- stderr --- 05:11:57 DEBUG 05:12:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713499975" 05:12:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499975" 05:12:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499975" 05:12:55 INFO [http_cmd]: http status code OK 05:12:55 DEBUG --- status code --- 05:12:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:12:55 DEBUG --- http response --- 05:12:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:12:55 DEBUG ----- output ----- 05:12:55 DEBUG { 05:12:55 DEBUG "status": "success", 05:12:55 DEBUG "data": { 05:12:55 DEBUG "resultType": "vector", 05:12:55 DEBUG "result": [ 05:12:55 DEBUG { 05:12:55 DEBUG "metric": { 05:12:55 DEBUG "pod": "ds-idrepo-0" 05:12:55 DEBUG }, 05:12:55 DEBUG "value": [ 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG [...] 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG "pod": "ds-cts-2" 05:12:55 DEBUG }, 05:12:55 DEBUG "value": [ 05:12:55 DEBUG 1713499975, 05:12:55 DEBUG "0" 05:12:55 DEBUG ] 05:12:55 DEBUG } 05:12:55 DEBUG ] 05:12:55 DEBUG } 05:12:55 DEBUG } 05:12:55 INFO [http_cmd]: http status code OK 05:12:55 DEBUG --- status code --- 05:12:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:12:55 DEBUG --- http response --- 05:12:55 INFO [http_cmd]: http status code OK 05:12:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:12:55 DEBUG --- status code --- 05:12:55 DEBUG ----- output ----- 05:12:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:12:55 DEBUG { 05:12:55 DEBUG --- http response --- 05:12:55 DEBUG "status": "success", 05:12:55 DEBUG "data": { 05:12:55 DEBUG "resultType": "vector", 05:12:55 DEBUG "result": [ 05:12:55 DEBUG { 05:12:55 DEBUG "metric": { 05:12:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:12:55 DEBUG }, 05:12:55 DEBUG "value": [ 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG [...] 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:12:55 DEBUG }, 05:12:55 DEBUG "value": [ 05:12:55 DEBUG 1713499975, 05:12:55 DEBUG "3342833.5" 05:12:55 DEBUG ] 05:12:55 DEBUG } 05:12:55 DEBUG ] 05:12:55 DEBUG } 05:12:55 DEBUG } 05:12:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:12:55 DEBUG ----- output ----- 05:12:55 DEBUG { 05:12:55 DEBUG "status": "success", 05:12:55 DEBUG "data": { 05:12:55 DEBUG "resultType": "vector", 05:12:55 DEBUG "result": [ 05:12:55 DEBUG { 05:12:55 DEBUG "metric": { 05:12:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:12:55 DEBUG }, 05:12:55 DEBUG "value": [ 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG [...] 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:12:55 DEBUG }, 05:12:55 DEBUG "value": [ 05:12:55 DEBUG 1713499975, 05:12:55 DEBUG "1135.0833879175693" 05:12:55 DEBUG ] 05:12:55 DEBUG } 05:12:55 DEBUG ] 05:12:55 DEBUG } 05:12:55 DEBUG } 05:12:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499975" 05:12:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713499975" 05:12:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713499975" 05:12:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499975" 05:12:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499975" 05:12:55 INFO [http_cmd]: http status code OK 05:12:55 DEBUG --- status code --- 05:12:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:12:55 DEBUG --- http response --- 05:12:55 ERROR [http_cmd]: ERROR 05:12:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:12:55 INFO [http_cmd]: http status code OK 05:12:55 ERROR --- status code --- 05:12:55 DEBUG ----- output ----- 05:12:55 DEBUG --- status code --- 05:12:55 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:12:55 DEBUG { 05:12:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:12:55 ERROR --- http response --- 05:12:55 DEBUG "status": "success", 05:12:55 DEBUG --- http response --- 05:12:55 DEBUG "data": { 05:12:55 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:12:55 DEBUG "resultType": "vector", 05:12:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:12:55 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:12:55 DEBUG "result": [ 05:12:55 DEBUG ----- output ----- 05:12:55 DEBUG { 05:12:55 DEBUG { 05:12:55 DEBUG "metric": { 05:12:55 DEBUG "status": "success", 05:12:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:12:55 DEBUG "data": { 05:12:55 DEBUG }, 05:12:55 DEBUG "resultType": "vector", 05:12:55 DEBUG "value": [ 05:12:55 DEBUG "result": [ 05:12:55 DEBUG 05:12:55 DEBUG { 05:12:55 DEBUG 05:12:55 DEBUG "metric": { 05:12:55 DEBUG 05:12:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:12:55 DEBUG [...] 05:12:55 DEBUG }, 05:12:55 DEBUG 05:12:55 DEBUG "value": [ 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:12:55 DEBUG 05:12:55 DEBUG }, 05:12:55 DEBUG [...] 05:12:55 DEBUG "value": [ 05:12:55 DEBUG 05:12:55 DEBUG 1713499975, 05:12:55 DEBUG 05:12:55 DEBUG "9.485415654759002" 05:12:55 DEBUG 05:12:55 DEBUG ] 05:12:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:12:55 DEBUG } 05:12:55 DEBUG }, 05:12:55 DEBUG ] 05:12:55 DEBUG "value": [ 05:12:55 DEBUG } 05:12:55 DEBUG 1713499975, 05:12:55 DEBUG } 05:12:55 DEBUG "6.503789264252398" 05:12:55 DEBUG ] 05:12:55 DEBUG } 05:12:55 DEBUG ] 05:12:55 DEBUG } 05:12:55 DEBUG } 05:12:55 INFO [http_cmd]: http status code OK 05:12:55 DEBUG --- status code --- 05:12:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:12:55 DEBUG --- http response --- 05:12:55 INFO [http_cmd]: http status code OK 05:12:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:12:55 DEBUG --- status code --- 05:12:55 DEBUG ----- output ----- 05:12:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:12:55 DEBUG { 05:12:55 DEBUG --- http response --- 05:12:55 DEBUG "status": "success", 05:12:55 DEBUG "data": { 05:12:55 DEBUG "resultType": "vector", 05:12:55 DEBUG "result": [ 05:12:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:12:55 DEBUG { 05:12:55 DEBUG ----- output ----- 05:12:55 DEBUG "metric": { 05:12:55 DEBUG { 05:12:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:12:55 DEBUG "status": "success", 05:12:55 DEBUG }, 05:12:55 DEBUG "data": { 05:12:55 DEBUG "value": [ 05:12:55 DEBUG "resultType": "vector", 05:12:55 DEBUG 05:12:55 DEBUG "result": [ 05:12:55 DEBUG 05:12:55 DEBUG { 05:12:55 DEBUG 05:12:55 DEBUG "metric": { 05:12:55 DEBUG [...] 05:12:55 DEBUG "instance": "10.142.0.107:9100" 05:12:55 DEBUG 05:12:55 DEBUG }, 05:12:55 DEBUG 05:12:55 DEBUG "value": [ 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:12:55 DEBUG 05:12:55 DEBUG }, 05:12:55 DEBUG 05:12:55 DEBUG "value": [ 05:12:55 DEBUG [...] 05:12:55 DEBUG 1713499975, 05:12:55 DEBUG 05:12:55 DEBUG "0" 05:12:55 DEBUG 05:12:55 DEBUG ] 05:12:55 DEBUG 05:12:55 DEBUG } 05:12:55 DEBUG "instance": "10.142.0.43:9100" 05:12:55 DEBUG ] 05:12:55 DEBUG }, 05:12:55 DEBUG } 05:12:55 DEBUG "value": [ 05:12:55 DEBUG } 05:12:55 DEBUG 1713499975, 05:12:55 DEBUG "0.06236666666666792" 05:12:55 DEBUG ] 05:12:55 DEBUG } 05:12:55 DEBUG ] 05:12:55 DEBUG } 05:12:55 DEBUG } 05:12:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499975" 05:12:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499975" 05:12:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499975" 05:12:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499975" 05:12:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499975" 05:12:55 INFO [http_cmd]: http status code OK 05:12:55 INFO [http_cmd]: http status code OK 05:12:55 INFO [http_cmd]: http status code OK 05:12:55 DEBUG --- status code --- 05:12:55 DEBUG --- status code --- 05:12:55 DEBUG --- status code --- 05:12:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:12:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:12:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:12:55 DEBUG --- http response --- 05:12:55 DEBUG --- http response --- 05:12:55 DEBUG --- http response --- 05:12:55 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:12:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:12:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:12:55 DEBUG ----- output ----- 05:12:55 DEBUG ----- output ----- 05:12:55 DEBUG { 05:12:55 DEBUG { 05:12:55 DEBUG "status": "success", 05:12:55 DEBUG "status": "success", 05:12:55 DEBUG "data": { 05:12:55 DEBUG "data": { 05:12:55 DEBUG "resultType": "vector", 05:12:55 DEBUG "resultType": "vector", 05:12:55 DEBUG "result": [ 05:12:55 DEBUG "result": [ 05:12:55 DEBUG { 05:12:55 DEBUG { 05:12:55 DEBUG "metric": { 05:12:55 DEBUG "metric": { 05:12:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:12:55 DEBUG "pod": "ds-cts-0" 05:12:55 DEBUG }, 05:12:55 DEBUG }, 05:12:55 DEBUG "value": [ 05:12:55 DEBUG "value": [ 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG [...] 05:12:55 DEBUG [...] 05:12:55 INFO [http_cmd]: http status code OK 05:12:55 DEBUG 05:12:55 INFO [http_cmd]: http status code OK 05:12:55 DEBUG 05:12:55 DEBUG --- status code --- 05:12:55 DEBUG 05:12:55 DEBUG --- status code --- 05:12:55 DEBUG 05:12:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:12:55 DEBUG 05:12:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:12:55 DEBUG 05:12:55 DEBUG --- http response --- 05:12:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:12:55 DEBUG --- http response --- 05:12:55 DEBUG "pod": "ds-cts-2" 05:12:55 DEBUG }, 05:12:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:12:55 DEBUG }, 05:12:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:12:55 DEBUG "value": [ 05:12:55 DEBUG ----- output ----- 05:12:55 DEBUG "value": [ 05:12:55 DEBUG ----- output ----- 05:12:55 DEBUG 1713499975, 05:12:55 DEBUG { 05:12:55 DEBUG 1713499975, 05:12:55 DEBUG { 05:12:55 DEBUG "2003.5" 05:12:55 DEBUG "status": "success", 05:12:55 DEBUG "0.000017660000000000004" 05:12:55 DEBUG "status": "success", 05:12:55 DEBUG ] 05:12:55 DEBUG "data": { 05:12:55 DEBUG ] 05:12:55 DEBUG "data": { 05:12:55 DEBUG } 05:12:55 DEBUG "resultType": "vector", 05:12:55 DEBUG } 05:12:55 DEBUG "resultType": "vector", 05:12:55 DEBUG ] 05:12:55 DEBUG "result": [ 05:12:55 DEBUG ] 05:12:55 DEBUG "result": [ 05:12:55 DEBUG } 05:12:55 DEBUG { 05:12:55 DEBUG } 05:12:55 DEBUG { 05:12:55 DEBUG } 05:12:55 DEBUG "metric": { 05:12:55 DEBUG } 05:12:55 DEBUG "metric": { 05:12:55 DEBUG "pod": "ds-idrepo-0" 05:12:55 DEBUG "pod": "ds-cts-0" 05:12:55 DEBUG }, 05:12:55 DEBUG }, 05:12:55 DEBUG "value": [ 05:12:55 DEBUG "value": [ 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG [...] 05:12:55 DEBUG [...] 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG "pod": "ds-idrepo-2" 05:12:55 DEBUG "pod": "ds-cts-2" 05:12:55 DEBUG }, 05:12:55 DEBUG }, 05:12:55 DEBUG "value": [ 05:12:55 DEBUG "value": [ 05:12:55 DEBUG 1713499975, 05:12:55 DEBUG 1713499975, 05:12:55 DEBUG "0" 05:12:55 DEBUG "3.6" 05:12:55 DEBUG ] 05:12:55 DEBUG ] 05:12:55 DEBUG } 05:12:55 DEBUG } 05:12:55 DEBUG ] 05:12:55 DEBUG ] 05:12:55 DEBUG } 05:12:55 DEBUG } 05:12:55 DEBUG } 05:12:55 DEBUG } 05:12:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499975" 05:12:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499975" 05:12:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499975" 05:12:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499975" 05:12:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499975" 05:12:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499975" 05:12:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499975" 05:12:55 INFO [http_cmd]: http status code OK 05:12:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499975" 05:12:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499975" 05:12:55 DEBUG --- status code --- 05:12:55 INFO [http_cmd]: http status code OK 05:12:55 INFO [http_cmd]: http status code OK 05:12:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:12:55 INFO [http_cmd]: http status code OK 05:12:55 DEBUG --- status code --- 05:12:55 DEBUG --- status code --- 05:12:55 DEBUG --- http response --- 05:12:55 DEBUG --- status code --- 05:12:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:12:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:12:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:12:55 INFO [http_cmd]: http status code OK 05:12:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:12:55 INFO [http_cmd]: http status code OK 05:12:55 DEBUG --- http response --- 05:12:55 DEBUG --- http response --- 05:12:55 DEBUG ----- output ----- 05:12:55 DEBUG --- status code --- 05:12:55 DEBUG --- http response --- 05:12:55 DEBUG --- status code --- 05:12:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:12:55 DEBUG { 05:12:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:12:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:12:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:12:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:12:55 DEBUG ----- output ----- 05:12:55 DEBUG "status": "success", 05:12:55 DEBUG ----- output ----- 05:12:55 DEBUG --- http response --- 05:12:55 DEBUG --- http response --- 05:12:55 DEBUG ----- output ----- 05:12:55 DEBUG { 05:12:55 DEBUG "data": { 05:12:55 DEBUG { 05:12:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:12:55 DEBUG { 05:12:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:12:55 DEBUG "status": "success", 05:12:55 DEBUG "resultType": "vector", 05:12:55 DEBUG "status": "success", 05:12:55 DEBUG ----- output ----- 05:12:55 DEBUG "status": "success", 05:12:55 INFO [http_cmd]: http status code OK 05:12:55 DEBUG ----- output ----- 05:12:55 DEBUG "data": { 05:12:55 DEBUG "result": [ 05:12:55 DEBUG "data": { 05:12:55 DEBUG { 05:12:55 DEBUG "data": { 05:12:55 DEBUG --- status code --- 05:12:55 DEBUG { 05:12:55 DEBUG "resultType": "vector", 05:12:55 DEBUG { 05:12:55 DEBUG "resultType": "vector", 05:12:55 DEBUG "status": "success", 05:12:55 DEBUG "resultType": "vector", 05:12:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:12:55 DEBUG "status": "success", 05:12:55 DEBUG "result": [ 05:12:55 DEBUG "metric": { 05:12:55 DEBUG "result": [ 05:12:55 DEBUG "data": { 05:12:55 DEBUG "result": [ 05:12:55 DEBUG --- http response --- 05:12:55 DEBUG "data": { 05:12:55 DEBUG { 05:12:55 DEBUG "instance": "10.142.0.107:9100" 05:12:55 DEBUG { 05:12:55 DEBUG "resultType": "vector", 05:12:55 DEBUG { 05:12:55 DEBUG "resultType": "vector", 05:12:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:12:55 DEBUG "metric": { 05:12:55 DEBUG }, 05:12:55 DEBUG "metric": { 05:12:55 DEBUG "result": [ 05:12:55 DEBUG "metric": { 05:12:55 INFO [http_cmd]: http status code OK 05:12:55 INFO [http_cmd]: http status code OK 05:12:55 DEBUG "result": [ 05:12:55 DEBUG ----- output ----- 05:12:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:12:55 DEBUG "value": [ 05:12:55 DEBUG "pod": "ds-idrepo-0" 05:12:55 DEBUG { 05:12:55 DEBUG "pod": "ds-idrepo-0" 05:12:55 DEBUG --- status code --- 05:12:55 DEBUG --- status code --- 05:12:55 DEBUG { 05:12:55 DEBUG { 05:12:55 DEBUG }, 05:12:55 DEBUG 05:12:55 DEBUG }, 05:12:55 DEBUG "metric": { 05:12:55 DEBUG }, 05:12:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:12:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:12:55 DEBUG "metric": { 05:12:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499975" 05:12:55 DEBUG "status": "success", 05:12:55 DEBUG "value": [ 05:12:55 DEBUG 05:12:55 DEBUG "value": [ 05:12:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:12:55 DEBUG "value": [ 05:12:55 DEBUG --- http response --- 05:12:55 DEBUG --- http response --- 05:12:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:12:55 DEBUG "data": { 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG }, 05:12:55 DEBUG 05:12:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:12:55 DEBUG }, 05:12:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:12:55 DEBUG "resultType": "vector", 05:12:55 DEBUG 05:12:55 DEBUG [...] 05:12:55 DEBUG 05:12:55 DEBUG "value": [ 05:12:55 DEBUG 05:12:55 DEBUG ----- output ----- 05:12:55 DEBUG "value": [ 05:12:55 DEBUG ----- output ----- 05:12:55 DEBUG "result": [ 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG { 05:12:55 DEBUG 05:12:55 DEBUG { 05:12:55 DEBUG { 05:12:55 DEBUG [...] 05:12:55 DEBUG 05:12:55 DEBUG [...] 05:12:55 DEBUG 05:12:55 DEBUG [...] 05:12:55 DEBUG "status": "success", 05:12:55 DEBUG 05:12:55 DEBUG "status": "success", 05:12:55 DEBUG "metric": { 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG "data": { 05:12:55 DEBUG 05:12:55 DEBUG "data": { 05:12:55 DEBUG "pod": "ds-cts-0" 05:12:55 DEBUG 05:12:55 DEBUG "instance": "10.142.0.43:9100" 05:12:55 DEBUG 05:12:55 DEBUG [...] 05:12:55 DEBUG 05:12:55 DEBUG "resultType": "vector", 05:12:55 DEBUG [...] 05:12:55 DEBUG "resultType": "vector", 05:12:55 DEBUG }, 05:12:55 DEBUG 05:12:55 DEBUG }, 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG "result": [ 05:12:55 DEBUG 05:12:55 DEBUG "result": [ 05:12:55 DEBUG "value": [ 05:12:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:12:55 DEBUG "value": [ 05:12:55 DEBUG "pod": "ds-idrepo-2" 05:12:55 DEBUG 05:12:55 DEBUG "pod": "ds-idrepo-2" 05:12:55 DEBUG { 05:12:55 DEBUG 05:12:55 DEBUG { 05:12:55 DEBUG 05:12:55 DEBUG }, 05:12:55 DEBUG 1713499975, 05:12:55 DEBUG }, 05:12:55 DEBUG 05:12:55 DEBUG }, 05:12:55 DEBUG "metric": { 05:12:55 DEBUG 05:12:55 DEBUG "metric": { 05:12:55 DEBUG 05:12:55 DEBUG "value": [ 05:12:55 DEBUG "47856093.03333333" 05:12:55 DEBUG "value": [ 05:12:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:12:55 DEBUG "value": [ 05:12:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:12:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:12:55 DEBUG "instance": "10.142.0.107:9100" 05:12:55 DEBUG 05:12:55 DEBUG 1713499975, 05:12:55 DEBUG ] 05:12:55 DEBUG 1713499975, 05:12:55 DEBUG }, 05:12:55 DEBUG 1713499975, 05:12:55 DEBUG }, 05:12:55 DEBUG }, 05:12:55 DEBUG }, 05:12:55 DEBUG [...] 05:12:55 DEBUG "412.75759560638886" 05:12:55 DEBUG } 05:12:55 DEBUG "0" 05:12:55 DEBUG "value": [ 05:12:55 DEBUG "0" 05:12:55 DEBUG "value": [ 05:12:55 DEBUG "value": [ 05:12:55 DEBUG "value": [ 05:12:55 DEBUG 05:12:55 DEBUG ] 05:12:55 DEBUG ] 05:12:55 DEBUG ] 05:12:55 DEBUG 1713499975, 05:12:55 DEBUG ] 05:12:55 DEBUG 05:12:55 DEBUG 1713499975, 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG } 05:12:55 DEBUG } 05:12:55 DEBUG } 05:12:55 DEBUG "0" 05:12:55 DEBUG } 05:12:55 DEBUG 05:12:55 DEBUG "2003.5" 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG ] 05:12:55 DEBUG } 05:12:55 DEBUG ] 05:12:55 DEBUG ] 05:12:55 DEBUG ] 05:12:55 DEBUG 05:12:55 DEBUG ] 05:12:55 DEBUG 05:12:55 DEBUG "pod": "ds-cts-2" 05:12:55 DEBUG } 05:12:55 DEBUG } 05:12:55 DEBUG } 05:12:55 DEBUG } 05:12:55 DEBUG [...] 05:12:55 DEBUG } 05:12:55 DEBUG [...] 05:12:55 DEBUG }, 05:12:55 DEBUG } 05:12:55 DEBUG } 05:12:55 DEBUG ] 05:12:55 DEBUG } 05:12:55 DEBUG 05:12:55 DEBUG ] 05:12:55 DEBUG 05:12:55 DEBUG "value": [ 05:12:55 DEBUG } 05:12:55 DEBUG 05:12:55 DEBUG } 05:12:55 INFO [http_cmd]: http status code OK 05:12:55 DEBUG 05:12:55 DEBUG 1713499975, 05:12:55 DEBUG } 05:12:55 DEBUG 05:12:55 DEBUG } 05:12:55 DEBUG --- status code --- 05:12:55 DEBUG 05:12:55 DEBUG "0" 05:12:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:12:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:12:55 DEBUG "instance": "10.142.0.43:9100" 05:12:55 DEBUG ] 05:12:55 DEBUG }, 05:12:55 DEBUG --- http response --- 05:12:55 DEBUG }, 05:12:55 DEBUG } 05:12:55 DEBUG "value": [ 05:12:55 DEBUG "value": [ 05:12:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:12:55 DEBUG ] 05:12:55 DEBUG 1713499975, 05:12:55 DEBUG 1713499975, 05:12:55 DEBUG ----- output ----- 05:12:55 DEBUG } 05:12:55 DEBUG "2003.4333333333334" 05:12:55 DEBUG "451509707776" 05:12:55 DEBUG { 05:12:55 DEBUG } 05:12:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713499975" 05:12:55 DEBUG ] 05:12:55 DEBUG ] 05:12:55 DEBUG "status": "success", 05:12:55 DEBUG } 05:12:55 DEBUG } 05:12:55 DEBUG "data": { 05:12:55 DEBUG ] 05:12:55 DEBUG ] 05:12:55 DEBUG "resultType": "vector", 05:12:55 DEBUG } 05:12:55 DEBUG } 05:12:55 DEBUG "result": [ 05:12:55 DEBUG } 05:12:55 DEBUG } 05:12:55 DEBUG { 05:12:55 DEBUG "metric": { 05:12:55 DEBUG "instance": "10.142.0.107:9100" 05:12:55 DEBUG }, 05:12:55 DEBUG "value": [ 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG [...] 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG "instance": "10.142.0.43:9100" 05:12:55 DEBUG }, 05:12:55 DEBUG "value": [ 05:12:55 DEBUG 1713499975, 05:12:55 DEBUG "47854213.7" 05:12:55 DEBUG ] 05:12:55 DEBUG } 05:12:55 DEBUG ] 05:12:55 DEBUG } 05:12:55 DEBUG } 05:12:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713499975" 05:12:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713499975" 05:12:55 INFO [http_cmd]: http status code OK 05:12:55 DEBUG --- status code --- 05:12:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:12:55 DEBUG --- http response --- 05:12:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:12:55 DEBUG ----- output ----- 05:12:55 DEBUG { 05:12:55 DEBUG "status": "success", 05:12:55 DEBUG "data": { 05:12:55 DEBUG "resultType": "vector", 05:12:55 DEBUG "result": [ 05:12:55 DEBUG { 05:12:55 DEBUG "metric": { 05:12:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:12:55 DEBUG }, 05:12:55 DEBUG "value": [ 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG [...] 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:12:55 DEBUG }, 05:12:55 DEBUG "value": [ 05:12:55 DEBUG 1713499975, 05:12:55 DEBUG "0" 05:12:55 DEBUG ] 05:12:55 DEBUG } 05:12:55 DEBUG ] 05:12:55 DEBUG } 05:12:55 DEBUG } 05:12:55 INFO [http_cmd]: http status code OK 05:12:55 DEBUG --- status code --- 05:12:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:12:55 DEBUG --- http response --- 05:12:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:12:55 DEBUG ----- output ----- 05:12:55 DEBUG { 05:12:55 DEBUG "status": "success", 05:12:55 DEBUG "data": { 05:12:55 DEBUG "resultType": "vector", 05:12:55 DEBUG "result": [ 05:12:55 DEBUG { 05:12:55 DEBUG "metric": { 05:12:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:12:55 DEBUG }, 05:12:55 DEBUG "value": [ 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG [...] 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:12:55 DEBUG }, 05:12:55 DEBUG "value": [ 05:12:55 DEBUG 1713499975, 05:12:55 DEBUG "2178654208" 05:12:55 DEBUG ] 05:12:55 DEBUG } 05:12:55 DEBUG ] 05:12:55 DEBUG } 05:12:55 DEBUG } 05:12:55 INFO [http_cmd]: http status code OK 05:12:55 DEBUG --- status code --- 05:12:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:12:55 DEBUG --- http response --- 05:12:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:12:55 DEBUG ----- output ----- 05:12:55 DEBUG { 05:12:55 DEBUG "status": "success", 05:12:55 DEBUG "data": { 05:12:55 DEBUG "resultType": "vector", 05:12:55 DEBUG "result": [ 05:12:55 DEBUG { 05:12:55 DEBUG "metric": { 05:12:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:12:55 DEBUG }, 05:12:55 DEBUG "value": [ 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG [...] 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG 05:12:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:12:55 DEBUG }, 05:12:55 DEBUG "value": [ 05:12:55 DEBUG 1713499975, 05:12:55 DEBUG "55.171499999999995" 05:12:55 DEBUG ] 05:12:55 DEBUG } 05:12:55 DEBUG ] 05:12:55 DEBUG } 05:12:55 DEBUG } 05:12:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713499975" 05:12:56 INFO [http_cmd]: http status code OK 05:12:56 DEBUG --- status code --- 05:12:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:12:56 DEBUG --- http response --- 05:12:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:12:56 DEBUG ----- output ----- 05:12:56 DEBUG { 05:12:56 DEBUG "status": "success", 05:12:56 DEBUG "data": { 05:12:56 DEBUG "resultType": "vector", 05:12:56 DEBUG "result": [ 05:12:56 DEBUG { 05:12:56 DEBUG "metric": { 05:12:56 DEBUG "instance": "10.142.0.107:9100" 05:12:56 DEBUG }, 05:12:56 DEBUG "value": [ 05:12:56 DEBUG 05:12:56 DEBUG 05:12:56 DEBUG 05:12:56 DEBUG [...] 05:12:56 DEBUG 05:12:56 DEBUG 05:12:56 DEBUG 05:12:56 DEBUG "instance": "10.142.0.43:9100" 05:12:56 DEBUG }, 05:12:56 DEBUG "value": [ 05:12:56 DEBUG 1713499975, 05:12:56 DEBUG "0.040666666666668044" 05:12:56 DEBUG ] 05:12:56 DEBUG } 05:12:56 DEBUG ] 05:12:56 DEBUG } 05:12:56 DEBUG } 05:12:56 INFO 05:12:56 INFO [loop_until]: kubectl --namespace=xlou top pods 05:12:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:12:56 INFO [loop_until]: OK (rc = 0) 05:12:56 DEBUG --- stdout --- 05:12:56 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 9930m 9122Mi am-6fd554f8f7-hvvkw 9494m 8877Mi am-6fd554f8f7-pmj4b 10351m 8975Mi ds-cts-0 7684m 12290Mi ds-cts-1 7520m 12214Mi ds-cts-2 7702m 12251Mi ds-idrepo-0 4627m 3256Mi ds-idrepo-1 4665m 3207Mi ds-idrepo-2 4601m 3066Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1172Mi idm-b85f9d978-wnhxb 3m 1215Mi lodemon-6b89d774d5-rf7sk 11m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 6303m 1987Mi 05:12:56 DEBUG --- stderr --- 05:12:56 DEBUG 05:12:57 INFO 05:12:57 INFO [loop_until]: kubectl --namespace=xlou top node 05:12:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:12:57 INFO [loop_until]: OK (rc = 0) 05:12:57 DEBUG --- stdout --- 05:12:57 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 71m 0% 6260Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 4668m 15% 6699Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9466m 31% 11587Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 3250Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 7838m 26% 14833Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 152m 0% 5173Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 9979m 33% 11901Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 10283m 34% 2734Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 11220m 37% 11182Mi 10% gke-xlou-cdm-ds-0f601751-j488 4830m 16% 6164Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 7488m 25% 16238Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 5144m 17% 6878Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 7833m 26% 14227Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 6680m 42% 5723Mi 9% 05:12:57 DEBUG --- stderr --- 05:12:57 DEBUG 05:13:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500035" 05:13:55 INFO [http_cmd]: http status code OK 05:13:55 DEBUG --- status code --- 05:13:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:13:55 DEBUG --- http response --- 05:13:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:13:55 DEBUG ----- output ----- 05:13:55 DEBUG { 05:13:55 DEBUG "status": "success", 05:13:55 DEBUG "data": { 05:13:55 DEBUG "resultType": "vector", 05:13:55 DEBUG "result": [ 05:13:55 DEBUG { 05:13:55 DEBUG "metric": { 05:13:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:13:55 DEBUG }, 05:13:55 DEBUG "value": [ 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG [...] 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:13:55 DEBUG }, 05:13:55 DEBUG "value": [ 05:13:55 DEBUG 1713500035, 05:13:55 DEBUG "3463155" 05:13:55 DEBUG ] 05:13:55 DEBUG } 05:13:55 DEBUG ] 05:13:55 DEBUG } 05:13:55 DEBUG } 05:13:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713500035" 05:13:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500035" 05:13:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500035" 05:13:55 INFO [http_cmd]: http status code OK 05:13:55 INFO [http_cmd]: http status code OK 05:13:55 DEBUG --- status code --- 05:13:55 INFO [http_cmd]: http status code OK 05:13:55 DEBUG --- status code --- 05:13:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:13:55 DEBUG --- status code --- 05:13:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:13:55 DEBUG --- http response --- 05:13:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:13:55 DEBUG --- http response --- 05:13:55 DEBUG --- http response --- 05:13:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 05:13:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:13:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 05:13:55 DEBUG ----- output ----- 05:13:55 DEBUG ----- output ----- 05:13:55 DEBUG ----- output ----- 05:13:55 DEBUG { 05:13:55 DEBUG { 05:13:55 DEBUG { 05:13:55 DEBUG "status": "success", 05:13:55 DEBUG "status": "success", 05:13:55 DEBUG "status": "success", 05:13:55 DEBUG "data": { 05:13:55 DEBUG "data": { 05:13:55 DEBUG "data": { 05:13:55 DEBUG "resultType": "vector", 05:13:55 DEBUG "resultType": "vector", 05:13:55 DEBUG "resultType": "vector", 05:13:55 DEBUG "result": [ 05:13:55 DEBUG "result": [ 05:13:55 DEBUG "result": [ 05:13:55 DEBUG { 05:13:55 DEBUG { 05:13:55 DEBUG { 05:13:55 DEBUG "metric": { 05:13:55 DEBUG "metric": { 05:13:55 DEBUG "metric": { 05:13:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:13:55 DEBUG "pod": "ds-idrepo-0" 05:13:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:13:55 DEBUG }, 05:13:55 DEBUG }, 05:13:55 DEBUG }, 05:13:55 DEBUG "value": [ 05:13:55 DEBUG "value": [ 05:13:55 DEBUG "value": [ 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG [...] 05:13:55 DEBUG [...] 05:13:55 DEBUG [...] 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:13:55 DEBUG "pod": "ds-cts-2" 05:13:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:13:55 DEBUG }, 05:13:55 DEBUG }, 05:13:55 DEBUG }, 05:13:55 DEBUG "value": [ 05:13:55 DEBUG "value": [ 05:13:55 DEBUG "value": [ 05:13:55 DEBUG 1713500035, 05:13:55 DEBUG 1713500035, 05:13:55 DEBUG 1713500035, 05:13:55 DEBUG "16610.508259302864" 05:13:55 DEBUG "0" 05:13:55 DEBUG "4.949649957971971" 05:13:55 DEBUG ] 05:13:55 DEBUG ] 05:13:55 DEBUG ] 05:13:55 DEBUG } 05:13:55 DEBUG } 05:13:55 DEBUG } 05:13:55 DEBUG ] 05:13:55 DEBUG ] 05:13:55 DEBUG ] 05:13:55 DEBUG } 05:13:55 DEBUG } 05:13:55 DEBUG } 05:13:55 DEBUG } 05:13:55 DEBUG } 05:13:55 DEBUG } 05:13:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713500035" 05:13:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713500035" 05:13:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500035" 05:13:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500035" 05:13:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500035" 05:13:55 INFO [http_cmd]: http status code OK 05:13:55 DEBUG --- status code --- 05:13:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:13:55 DEBUG --- http response --- 05:13:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:13:55 DEBUG ----- output ----- 05:13:55 DEBUG { 05:13:55 DEBUG "status": "success", 05:13:55 DEBUG "data": { 05:13:55 DEBUG "resultType": "vector", 05:13:55 DEBUG "result": [ 05:13:55 DEBUG { 05:13:55 DEBUG "metric": { 05:13:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:13:55 DEBUG }, 05:13:55 DEBUG "value": [ 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG [...] 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:13:55 DEBUG }, 05:13:55 DEBUG "value": [ 05:13:55 DEBUG 1713500035, 05:13:55 DEBUG "6.526859481321285" 05:13:55 DEBUG ] 05:13:55 DEBUG } 05:13:55 DEBUG ] 05:13:55 DEBUG } 05:13:55 DEBUG } 05:13:55 ERROR [http_cmd]: ERROR 05:13:55 INFO [http_cmd]: http status code OK 05:13:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500035" 05:13:55 INFO [http_cmd]: http status code OK 05:13:55 INFO [http_cmd]: http status code OK 05:13:55 ERROR --- status code --- 05:13:55 DEBUG --- status code --- 05:13:55 DEBUG --- status code --- 05:13:55 DEBUG --- status code --- 05:13:55 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:13:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:13:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:13:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:13:55 ERROR --- http response --- 05:13:55 DEBUG --- http response --- 05:13:55 DEBUG --- http response --- 05:13:55 DEBUG --- http response --- 05:13:55 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:13:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:13:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:13:55 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:13:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:13:55 DEBUG ----- output ----- 05:13:55 DEBUG ----- output ----- 05:13:55 DEBUG ----- output ----- 05:13:55 DEBUG { 05:13:55 DEBUG { 05:13:55 DEBUG { 05:13:55 DEBUG "status": "success", 05:13:55 DEBUG "status": "success", 05:13:55 DEBUG "status": "success", 05:13:55 DEBUG "data": { 05:13:55 DEBUG "data": { 05:13:55 DEBUG "data": { 05:13:55 DEBUG "resultType": "vector", 05:13:55 DEBUG "resultType": "vector", 05:13:55 DEBUG "resultType": "vector", 05:13:55 DEBUG "result": [ 05:13:55 DEBUG "result": [ 05:13:55 DEBUG "result": [ 05:13:55 DEBUG { 05:13:55 DEBUG { 05:13:55 DEBUG { 05:13:55 DEBUG "metric": { 05:13:55 DEBUG "metric": { 05:13:55 DEBUG "metric": { 05:13:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:13:55 DEBUG "pod": "ds-idrepo-0" 05:13:55 DEBUG "pod": "ds-cts-0" 05:13:55 DEBUG }, 05:13:55 DEBUG }, 05:13:55 DEBUG }, 05:13:55 DEBUG "value": [ 05:13:55 DEBUG "value": [ 05:13:55 DEBUG "value": [ 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG [...] 05:13:55 DEBUG [...] 05:13:55 DEBUG [...] 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:13:55 DEBUG "pod": "ds-idrepo-2" 05:13:55 DEBUG "pod": "ds-cts-2" 05:13:55 DEBUG }, 05:13:55 DEBUG }, 05:13:55 DEBUG }, 05:13:55 DEBUG "value": [ 05:13:55 DEBUG "value": [ 05:13:55 DEBUG "value": [ 05:13:55 DEBUG 1713500035, 05:13:55 DEBUG 1713500035, 05:13:55 DEBUG 1713500035, 05:13:55 DEBUG "0" 05:13:55 DEBUG "0" 05:13:55 DEBUG "0" 05:13:55 DEBUG ] 05:13:55 DEBUG ] 05:13:55 DEBUG ] 05:13:55 DEBUG } 05:13:55 DEBUG } 05:13:55 DEBUG } 05:13:55 DEBUG ] 05:13:55 DEBUG ] 05:13:55 DEBUG ] 05:13:55 DEBUG } 05:13:55 DEBUG } 05:13:55 DEBUG } 05:13:55 DEBUG } 05:13:55 DEBUG } 05:13:55 DEBUG } 05:13:55 INFO [http_cmd]: http status code OK 05:13:55 DEBUG --- status code --- 05:13:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:13:55 DEBUG --- http response --- 05:13:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:13:55 DEBUG ----- output ----- 05:13:55 DEBUG { 05:13:55 DEBUG "status": "success", 05:13:55 DEBUG "data": { 05:13:55 DEBUG "resultType": "vector", 05:13:55 DEBUG "result": [ 05:13:55 DEBUG { 05:13:55 DEBUG "metric": { 05:13:55 DEBUG "instance": "10.142.0.107:9100" 05:13:55 DEBUG }, 05:13:55 DEBUG "value": [ 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG [...] 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG "instance": "10.142.0.43:9100" 05:13:55 DEBUG }, 05:13:55 DEBUG "value": [ 05:13:55 DEBUG 1713500035, 05:13:55 DEBUG "0.06539999999999963" 05:13:55 DEBUG ] 05:13:55 DEBUG } 05:13:55 DEBUG ] 05:13:55 DEBUG } 05:13:55 DEBUG } 05:13:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500035" 05:13:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500035" 05:13:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500035" 05:13:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500035" 05:13:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500035" 05:13:55 INFO [http_cmd]: http status code OK 05:13:55 DEBUG --- status code --- 05:13:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:13:55 DEBUG --- http response --- 05:13:55 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:13:55 INFO [http_cmd]: http status code OK 05:13:55 DEBUG --- status code --- 05:13:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:13:55 DEBUG --- http response --- 05:13:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:13:55 DEBUG ----- output ----- 05:13:55 DEBUG { 05:13:55 DEBUG "status": "success", 05:13:55 DEBUG "data": { 05:13:55 DEBUG "resultType": "vector", 05:13:55 DEBUG "result": [ 05:13:55 DEBUG { 05:13:55 DEBUG "metric": { 05:13:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:13:55 DEBUG }, 05:13:55 DEBUG "value": [ 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG [...] 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:13:55 DEBUG }, 05:13:55 DEBUG "value": [ 05:13:55 DEBUG 1713500035, 05:13:55 DEBUG "2003.6666666666667" 05:13:55 DEBUG ] 05:13:55 DEBUG } 05:13:55 DEBUG ] 05:13:55 DEBUG } 05:13:55 DEBUG } 05:13:55 INFO [http_cmd]: http status code OK 05:13:55 DEBUG --- status code --- 05:13:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:13:55 DEBUG --- http response --- 05:13:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:13:55 DEBUG ----- output ----- 05:13:55 DEBUG { 05:13:55 DEBUG "status": "success", 05:13:55 DEBUG "data": { 05:13:55 DEBUG "resultType": "vector", 05:13:55 DEBUG "result": [ 05:13:55 DEBUG { 05:13:55 DEBUG "metric": { 05:13:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:13:55 DEBUG }, 05:13:55 DEBUG "value": [ 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG [...] 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:13:55 DEBUG }, 05:13:55 DEBUG "value": [ 05:13:55 DEBUG 1713500035, 05:13:55 DEBUG "2003.5" 05:13:55 DEBUG ] 05:13:55 DEBUG } 05:13:55 DEBUG ] 05:13:55 DEBUG } 05:13:55 DEBUG } 05:13:55 INFO [http_cmd]: http status code OK 05:13:55 DEBUG --- status code --- 05:13:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:13:55 DEBUG --- http response --- 05:13:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:13:55 DEBUG ----- output ----- 05:13:55 DEBUG { 05:13:55 DEBUG "status": "success", 05:13:55 DEBUG "data": { 05:13:55 DEBUG "resultType": "vector", 05:13:55 DEBUG "result": [ 05:13:55 DEBUG { 05:13:55 DEBUG "metric": { 05:13:55 DEBUG "pod": "ds-cts-0" 05:13:55 DEBUG }, 05:13:55 DEBUG "value": [ 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG [...] 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG "pod": "ds-cts-2" 05:13:55 DEBUG }, 05:13:55 DEBUG "value": [ 05:13:55 DEBUG 1713500035, 05:13:55 DEBUG "3.6" 05:13:55 DEBUG ] 05:13:55 DEBUG } 05:13:55 DEBUG ] 05:13:55 DEBUG } 05:13:55 DEBUG } 05:13:55 INFO [http_cmd]: http status code OK 05:13:55 DEBUG --- status code --- 05:13:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:13:55 DEBUG --- http response --- 05:13:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:13:55 DEBUG ----- output ----- 05:13:55 DEBUG { 05:13:55 DEBUG "status": "success", 05:13:55 DEBUG "data": { 05:13:55 DEBUG "resultType": "vector", 05:13:55 DEBUG "result": [ 05:13:55 DEBUG { 05:13:55 DEBUG "metric": { 05:13:55 DEBUG "instance": "10.142.0.107:9100" 05:13:55 DEBUG }, 05:13:55 DEBUG "value": [ 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG [...] 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG "instance": "10.142.0.43:9100" 05:13:55 DEBUG }, 05:13:55 DEBUG "value": [ 05:13:55 DEBUG 1713500035, 05:13:55 DEBUG "47748837.83333333" 05:13:55 DEBUG ] 05:13:55 DEBUG } 05:13:55 DEBUG ] 05:13:55 DEBUG } 05:13:55 DEBUG } 05:13:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500035" 05:13:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713500035" 05:13:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500035" 05:13:55 INFO [http_cmd]: http status code OK 05:13:55 DEBUG --- status code --- 05:13:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:13:55 DEBUG --- http response --- 05:13:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 05:13:55 DEBUG ----- output ----- 05:13:55 DEBUG { 05:13:55 DEBUG "status": "success", 05:13:55 DEBUG "data": { 05:13:55 DEBUG "resultType": "vector", 05:13:55 DEBUG "result": [ 05:13:55 DEBUG { 05:13:55 DEBUG "metric": { 05:13:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:13:55 DEBUG }, 05:13:55 DEBUG "value": [ 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG [...] 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:13:55 DEBUG }, 05:13:55 DEBUG "value": [ 05:13:55 DEBUG 1713500035, 05:13:55 DEBUG "412.6834350137358" 05:13:55 DEBUG ] 05:13:55 DEBUG } 05:13:55 DEBUG ] 05:13:55 DEBUG } 05:13:55 DEBUG } 05:13:55 INFO [http_cmd]: http status code OK 05:13:55 DEBUG --- status code --- 05:13:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:13:55 DEBUG --- http response --- 05:13:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:13:55 DEBUG ----- output ----- 05:13:55 DEBUG { 05:13:55 DEBUG "status": "success", 05:13:55 DEBUG "data": { 05:13:55 DEBUG "resultType": "vector", 05:13:55 DEBUG "result": [ 05:13:55 DEBUG { 05:13:55 DEBUG "metric": { 05:13:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:13:55 DEBUG }, 05:13:55 DEBUG "value": [ 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG [...] 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:13:55 DEBUG }, 05:13:55 DEBUG "value": [ 05:13:55 DEBUG 1713500035, 05:13:55 DEBUG "2181697536" 05:13:55 DEBUG ] 05:13:55 DEBUG } 05:13:55 DEBUG ] 05:13:55 DEBUG } 05:13:55 DEBUG } 05:13:55 INFO [http_cmd]: http status code OK 05:13:55 DEBUG --- status code --- 05:13:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:13:55 DEBUG --- http response --- 05:13:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:13:55 DEBUG ----- output ----- 05:13:55 DEBUG { 05:13:55 DEBUG "status": "success", 05:13:55 DEBUG "data": { 05:13:55 DEBUG "resultType": "vector", 05:13:55 DEBUG "result": [ 05:13:55 DEBUG { 05:13:55 DEBUG "metric": { 05:13:55 DEBUG "pod": "ds-cts-0" 05:13:55 DEBUG }, 05:13:55 DEBUG "value": [ 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG [...] 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG "pod": "ds-cts-2" 05:13:55 DEBUG }, 05:13:55 DEBUG "value": [ 05:13:55 DEBUG 1713500035, 05:13:55 DEBUG "0" 05:13:55 DEBUG ] 05:13:55 DEBUG } 05:13:55 DEBUG ] 05:13:55 DEBUG } 05:13:55 DEBUG } 05:13:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500035" 05:13:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500035" 05:13:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500035" 05:13:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500035" 05:13:55 INFO [http_cmd]: http status code OK 05:13:55 DEBUG --- status code --- 05:13:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:13:55 DEBUG --- http response --- 05:13:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:13:55 DEBUG ----- output ----- 05:13:55 DEBUG { 05:13:55 DEBUG "status": "success", 05:13:55 DEBUG "data": { 05:13:55 DEBUG "resultType": "vector", 05:13:55 DEBUG "result": [ 05:13:55 DEBUG { 05:13:55 DEBUG "metric": { 05:13:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:13:55 DEBUG }, 05:13:55 DEBUG "value": [ 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG [...] 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:13:55 DEBUG }, 05:13:55 DEBUG "value": [ 05:13:55 DEBUG 1713500035, 05:13:55 DEBUG "0" 05:13:55 DEBUG ] 05:13:55 DEBUG } 05:13:55 DEBUG ] 05:13:55 DEBUG } 05:13:55 DEBUG } 05:13:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500035" 05:13:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500035" 05:13:55 INFO [http_cmd]: http status code OK 05:13:55 DEBUG --- status code --- 05:13:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:13:55 DEBUG --- http response --- 05:13:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:13:55 DEBUG ----- output ----- 05:13:55 DEBUG { 05:13:55 DEBUG "status": "success", 05:13:55 DEBUG "data": { 05:13:55 DEBUG "resultType": "vector", 05:13:55 DEBUG "result": [ 05:13:55 DEBUG { 05:13:55 DEBUG "metric": { 05:13:55 DEBUG "instance": "10.142.0.107:9100" 05:13:55 DEBUG }, 05:13:55 DEBUG "value": [ 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG [...] 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG "instance": "10.142.0.43:9100" 05:13:55 DEBUG }, 05:13:55 DEBUG "value": [ 05:13:55 DEBUG 1713500035, 05:13:55 DEBUG "47753088.266666666" 05:13:55 DEBUG ] 05:13:55 DEBUG } 05:13:55 DEBUG ] 05:13:55 DEBUG } 05:13:55 DEBUG } 05:13:55 INFO [http_cmd]: http status code OK 05:13:55 DEBUG --- status code --- 05:13:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:13:55 DEBUG --- http response --- 05:13:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500035" 05:13:55 INFO [http_cmd]: http status code OK 05:13:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:13:55 DEBUG --- status code --- 05:13:55 DEBUG ----- output ----- 05:13:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:13:55 DEBUG { 05:13:55 DEBUG --- http response --- 05:13:55 DEBUG "status": "success", 05:13:55 DEBUG "data": { 05:13:55 DEBUG "resultType": "vector", 05:13:55 DEBUG "result": [ 05:13:55 DEBUG { 05:13:55 DEBUG "metric": { 05:13:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:13:55 DEBUG "pod": "ds-idrepo-0" 05:13:55 DEBUG ----- output ----- 05:13:55 DEBUG }, 05:13:55 DEBUG { 05:13:55 DEBUG "value": [ 05:13:55 DEBUG "status": "success", 05:13:55 DEBUG 05:13:55 DEBUG "data": { 05:13:55 DEBUG 05:13:55 DEBUG "resultType": "vector", 05:13:55 DEBUG 05:13:55 DEBUG "result": [ 05:13:55 DEBUG [...] 05:13:55 DEBUG { 05:13:55 DEBUG 05:13:55 DEBUG "metric": { 05:13:55 DEBUG 05:13:55 DEBUG "pod": "ds-idrepo-0" 05:13:55 DEBUG 05:13:55 DEBUG }, 05:13:55 DEBUG "pod": "ds-idrepo-2" 05:13:55 DEBUG "value": [ 05:13:55 DEBUG }, 05:13:55 DEBUG 05:13:55 DEBUG "value": [ 05:13:55 DEBUG 05:13:55 DEBUG 1713500035, 05:13:55 DEBUG 05:13:55 DEBUG "0" 05:13:55 DEBUG [...] 05:13:55 DEBUG ] 05:13:55 DEBUG 05:13:55 DEBUG } 05:13:55 DEBUG 05:13:55 INFO [http_cmd]: http status code OK 05:13:55 DEBUG ] 05:13:55 INFO [http_cmd]: http status code OK 05:13:55 DEBUG 05:13:55 DEBUG --- status code --- 05:13:55 DEBUG } 05:13:55 DEBUG --- status code --- 05:13:55 DEBUG "pod": "ds-idrepo-2" 05:13:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:13:55 DEBUG } 05:13:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:13:55 DEBUG }, 05:13:55 DEBUG --- http response --- 05:13:55 DEBUG --- http response --- 05:13:55 DEBUG "value": [ 05:13:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:13:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:13:55 DEBUG 1713500035, 05:13:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713500035" 05:13:55 DEBUG ----- output ----- 05:13:55 DEBUG ----- output ----- 05:13:55 DEBUG "0" 05:13:55 DEBUG { 05:13:55 DEBUG { 05:13:55 DEBUG ] 05:13:55 DEBUG "status": "success", 05:13:55 DEBUG "status": "success", 05:13:55 DEBUG } 05:13:55 DEBUG "data": { 05:13:55 DEBUG "data": { 05:13:55 DEBUG ] 05:13:55 DEBUG "resultType": "vector", 05:13:55 DEBUG "resultType": "vector", 05:13:55 DEBUG } 05:13:55 DEBUG "result": [ 05:13:55 DEBUG "result": [ 05:13:55 DEBUG } 05:13:55 DEBUG { 05:13:55 DEBUG { 05:13:55 DEBUG "metric": { 05:13:55 DEBUG "metric": { 05:13:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:13:55 DEBUG "instance": "10.142.0.107:9100" 05:13:55 DEBUG }, 05:13:55 DEBUG }, 05:13:55 DEBUG "value": [ 05:13:55 DEBUG "value": [ 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG [...] 05:13:55 DEBUG [...] 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:13:55 DEBUG "instance": "10.142.0.43:9100" 05:13:55 DEBUG }, 05:13:55 DEBUG }, 05:13:55 DEBUG "value": [ 05:13:55 DEBUG "value": [ 05:13:55 DEBUG 1713500035, 05:13:55 DEBUG 1713500035, 05:13:55 DEBUG "2003.6666666666667" 05:13:55 DEBUG "453939055616" 05:13:55 DEBUG ] 05:13:55 DEBUG ] 05:13:55 DEBUG } 05:13:55 DEBUG } 05:13:55 DEBUG ] 05:13:55 DEBUG ] 05:13:55 DEBUG } 05:13:55 DEBUG } 05:13:55 DEBUG } 05:13:55 DEBUG } 05:13:55 INFO [http_cmd]: http status code OK 05:13:55 DEBUG --- status code --- 05:13:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:13:55 DEBUG --- http response --- 05:13:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 05:13:55 DEBUG ----- output ----- 05:13:55 DEBUG { 05:13:55 DEBUG "status": "success", 05:13:55 DEBUG "data": { 05:13:55 DEBUG "resultType": "vector", 05:13:55 DEBUG "result": [ 05:13:55 DEBUG { 05:13:55 DEBUG "metric": { 05:13:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:13:55 DEBUG }, 05:13:55 DEBUG "value": [ 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG [...] 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:13:55 DEBUG }, 05:13:55 DEBUG "value": [ 05:13:55 DEBUG 1713500035, 05:13:55 DEBUG "0" 05:13:55 DEBUG ] 05:13:55 DEBUG } 05:13:55 DEBUG ] 05:13:55 DEBUG } 05:13:55 DEBUG } 05:13:55 INFO [http_cmd]: http status code OK 05:13:55 DEBUG --- status code --- 05:13:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:13:55 DEBUG --- http response --- 05:13:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:13:55 DEBUG ----- output ----- 05:13:55 DEBUG { 05:13:55 DEBUG "status": "success", 05:13:55 DEBUG "data": { 05:13:55 DEBUG "resultType": "vector", 05:13:55 DEBUG "result": [ 05:13:55 DEBUG { 05:13:55 DEBUG "metric": { 05:13:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:13:55 DEBUG }, 05:13:55 DEBUG "value": [ 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG [...] 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG 05:13:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:13:55 DEBUG }, 05:13:55 DEBUG "value": [ 05:13:55 DEBUG 1713500035, 05:13:55 DEBUG "55.172" 05:13:55 DEBUG ] 05:13:55 DEBUG } 05:13:55 DEBUG ] 05:13:55 DEBUG } 05:13:55 DEBUG } 05:13:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500036" 05:13:56 INFO [http_cmd]: http status code OK 05:13:56 DEBUG --- status code --- 05:13:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:13:56 DEBUG --- http response --- 05:13:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:13:56 DEBUG ----- output ----- 05:13:56 DEBUG { 05:13:56 DEBUG "status": "success", 05:13:56 DEBUG "data": { 05:13:56 DEBUG "resultType": "vector", 05:13:56 DEBUG "result": [ 05:13:56 DEBUG { 05:13:56 DEBUG "metric": { 05:13:56 DEBUG "instance": "10.142.0.107:9100" 05:13:56 DEBUG }, 05:13:56 DEBUG "value": [ 05:13:56 DEBUG 05:13:56 DEBUG 05:13:56 DEBUG 05:13:56 DEBUG [...] 05:13:56 DEBUG 05:13:56 DEBUG 05:13:56 DEBUG 05:13:56 DEBUG "instance": "10.142.0.43:9100" 05:13:56 DEBUG }, 05:13:56 DEBUG "value": [ 05:13:56 DEBUG 1713500036, 05:13:56 DEBUG "0.04300000000000257" 05:13:56 DEBUG ] 05:13:56 DEBUG } 05:13:56 DEBUG ] 05:13:56 DEBUG } 05:13:56 DEBUG } 05:13:57 INFO 05:13:57 INFO [loop_until]: kubectl --namespace=xlou top pods 05:13:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:13:57 INFO [loop_until]: OK (rc = 0) 05:13:57 DEBUG --- stdout --- 05:13:57 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 9929m 9122Mi am-6fd554f8f7-hvvkw 10050m 8873Mi am-6fd554f8f7-pmj4b 10742m 8975Mi ds-cts-0 7923m 12295Mi ds-cts-1 7594m 12220Mi ds-cts-2 7792m 12258Mi ds-idrepo-0 4666m 3257Mi ds-idrepo-1 4796m 3207Mi ds-idrepo-2 4577m 3065Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 4m 1215Mi lodemon-6b89d774d5-rf7sk 8m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 6666m 1988Mi 05:13:57 DEBUG --- stderr --- 05:13:57 DEBUG 05:13:57 INFO 05:13:57 INFO [loop_until]: kubectl --namespace=xlou top node 05:13:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:13:57 INFO [loop_until]: OK (rc = 0) 05:13:57 DEBUG --- stdout --- 05:13:57 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 71m 0% 6259Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 4660m 15% 6696Mi 5% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9445m 31% 11593Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 78m 0% 3245Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 7824m 26% 14852Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 154m 0% 5167Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 9946m 33% 11894Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 10332m 34% 2712Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 11192m 37% 11182Mi 10% gke-xlou-cdm-ds-0f601751-j488 4869m 16% 6168Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 7571m 25% 16253Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 5112m 17% 6882Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 7892m 26% 14247Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 6656m 41% 5721Mi 9% 05:13:57 DEBUG --- stderr --- 05:13:57 DEBUG 05:14:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500095" 05:14:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713500095" 05:14:55 INFO [http_cmd]: http status code OK 05:14:55 DEBUG --- status code --- 05:14:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:14:55 DEBUG --- http response --- 05:14:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:14:55 DEBUG ----- output ----- 05:14:55 DEBUG { 05:14:55 DEBUG "status": "success", 05:14:55 DEBUG "data": { 05:14:55 DEBUG "resultType": "vector", 05:14:55 DEBUG "result": [ 05:14:55 DEBUG { 05:14:55 DEBUG "metric": { 05:14:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:14:55 DEBUG }, 05:14:55 DEBUG "value": [ 05:14:55 DEBUG 05:14:55 DEBUG 05:14:55 DEBUG 05:14:55 DEBUG [...] 05:14:55 DEBUG 05:14:55 DEBUG 05:14:55 DEBUG 05:14:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:14:55 DEBUG }, 05:14:55 DEBUG "value": [ 05:14:55 DEBUG 1713500095, 05:14:55 DEBUG "3576050" 05:14:55 DEBUG ] 05:14:55 DEBUG } 05:14:55 DEBUG ] 05:14:55 DEBUG } 05:14:55 DEBUG } 05:14:55 INFO [http_cmd]: http status code OK 05:14:55 DEBUG --- status code --- 05:14:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:14:55 DEBUG --- http response --- 05:14:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:14:55 DEBUG ----- output ----- 05:14:55 DEBUG { 05:14:55 DEBUG "status": "success", 05:14:55 DEBUG "data": { 05:14:55 DEBUG "resultType": "vector", 05:14:55 DEBUG "result": [ 05:14:55 DEBUG { 05:14:55 DEBUG "metric": { 05:14:55 DEBUG "pod": "ds-idrepo-0" 05:14:55 DEBUG }, 05:14:55 DEBUG "value": [ 05:14:55 DEBUG 05:14:55 DEBUG 05:14:55 DEBUG 05:14:55 DEBUG [...] 05:14:55 DEBUG 05:14:55 DEBUG 05:14:55 DEBUG 05:14:55 DEBUG "pod": "ds-cts-2" 05:14:55 DEBUG }, 05:14:55 DEBUG "value": [ 05:14:55 DEBUG 1713500095, 05:14:55 DEBUG "0" 05:14:55 DEBUG ] 05:14:55 DEBUG } 05:14:55 DEBUG ] 05:14:55 DEBUG } 05:14:55 DEBUG } 05:14:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500095" 05:14:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500095" 05:14:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500095" 05:14:55 INFO [http_cmd]: http status code OK 05:14:55 DEBUG --- status code --- 05:14:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:14:55 DEBUG --- http response --- 05:14:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:14:55 DEBUG ----- output ----- 05:14:55 DEBUG { 05:14:55 DEBUG "status": "success", 05:14:55 DEBUG "data": { 05:14:55 DEBUG "resultType": "vector", 05:14:55 DEBUG "result": [ 05:14:55 DEBUG { 05:14:55 DEBUG "metric": { 05:14:55 DEBUG "pod": "ds-idrepo-0" 05:14:55 DEBUG }, 05:14:55 DEBUG "value": [ 05:14:55 DEBUG 05:14:55 DEBUG 05:14:55 DEBUG 05:14:55 DEBUG [...] 05:14:55 DEBUG 05:14:55 DEBUG 05:14:55 DEBUG 05:14:55 DEBUG "pod": "ds-idrepo-2" 05:14:55 DEBUG }, 05:14:55 DEBUG "value": [ 05:14:55 DEBUG 1713500095, 05:14:55 DEBUG "0" 05:14:55 DEBUG ] 05:14:55 DEBUG } 05:14:55 DEBUG ] 05:14:55 DEBUG } 05:14:55 DEBUG } 05:14:55 INFO [http_cmd]: http status code OK 05:14:55 DEBUG --- status code --- 05:14:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:14:55 DEBUG --- http response --- 05:14:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:14:55 DEBUG ----- output ----- 05:14:55 DEBUG { 05:14:55 DEBUG "status": "success", 05:14:55 DEBUG "data": { 05:14:55 DEBUG "resultType": "vector", 05:14:55 DEBUG "result": [ 05:14:55 DEBUG { 05:14:55 DEBUG "metric": { 05:14:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:14:55 DEBUG }, 05:14:55 DEBUG "value": [ 05:14:55 DEBUG 05:14:55 DEBUG 05:14:55 DEBUG 05:14:55 DEBUG [...] 05:14:55 DEBUG 05:14:55 DEBUG 05:14:55 DEBUG 05:14:55 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:14:55 DEBUG }, 05:14:55 DEBUG "value": [ 05:14:55 DEBUG 1713500095, 05:14:55 DEBUG "6468.628770160818" 05:14:55 DEBUG ] 05:14:55 DEBUG } 05:14:55 DEBUG ] 05:14:55 DEBUG } 05:14:55 DEBUG } 05:14:55 INFO [http_cmd]: http status code OK 05:14:55 DEBUG --- status code --- 05:14:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:14:55 DEBUG --- http response --- 05:14:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:14:55 DEBUG ----- output ----- 05:14:55 DEBUG { 05:14:55 DEBUG "status": "success", 05:14:55 DEBUG "data": { 05:14:55 DEBUG "resultType": "vector", 05:14:55 DEBUG "result": [ 05:14:55 DEBUG { 05:14:55 DEBUG "metric": { 05:14:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:14:55 DEBUG }, 05:14:55 DEBUG "value": [ 05:14:55 DEBUG 05:14:55 DEBUG 05:14:55 DEBUG 05:14:55 DEBUG [...] 05:14:55 DEBUG 05:14:55 DEBUG 05:14:55 DEBUG 05:14:55 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:14:55 DEBUG }, 05:14:55 DEBUG "value": [ 05:14:55 DEBUG 1713500095, 05:14:55 DEBUG "3.082542166120873" 05:14:55 DEBUG ] 05:14:55 DEBUG } 05:14:55 DEBUG ] 05:14:55 DEBUG } 05:14:55 DEBUG } 05:14:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500095" 05:14:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713500095" 05:14:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713500095" 05:14:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500095" 05:14:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500095" 05:14:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500095" 05:14:55 INFO [http_cmd]: http status code OK 05:14:55 DEBUG --- status code --- 05:14:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:14:55 DEBUG --- http response --- 05:14:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:14:55 DEBUG ----- output ----- 05:14:55 DEBUG { 05:14:55 DEBUG "status": "success", 05:14:55 DEBUG "data": { 05:14:55 DEBUG "resultType": "vector", 05:14:55 DEBUG "result": [ 05:14:55 DEBUG { 05:14:55 DEBUG "metric": { 05:14:55 DEBUG "instance": "10.142.0.107:9100" 05:14:55 DEBUG }, 05:14:55 DEBUG "value": [ 05:14:55 DEBUG 05:14:55 DEBUG 05:14:55 DEBUG 05:14:55 DEBUG [...] 05:14:55 DEBUG 05:14:55 DEBUG 05:14:55 DEBUG 05:14:55 DEBUG "instance": "10.142.0.43:9100" 05:14:55 DEBUG }, 05:14:55 DEBUG "value": [ 05:14:55 DEBUG 1713500095, 05:14:55 DEBUG "0.1913333333333327" 05:14:55 DEBUG ] 05:14:55 DEBUG } 05:14:55 DEBUG ] 05:14:55 DEBUG } 05:14:55 DEBUG } 05:14:55 INFO [http_cmd]: http status code OK 05:14:55 DEBUG --- status code --- 05:14:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:14:55 DEBUG --- http response --- 05:14:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:14:55 DEBUG ----- output ----- 05:14:55 DEBUG { 05:14:55 DEBUG "status": "success", 05:14:55 DEBUG "data": { 05:14:55 DEBUG "resultType": "vector", 05:14:55 DEBUG "result": [ 05:14:55 DEBUG { 05:14:55 DEBUG "metric": { 05:14:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:14:55 DEBUG }, 05:14:55 DEBUG "value": [ 05:14:55 DEBUG 05:14:55 DEBUG 05:14:55 DEBUG 05:14:55 DEBUG [...] 05:14:55 DEBUG 05:14:55 DEBUG 05:14:55 DEBUG 05:14:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:14:55 DEBUG }, 05:14:55 DEBUG "value": [ 05:14:55 DEBUG 1713500095, 05:14:55 DEBUG "6.510564112323368" 05:14:55 DEBUG ] 05:14:55 DEBUG } 05:14:55 DEBUG ] 05:14:55 DEBUG } 05:14:55 DEBUG } 05:14:55 ERROR [http_cmd]: ERROR 05:14:55 ERROR --- status code --- 05:14:55 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:14:55 ERROR --- http response --- 05:14:55 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:14:55 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:14:55 INFO [http_cmd]: http status code OK 05:14:55 INFO [http_cmd]: http status code OK 05:14:55 DEBUG --- status code --- 05:14:55 DEBUG --- status code --- 05:14:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:14:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:14:55 DEBUG --- http response --- 05:14:55 DEBUG --- http response --- 05:14:55 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:14:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:14:55 DEBUG ----- output ----- 05:14:55 DEBUG { 05:14:55 DEBUG "status": "success", 05:14:55 DEBUG "data": { 05:14:55 DEBUG "resultType": "vector", 05:14:55 DEBUG "result": [ 05:14:55 DEBUG { 05:14:55 DEBUG "metric": { 05:14:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:14:55 DEBUG }, 05:14:55 DEBUG "value": [ 05:14:55 DEBUG 05:14:55 DEBUG 05:14:55 DEBUG 05:14:55 DEBUG [...] 05:14:55 DEBUG 05:14:55 DEBUG 05:14:55 DEBUG 05:14:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:14:55 DEBUG }, 05:14:55 DEBUG "value": [ 05:14:55 DEBUG 1713500095, 05:14:55 DEBUG "0" 05:14:55 DEBUG ] 05:14:55 DEBUG } 05:14:55 DEBUG ] 05:14:55 DEBUG } 05:14:55 DEBUG } 05:14:55 INFO [http_cmd]: http status code OK 05:14:55 DEBUG --- status code --- 05:14:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:14:55 DEBUG --- http response --- 05:14:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:14:55 DEBUG ----- output ----- 05:14:55 DEBUG { 05:14:55 DEBUG "status": "success", 05:14:55 DEBUG "data": { 05:14:55 DEBUG "resultType": "vector", 05:14:55 DEBUG "result": [ 05:14:55 DEBUG { 05:14:55 DEBUG "metric": { 05:14:55 DEBUG "pod": "ds-cts-0" 05:14:55 DEBUG }, 05:14:55 DEBUG "value": [ 05:14:55 DEBUG 05:14:55 DEBUG 05:14:55 DEBUG 05:14:55 DEBUG [...] 05:14:55 DEBUG 05:14:55 DEBUG 05:14:55 DEBUG 05:14:55 DEBUG "pod": "ds-cts-2" 05:14:55 DEBUG }, 05:14:55 DEBUG "value": [ 05:14:55 DEBUG 1713500095, 05:14:55 DEBUG "0.0006680999999999999" 05:14:55 DEBUG ] 05:14:55 DEBUG } 05:14:55 DEBUG ] 05:14:55 DEBUG } 05:14:55 DEBUG } 05:14:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500095" 05:14:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500095" 05:14:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500095" 05:14:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500095" 05:14:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500095" 05:14:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500095" 05:14:55 INFO [http_cmd]: http status code OK 05:14:55 DEBUG --- status code --- 05:14:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:14:55 DEBUG --- http response --- 05:14:55 INFO [http_cmd]: http status code OK 05:14:55 INFO [http_cmd]: http status code OK 05:14:55 INFO [http_cmd]: http status code OK 05:14:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:14:55 DEBUG --- status code --- 05:14:55 DEBUG --- status code --- 05:14:55 DEBUG --- status code --- 05:14:55 DEBUG ----- output ----- 05:14:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:14:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:14:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:14:55 DEBUG { 05:14:55 DEBUG --- http response --- 05:14:55 DEBUG --- http response --- 05:14:55 DEBUG --- http response --- 05:14:55 DEBUG "status": "success", 05:14:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:14:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:14:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:14:55 DEBUG "data": { 05:14:55 DEBUG ----- output ----- 05:14:55 DEBUG ----- output ----- 05:14:55 DEBUG ----- output ----- 05:14:55 DEBUG "resultType": "vector", 05:14:55 DEBUG { 05:14:55 DEBUG { 05:14:55 DEBUG { 05:14:55 DEBUG "result": [ 05:14:55 DEBUG "status": "success", 05:14:55 DEBUG "status": "success", 05:14:55 DEBUG "status": "success", 05:14:55 DEBUG { 05:14:55 DEBUG "data": { 05:14:55 DEBUG "data": { 05:14:55 DEBUG "data": { 05:14:55 DEBUG "metric": { 05:14:55 DEBUG "resultType": "vector", 05:14:55 DEBUG "resultType": "vector", 05:14:55 DEBUG "resultType": "vector", 05:14:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:14:55 DEBUG "result": [ 05:14:55 DEBUG "result": [ 05:14:55 DEBUG "result": [ 05:14:55 DEBUG }, 05:14:55 DEBUG { 05:14:55 DEBUG { 05:14:55 DEBUG { 05:14:55 DEBUG "value": [ 05:14:55 DEBUG "metric": { 05:14:55 DEBUG "metric": { 05:14:55 DEBUG "metric": { 05:14:55 DEBUG 05:14:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:14:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:14:55 DEBUG "instance": "10.142.0.107:9100" 05:14:55 INFO [http_cmd]: http status code OK 05:14:55 DEBUG 05:14:55 DEBUG }, 05:14:55 DEBUG }, 05:14:55 INFO [http_cmd]: http status code OK 05:14:55 DEBUG }, 05:14:55 DEBUG --- status code --- 05:14:55 DEBUG 05:14:55 DEBUG "value": [ 05:14:55 DEBUG "value": [ 05:14:55 DEBUG --- status code --- 05:14:55 DEBUG "value": [ 05:14:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:14:55 DEBUG [...] 05:14:55 DEBUG 05:14:55 DEBUG 05:14:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:14:55 DEBUG 05:14:55 DEBUG --- http response --- 05:14:55 DEBUG 05:14:55 DEBUG 05:14:55 DEBUG 05:14:55 DEBUG --- http response --- 05:14:55 DEBUG 05:14:55 DEBUG 05:14:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:14:55 DEBUG 05:14:55 DEBUG 05:14:55 DEBUG 05:14:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:14:55 DEBUG 05:14:55 DEBUG ----- output ----- 05:14:55 DEBUG [...] 05:14:55 DEBUG [...] 05:14:55 DEBUG [...] 05:14:55 DEBUG ----- output ----- 05:14:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:14:55 DEBUG { 05:14:56 DEBUG 05:14:56 DEBUG 05:14:56 DEBUG 05:14:56 DEBUG { 05:14:56 DEBUG }, 05:14:56 DEBUG "status": "success", 05:14:56 DEBUG 05:14:56 DEBUG 05:14:56 DEBUG 05:14:56 DEBUG "status": "success", 05:14:56 DEBUG "value": [ 05:14:56 DEBUG "data": { 05:14:56 DEBUG 05:14:56 DEBUG 05:14:56 DEBUG 05:14:56 DEBUG "data": { 05:14:56 DEBUG 1713500095, 05:14:56 DEBUG "resultType": "vector", 05:14:56 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:14:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:14:56 DEBUG "instance": "10.142.0.43:9100" 05:14:56 DEBUG "resultType": "vector", 05:14:56 DEBUG "1995.5666666666666" 05:14:56 DEBUG "result": [ 05:14:56 DEBUG }, 05:14:56 DEBUG }, 05:14:56 DEBUG }, 05:14:56 DEBUG "result": [ 05:14:56 DEBUG ] 05:14:56 DEBUG { 05:14:56 DEBUG "value": [ 05:14:56 DEBUG "value": [ 05:14:56 DEBUG "value": [ 05:14:56 DEBUG { 05:14:56 DEBUG } 05:14:56 DEBUG "metric": { 05:14:56 DEBUG 1713500095, 05:14:56 DEBUG 1713500095, 05:14:56 DEBUG 1713500095, 05:14:56 DEBUG "metric": { 05:14:56 DEBUG ] 05:14:56 DEBUG "pod": "ds-cts-0" 05:14:56 DEBUG "0" 05:14:56 DEBUG "1995.7" 05:14:56 DEBUG "51070266.33333333" 05:14:56 DEBUG "instance": "10.142.0.107:9100" 05:14:56 DEBUG } 05:14:56 DEBUG }, 05:14:56 DEBUG ] 05:14:56 DEBUG ] 05:14:56 DEBUG ] 05:14:56 DEBUG }, 05:14:56 DEBUG } 05:14:56 DEBUG "value": [ 05:14:56 DEBUG } 05:14:56 DEBUG } 05:14:56 DEBUG } 05:14:56 DEBUG "value": [ 05:14:56 DEBUG 05:14:56 DEBUG ] 05:14:56 DEBUG ] 05:14:56 DEBUG ] 05:14:56 DEBUG 05:14:56 DEBUG 05:14:56 DEBUG } 05:14:56 DEBUG } 05:14:56 DEBUG } 05:14:56 DEBUG 05:14:56 DEBUG 05:14:56 DEBUG } 05:14:56 DEBUG } 05:14:56 DEBUG } 05:14:56 DEBUG 05:14:56 DEBUG [...] 05:14:56 DEBUG [...] 05:14:56 DEBUG 05:14:56 DEBUG 05:14:56 DEBUG 05:14:56 DEBUG 05:14:56 DEBUG 05:14:56 DEBUG 05:14:56 DEBUG "pod": "ds-cts-2" 05:14:56 DEBUG "instance": "10.142.0.43:9100" 05:14:56 DEBUG }, 05:14:56 DEBUG }, 05:14:56 DEBUG "value": [ 05:14:56 DEBUG "value": [ 05:14:56 DEBUG 1713500095, 05:14:56 DEBUG 1713500095, 05:14:56 DEBUG "249.63333333333333" 05:14:56 DEBUG "51066599.56666666" 05:14:56 DEBUG ] 05:14:56 DEBUG ] 05:14:56 DEBUG } 05:14:56 DEBUG } 05:14:56 DEBUG ] 05:14:56 DEBUG ] 05:14:56 DEBUG } 05:14:56 DEBUG } 05:14:56 DEBUG } 05:14:56 DEBUG } 05:14:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713500096" 05:14:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500096" 05:14:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500096" 05:14:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500096" 05:14:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500096" 05:14:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500096" 05:14:56 INFO [http_cmd]: http status code OK 05:14:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713500096" 05:14:56 DEBUG --- status code --- 05:14:56 INFO [http_cmd]: http status code OK 05:14:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500096" 05:14:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:14:56 DEBUG --- status code --- 05:14:56 DEBUG --- http response --- 05:14:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:14:56 INFO [http_cmd]: http status code OK 05:14:56 DEBUG --- http response --- 05:14:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:14:56 DEBUG --- status code --- 05:14:56 DEBUG ----- output ----- 05:14:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:14:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:14:56 DEBUG { 05:14:56 DEBUG --- http response --- 05:14:56 DEBUG ----- output ----- 05:14:56 DEBUG "status": "success", 05:14:56 DEBUG { 05:14:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:14:56 DEBUG "data": { 05:14:56 DEBUG "status": "success", 05:14:56 INFO [http_cmd]: http status code OK 05:14:56 DEBUG ----- output ----- 05:14:56 DEBUG "resultType": "vector", 05:14:56 DEBUG "data": { 05:14:56 DEBUG --- status code --- 05:14:56 DEBUG { 05:14:56 DEBUG "result": [ 05:14:56 DEBUG "resultType": "vector", 05:14:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:14:56 DEBUG "status": "success", 05:14:56 DEBUG { 05:14:56 DEBUG "result": [ 05:14:56 DEBUG --- http response --- 05:14:56 DEBUG "data": { 05:14:56 DEBUG "metric": { 05:14:56 DEBUG { 05:14:56 DEBUG "resultType": "vector", 05:14:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:14:56 DEBUG "pod": "ds-cts-0" 05:14:56 DEBUG "metric": { 05:14:56 DEBUG "result": [ 05:14:56 INFO [http_cmd]: http status code OK 05:14:56 INFO [http_cmd]: http status code OK 05:14:56 INFO [http_cmd]: http status code OK 05:14:56 DEBUG ----- output ----- 05:14:56 DEBUG }, 05:14:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:14:56 DEBUG { 05:14:56 DEBUG --- status code --- 05:14:56 DEBUG --- status code --- 05:14:56 DEBUG --- status code --- 05:14:56 DEBUG { 05:14:56 DEBUG "value": [ 05:14:56 DEBUG }, 05:14:56 DEBUG "metric": { 05:14:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:14:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:14:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:14:56 DEBUG "status": "success", 05:14:56 DEBUG 05:14:56 DEBUG "value": [ 05:14:56 DEBUG "instance": "10.142.0.107:9100" 05:14:56 DEBUG --- http response --- 05:14:56 DEBUG --- http response --- 05:14:56 DEBUG --- http response --- 05:14:56 DEBUG "data": { 05:14:56 DEBUG 05:14:56 DEBUG 05:14:56 DEBUG }, 05:14:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:14:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:14:56 DEBUG "resultType": "vector", 05:14:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:14:56 DEBUG 05:14:56 DEBUG 05:14:56 DEBUG "value": [ 05:14:56 DEBUG ----- output ----- 05:14:56 DEBUG ----- output ----- 05:14:56 DEBUG "result": [ 05:14:56 DEBUG ----- output ----- 05:14:56 DEBUG [...] 05:14:56 DEBUG 05:14:56 DEBUG 05:14:56 DEBUG { 05:14:56 DEBUG { 05:14:56 DEBUG { 05:14:56 DEBUG { 05:14:56 DEBUG 05:14:56 DEBUG [...] 05:14:56 DEBUG 05:14:56 DEBUG "status": "success", 05:14:56 DEBUG "status": "success", 05:14:56 INFO [http_cmd]: http status code OK 05:14:56 DEBUG "metric": { 05:14:56 DEBUG "status": "success", 05:14:56 DEBUG 05:14:56 DEBUG 05:14:56 DEBUG 05:14:56 DEBUG "data": { 05:14:56 DEBUG "data": { 05:14:56 DEBUG --- status code --- 05:14:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:14:56 DEBUG "data": { 05:14:56 DEBUG 05:14:56 DEBUG 05:14:56 DEBUG [...] 05:14:56 DEBUG "resultType": "vector", 05:14:56 DEBUG "resultType": "vector", 05:14:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:14:56 DEBUG }, 05:14:56 DEBUG "resultType": "vector", 05:14:56 DEBUG "pod": "ds-cts-2" 05:14:56 DEBUG 05:14:56 DEBUG 05:14:56 DEBUG "result": [ 05:14:56 DEBUG "result": [ 05:14:56 DEBUG --- http response --- 05:14:56 DEBUG "value": [ 05:14:56 DEBUG "result": [ 05:14:56 DEBUG }, 05:14:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:14:56 DEBUG 05:14:56 DEBUG { 05:14:56 DEBUG { 05:14:56 DEBUG 05:14:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:14:56 DEBUG { 05:14:56 DEBUG "value": [ 05:14:56 DEBUG }, 05:14:56 DEBUG 05:14:56 DEBUG "metric": { 05:14:56 DEBUG "metric": { 05:14:56 DEBUG 05:14:56 DEBUG ----- output ----- 05:14:56 DEBUG "metric": { 05:14:56 DEBUG 1713500096, 05:14:56 DEBUG "value": [ 05:14:56 DEBUG "instance": "10.142.0.43:9100" 05:14:56 DEBUG "pod": "ds-idrepo-0" 05:14:56 DEBUG "pod": "ds-idrepo-0" 05:14:56 DEBUG 05:14:56 DEBUG { 05:14:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:14:56 DEBUG "0" 05:14:56 DEBUG 1713500096, 05:14:56 DEBUG }, 05:14:56 DEBUG }, 05:14:56 DEBUG }, 05:14:56 DEBUG [...] 05:14:56 DEBUG "status": "success", 05:14:56 DEBUG }, 05:14:56 DEBUG ] 05:14:56 DEBUG "2183634944" 05:14:56 DEBUG "value": [ 05:14:56 DEBUG "value": [ 05:14:56 DEBUG "value": [ 05:14:56 DEBUG 05:14:56 DEBUG "data": { 05:14:56 DEBUG "value": [ 05:14:56 DEBUG } 05:14:56 DEBUG ] 05:14:56 DEBUG 1713500096, 05:14:56 DEBUG 05:14:56 DEBUG 05:14:56 DEBUG 05:14:56 DEBUG "resultType": "vector", 05:14:56 DEBUG 05:14:56 DEBUG ] 05:14:56 DEBUG } 05:14:56 DEBUG "456387785728" 05:14:56 DEBUG 05:14:56 DEBUG 05:14:56 DEBUG 05:14:56 DEBUG "result": [ 05:14:56 DEBUG 05:14:56 DEBUG } 05:14:56 DEBUG ] 05:14:56 DEBUG ] 05:14:56 DEBUG 05:14:56 DEBUG 05:14:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:14:56 DEBUG { 05:14:56 DEBUG 05:14:56 DEBUG } 05:14:56 DEBUG } 05:14:56 DEBUG } 05:14:56 DEBUG [...] 05:14:56 DEBUG [...] 05:14:56 DEBUG }, 05:14:56 DEBUG "metric": { 05:14:56 DEBUG [...] 05:14:56 DEBUG } 05:14:56 DEBUG ] 05:14:56 DEBUG 05:14:56 DEBUG 05:14:56 DEBUG "value": [ 05:14:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:14:56 DEBUG 05:14:56 DEBUG } 05:14:56 DEBUG 05:14:56 DEBUG 05:14:56 DEBUG 1713500096, 05:14:56 DEBUG }, 05:14:56 DEBUG 05:14:56 DEBUG } 05:14:56 DEBUG 05:14:56 DEBUG 05:14:56 DEBUG "1995.6333333333334" 05:14:56 DEBUG "value": [ 05:14:56 DEBUG 05:14:56 DEBUG "pod": "ds-idrepo-2" 05:14:56 DEBUG "pod": "ds-idrepo-2" 05:14:56 DEBUG ] 05:14:56 DEBUG 05:14:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:14:56 DEBUG }, 05:14:56 DEBUG }, 05:14:56 DEBUG } 05:14:56 DEBUG 05:14:56 DEBUG }, 05:14:56 DEBUG "value": [ 05:14:56 DEBUG "value": [ 05:14:56 DEBUG ] 05:14:56 DEBUG 05:14:56 DEBUG "value": [ 05:14:56 DEBUG 1713500096, 05:14:56 DEBUG 1713500096, 05:14:56 DEBUG } 05:14:56 DEBUG [...] 05:14:56 DEBUG 1713500096, 05:14:56 DEBUG "0" 05:14:56 DEBUG "0" 05:14:56 DEBUG } 05:14:56 DEBUG 05:14:56 DEBUG "55.172" 05:14:56 DEBUG ] 05:14:56 DEBUG ] 05:14:56 DEBUG 05:14:56 DEBUG ] 05:14:56 DEBUG } 05:14:56 DEBUG } 05:14:56 DEBUG 05:14:56 DEBUG } 05:14:56 DEBUG ] 05:14:56 DEBUG ] 05:14:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:14:56 DEBUG ] 05:14:56 DEBUG } 05:14:56 DEBUG } 05:14:56 DEBUG }, 05:14:56 DEBUG } 05:14:56 DEBUG } 05:14:56 DEBUG } 05:14:56 DEBUG "value": [ 05:14:56 DEBUG } 05:14:56 DEBUG 1713500096, 05:14:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500096" 05:14:56 DEBUG "465.56666666666666" 05:14:56 DEBUG ] 05:14:56 DEBUG } 05:14:56 DEBUG ] 05:14:56 DEBUG } 05:14:56 DEBUG } 05:14:56 INFO [http_cmd]: http status code OK 05:14:56 DEBUG --- status code --- 05:14:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:14:56 DEBUG --- http response --- 05:14:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:14:56 DEBUG ----- output ----- 05:14:56 DEBUG { 05:14:56 DEBUG "status": "success", 05:14:56 DEBUG "data": { 05:14:56 DEBUG "resultType": "vector", 05:14:56 DEBUG "result": [ 05:14:56 DEBUG { 05:14:56 DEBUG "metric": { 05:14:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:14:56 DEBUG }, 05:14:56 DEBUG "value": [ 05:14:56 DEBUG 05:14:56 DEBUG 05:14:56 DEBUG 05:14:56 DEBUG [...] 05:14:56 DEBUG 05:14:56 DEBUG 05:14:56 DEBUG 05:14:56 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:14:56 DEBUG }, 05:14:56 DEBUG "value": [ 05:14:56 DEBUG 1713500096, 05:14:56 DEBUG "0" 05:14:56 DEBUG ] 05:14:56 DEBUG } 05:14:56 DEBUG ] 05:14:56 DEBUG } 05:14:56 DEBUG } 05:14:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500096" 05:14:56 INFO [http_cmd]: http status code OK 05:14:56 DEBUG --- status code --- 05:14:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:14:56 DEBUG --- http response --- 05:14:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:14:56 DEBUG ----- output ----- 05:14:56 DEBUG { 05:14:56 DEBUG "status": "success", 05:14:56 DEBUG "data": { 05:14:56 DEBUG "resultType": "vector", 05:14:56 DEBUG "result": [ 05:14:56 DEBUG { 05:14:56 DEBUG "metric": { 05:14:56 DEBUG "instance": "10.142.0.107:9100" 05:14:56 DEBUG }, 05:14:56 DEBUG "value": [ 05:14:56 DEBUG 05:14:56 DEBUG 05:14:56 DEBUG 05:14:56 DEBUG [...] 05:14:56 DEBUG 05:14:56 DEBUG 05:14:56 DEBUG 05:14:56 DEBUG "instance": "10.142.0.43:9100" 05:14:56 DEBUG }, 05:14:56 DEBUG "value": [ 05:14:56 DEBUG 1713500096, 05:14:56 DEBUG "0.0906666666666704" 05:14:56 DEBUG ] 05:14:56 DEBUG } 05:14:56 DEBUG ] 05:14:56 DEBUG } 05:14:56 DEBUG } 05:14:57 INFO 05:14:57 INFO [loop_until]: kubectl --namespace=xlou top pods 05:14:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:14:57 INFO [loop_until]: OK (rc = 0) 05:14:57 DEBUG --- stdout --- 05:14:57 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 11159m 9134Mi am-6fd554f8f7-hvvkw 10564m 8881Mi am-6fd554f8f7-pmj4b 11118m 8985Mi ds-cts-0 11817m 12502Mi ds-cts-1 10598m 12426Mi ds-cts-2 13100m 12440Mi ds-idrepo-0 4964m 3267Mi ds-idrepo-1 3616m 3207Mi ds-idrepo-2 4817m 3065Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 4m 1215Mi lodemon-6b89d774d5-rf7sk 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 6600m 1988Mi 05:14:57 DEBUG --- stderr --- 05:14:57 DEBUG 05:14:57 INFO 05:14:57 INFO [loop_until]: kubectl --namespace=xlou top node 05:14:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:14:57 INFO [loop_until]: OK (rc = 0) 05:14:57 DEBUG --- stdout --- 05:14:57 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 71m 0% 6254Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 4136m 13% 6706Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 8843m 29% 11604Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 85m 0% 3242Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 11819m 39% 15078Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 158m 0% 5167Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 9567m 32% 11929Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 9573m 32% 2730Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 12072m 40% 11200Mi 10% gke-xlou-cdm-ds-0f601751-j488 4405m 14% 6166Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 11563m 38% 16409Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 4853m 16% 6873Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 12743m 42% 14367Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 6196m 38% 5253Mi 8% 05:14:57 DEBUG --- stderr --- 05:14:57 DEBUG 05:15:09 WARNING File /tmp/lodemon_data-20240419_044251/node_disk_read_bytes_total.json does not exist. Skipping average value computation... 127.0.0.1 - - [19/Apr/2024 05:15:09] "GET /monitoring/average?start_time=20240419_034556&stop_time=20240419_041308 HTTP/1.1" 200 - 05:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500155" 05:15:55 INFO [http_cmd]: http status code OK 05:15:55 DEBUG --- status code --- 05:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:15:55 DEBUG --- http response --- 05:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:15:55 DEBUG ----- output ----- 05:15:55 DEBUG { 05:15:55 DEBUG "status": "success", 05:15:55 DEBUG "data": { 05:15:55 DEBUG "resultType": "vector", 05:15:55 DEBUG "result": [ 05:15:55 DEBUG { 05:15:55 DEBUG "metric": { 05:15:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:15:55 DEBUG }, 05:15:55 DEBUG "value": [ 05:15:55 DEBUG 05:15:55 DEBUG 05:15:55 DEBUG 05:15:55 DEBUG [...] 05:15:55 DEBUG 05:15:55 DEBUG 05:15:55 DEBUG 05:15:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:15:55 DEBUG }, 05:15:55 DEBUG "value": [ 05:15:55 DEBUG 1713500155, 05:15:55 DEBUG "3532021.5" 05:15:55 DEBUG ] 05:15:55 DEBUG } 05:15:55 DEBUG ] 05:15:55 DEBUG } 05:15:55 DEBUG } 05:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713500155" 05:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500155" 05:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713500155" 05:15:55 INFO [http_cmd]: http status code OK 05:15:55 INFO [http_cmd]: http status code OK 05:15:55 DEBUG --- status code --- 05:15:55 DEBUG --- status code --- 05:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:15:55 DEBUG --- http response --- 05:15:55 DEBUG --- http response --- 05:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:15:55 DEBUG ----- output ----- 05:15:55 DEBUG { 05:15:55 DEBUG "status": "success", 05:15:55 DEBUG "data": { 05:15:55 DEBUG "resultType": "vector", 05:15:55 DEBUG "result": [ 05:15:55 DEBUG { 05:15:55 DEBUG "metric": { 05:15:55 DEBUG "pod": "ds-idrepo-0" 05:15:55 DEBUG }, 05:15:55 DEBUG "value": [ 05:15:55 DEBUG 05:15:55 DEBUG 05:15:55 DEBUG 05:15:55 DEBUG [...] 05:15:55 DEBUG 05:15:55 DEBUG 05:15:55 DEBUG 05:15:55 DEBUG "pod": "ds-cts-2" 05:15:55 DEBUG }, 05:15:55 DEBUG "value": [ 05:15:55 DEBUG 1713500155, 05:15:55 DEBUG "0" 05:15:55 DEBUG ] 05:15:55 DEBUG } 05:15:55 DEBUG ] 05:15:55 DEBUG } 05:15:55 DEBUG } 05:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:15:55 DEBUG ----- output ----- 05:15:55 DEBUG { 05:15:55 DEBUG "status": "success", 05:15:55 DEBUG "data": { 05:15:55 DEBUG "resultType": "vector", 05:15:55 DEBUG "result": [ 05:15:55 DEBUG { 05:15:55 DEBUG "metric": { 05:15:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:15:55 DEBUG }, 05:15:55 DEBUG "value": [ 05:15:55 DEBUG 05:15:55 DEBUG 05:15:55 DEBUG 05:15:55 DEBUG [...] 05:15:55 DEBUG 05:15:55 DEBUG 05:15:55 DEBUG 05:15:55 DEBUG "pod": "idm-b85f9d978-wnhxb" 05:15:55 DEBUG }, 05:15:55 DEBUG "value": [ 05:15:55 DEBUG 1713500155, 05:15:55 DEBUG "77478588.73063898" 05:15:55 DEBUG ] 05:15:55 DEBUG } 05:15:55 DEBUG ] 05:15:55 DEBUG } 05:15:55 DEBUG } 05:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500155" 05:15:55 INFO [http_cmd]: http status code OK 05:15:55 DEBUG --- status code --- 05:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:15:55 DEBUG --- http response --- 05:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:15:55 DEBUG ----- output ----- 05:15:55 DEBUG { 05:15:55 DEBUG "status": "success", 05:15:55 DEBUG "data": { 05:15:55 DEBUG "resultType": "vector", 05:15:55 DEBUG "result": [ 05:15:55 DEBUG { 05:15:55 DEBUG "metric": { 05:15:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:15:55 DEBUG }, 05:15:55 DEBUG "value": [ 05:15:55 DEBUG 05:15:55 DEBUG 05:15:55 DEBUG 05:15:55 DEBUG [...] 05:15:55 DEBUG 05:15:55 DEBUG 05:15:55 DEBUG 05:15:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:15:55 DEBUG }, 05:15:55 DEBUG "value": [ 05:15:55 DEBUG 1713500155, 05:15:55 DEBUG "1.1324456629387585" 05:15:55 DEBUG ] 05:15:55 DEBUG } 05:15:55 DEBUG ] 05:15:55 DEBUG } 05:15:55 DEBUG } 05:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500155" 05:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713500155" 05:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500155" 05:15:55 INFO [http_cmd]: http status code OK 05:15:55 INFO [http_cmd]: http status code OK 05:15:55 DEBUG --- status code --- 05:15:55 DEBUG --- status code --- 05:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:15:55 DEBUG --- http response --- 05:15:55 DEBUG --- http response --- 05:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:15:55 DEBUG ----- output ----- 05:15:55 DEBUG { 05:15:55 DEBUG "status": "success", 05:15:55 DEBUG "data": { 05:15:55 DEBUG "resultType": "vector", 05:15:55 DEBUG "result": [ 05:15:55 DEBUG { 05:15:55 DEBUG "metric": { 05:15:55 DEBUG "pod": "ds-idrepo-0" 05:15:55 DEBUG }, 05:15:55 DEBUG "value": [ 05:15:55 DEBUG 05:15:55 DEBUG 05:15:55 DEBUG 05:15:55 DEBUG [...] 05:15:55 DEBUG 05:15:55 DEBUG 05:15:55 DEBUG 05:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:15:55 DEBUG "pod": "ds-idrepo-2" 05:15:55 DEBUG ----- output ----- 05:15:55 DEBUG }, 05:15:55 DEBUG { 05:15:55 DEBUG "value": [ 05:15:55 DEBUG "status": "success", 05:15:55 DEBUG 1713500155, 05:15:55 DEBUG "data": { 05:15:55 DEBUG "0" 05:15:55 DEBUG "resultType": "vector", 05:15:55 DEBUG ] 05:15:55 DEBUG "result": [ 05:15:55 DEBUG } 05:15:55 DEBUG { 05:15:55 DEBUG ] 05:15:55 DEBUG "metric": { 05:15:55 DEBUG } 05:15:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:15:55 DEBUG } 05:15:55 DEBUG }, 05:15:55 DEBUG "value": [ 05:15:55 DEBUG 05:15:55 DEBUG 05:15:55 DEBUG 05:15:55 DEBUG [...] 05:15:55 DEBUG 05:15:55 DEBUG 05:15:55 DEBUG 05:15:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:15:55 DEBUG }, 05:15:55 DEBUG "value": [ 05:15:55 DEBUG 1713500155, 05:15:55 DEBUG "0" 05:15:55 DEBUG ] 05:15:55 DEBUG } 05:15:55 DEBUG ] 05:15:55 DEBUG } 05:15:55 DEBUG } 05:15:56 ERROR [http_cmd]: ERROR 05:15:56 ERROR --- status code --- 05:15:56 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:15:56 ERROR --- http response --- 05:15:56 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:15:56 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:15:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500156" 05:15:56 INFO [http_cmd]: http status code OK 05:15:56 DEBUG --- status code --- 05:15:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:15:56 DEBUG --- http response --- 05:15:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:15:56 DEBUG ----- output ----- 05:15:56 DEBUG { 05:15:56 DEBUG "status": "success", 05:15:56 DEBUG "data": { 05:15:56 DEBUG "resultType": "vector", 05:15:56 DEBUG "result": [ 05:15:56 DEBUG { 05:15:56 DEBUG "metric": { 05:15:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:15:56 DEBUG }, 05:15:56 DEBUG "value": [ 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG [...] 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG "pod": "idm-b85f9d978-wnhxb" 05:15:56 DEBUG }, 05:15:56 DEBUG "value": [ 05:15:56 DEBUG 1713500155, 05:15:56 DEBUG "0.006948037626662772" 05:15:56 DEBUG ] 05:15:56 DEBUG } 05:15:56 DEBUG ] 05:15:56 DEBUG } 05:15:56 DEBUG } 05:15:56 INFO [http_cmd]: http status code OK 05:15:56 DEBUG --- status code --- 05:15:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:15:56 DEBUG --- http response --- 05:15:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:15:56 DEBUG ----- output ----- 05:15:56 DEBUG { 05:15:56 DEBUG "status": "success", 05:15:56 DEBUG "data": { 05:15:56 DEBUG "resultType": "vector", 05:15:56 DEBUG "result": [ 05:15:56 DEBUG { 05:15:56 DEBUG "metric": { 05:15:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:15:56 DEBUG }, 05:15:56 DEBUG "value": [ 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG [...] 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:15:56 DEBUG }, 05:15:56 DEBUG "value": [ 05:15:56 DEBUG 1713500156, 05:15:56 DEBUG "81.6" 05:15:56 DEBUG ] 05:15:56 DEBUG } 05:15:56 DEBUG ] 05:15:56 DEBUG } 05:15:56 DEBUG } 05:15:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500156" 05:15:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500156" 05:15:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500156" 05:15:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500156" 05:15:56 INFO [http_cmd]: http status code OK 05:15:56 DEBUG --- status code --- 05:15:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:15:56 DEBUG --- http response --- 05:15:56 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:15:56 INFO [http_cmd]: http status code OK 05:15:56 INFO [http_cmd]: http status code OK 05:15:56 INFO [http_cmd]: http status code OK 05:15:56 DEBUG --- status code --- 05:15:56 DEBUG --- status code --- 05:15:56 DEBUG --- status code --- 05:15:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:15:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:15:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:15:56 DEBUG --- http response --- 05:15:56 DEBUG --- http response --- 05:15:56 DEBUG --- http response --- 05:15:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:15:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:15:56 DEBUG ----- output ----- 05:15:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:15:56 DEBUG ----- output ----- 05:15:56 DEBUG { 05:15:56 DEBUG ----- output ----- 05:15:56 DEBUG { 05:15:56 DEBUG "status": "success", 05:15:56 DEBUG { 05:15:56 DEBUG "status": "success", 05:15:56 DEBUG "data": { 05:15:56 DEBUG "status": "success", 05:15:56 DEBUG "data": { 05:15:56 DEBUG "resultType": "vector", 05:15:56 DEBUG "data": { 05:15:56 DEBUG "resultType": "vector", 05:15:56 DEBUG "result": [ 05:15:56 DEBUG "resultType": "vector", 05:15:56 DEBUG "result": [ 05:15:56 DEBUG { 05:15:56 DEBUG "result": [ 05:15:56 DEBUG { 05:15:56 DEBUG "metric": { 05:15:56 DEBUG { 05:15:56 DEBUG "metric": { 05:15:56 DEBUG "pod": "ds-cts-0" 05:15:56 DEBUG "metric": { 05:15:56 DEBUG "instance": "10.142.0.107:9100" 05:15:56 DEBUG }, 05:15:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:15:56 DEBUG }, 05:15:56 DEBUG "value": [ 05:15:56 DEBUG }, 05:15:56 DEBUG "value": [ 05:15:56 DEBUG 05:15:56 DEBUG "value": [ 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG [...] 05:15:56 DEBUG 05:15:56 DEBUG [...] 05:15:56 DEBUG 05:15:56 DEBUG [...] 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG "pod": "ds-cts-2" 05:15:56 DEBUG 05:15:56 DEBUG "instance": "10.142.0.43:9100" 05:15:56 DEBUG }, 05:15:56 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 05:15:56 DEBUG }, 05:15:56 DEBUG "value": [ 05:15:56 DEBUG }, 05:15:56 DEBUG "value": [ 05:15:56 DEBUG 1713500156, 05:15:56 DEBUG "value": [ 05:15:56 DEBUG 1713500156, 05:15:56 DEBUG "0.00035501777777777784" 05:15:56 DEBUG 1713500156, 05:15:56 DEBUG "0.5295666666666649" 05:15:56 DEBUG ] 05:15:56 DEBUG "0" 05:15:56 DEBUG ] 05:15:56 DEBUG } 05:15:56 DEBUG ] 05:15:56 DEBUG } 05:15:56 DEBUG ] 05:15:56 DEBUG } 05:15:56 DEBUG ] 05:15:56 DEBUG } 05:15:56 DEBUG ] 05:15:56 DEBUG } 05:15:56 DEBUG } 05:15:56 DEBUG } 05:15:56 DEBUG } 05:15:56 DEBUG } 05:15:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713500156" 05:15:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500156" 05:15:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500156" 05:15:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500156" 05:15:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500156" 05:15:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500156" 05:15:56 INFO [http_cmd]: http status code OK 05:15:56 INFO [http_cmd]: http status code OK 05:15:56 DEBUG --- status code --- 05:15:56 DEBUG --- status code --- 05:15:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:15:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:15:56 DEBUG --- http response --- 05:15:56 DEBUG --- http response --- 05:15:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:15:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:15:56 DEBUG ----- output ----- 05:15:56 DEBUG ----- output ----- 05:15:56 DEBUG { 05:15:56 DEBUG { 05:15:56 DEBUG "status": "success", 05:15:56 DEBUG "status": "success", 05:15:56 DEBUG "data": { 05:15:56 DEBUG "data": { 05:15:56 DEBUG "resultType": "vector", 05:15:56 DEBUG "resultType": "vector", 05:15:56 DEBUG "result": [ 05:15:56 DEBUG "result": [ 05:15:56 DEBUG { 05:15:56 INFO [http_cmd]: http status code OK 05:15:56 DEBUG { 05:15:56 INFO [http_cmd]: http status code OK 05:15:56 DEBUG "metric": { 05:15:56 DEBUG --- status code --- 05:15:56 DEBUG "metric": { 05:15:56 DEBUG --- status code --- 05:15:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:15:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:15:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:15:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:15:56 DEBUG }, 05:15:56 DEBUG --- http response --- 05:15:56 DEBUG }, 05:15:56 DEBUG --- http response --- 05:15:56 DEBUG "value": [ 05:15:56 DEBUG "value": [ 05:15:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:15:56 DEBUG 05:15:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:15:56 DEBUG 05:15:56 INFO [http_cmd]: http status code OK 05:15:56 DEBUG ----- output ----- 05:15:56 DEBUG 05:15:56 DEBUG ----- output ----- 05:15:56 DEBUG 05:15:56 DEBUG --- status code --- 05:15:56 DEBUG { 05:15:56 DEBUG 05:15:56 DEBUG { 05:15:56 DEBUG 05:15:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:15:56 DEBUG "status": "success", 05:15:56 DEBUG [...] 05:15:56 DEBUG "status": "success", 05:15:56 DEBUG [...] 05:15:56 DEBUG --- http response --- 05:15:56 DEBUG "data": { 05:15:56 DEBUG 05:15:56 DEBUG "data": { 05:15:56 DEBUG 05:15:56 DEBUG "resultType": "vector", 05:15:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:15:56 DEBUG 05:15:56 DEBUG "resultType": "vector", 05:15:56 DEBUG 05:15:56 DEBUG "result": [ 05:15:56 DEBUG ----- output ----- 05:15:56 DEBUG 05:15:56 DEBUG "result": [ 05:15:56 DEBUG 05:15:56 DEBUG { 05:15:56 DEBUG { 05:15:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:15:56 DEBUG { 05:15:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:15:56 DEBUG "metric": { 05:15:56 DEBUG "status": "success", 05:15:56 DEBUG }, 05:15:56 DEBUG "metric": { 05:15:56 DEBUG }, 05:15:56 DEBUG "instance": "10.142.0.107:9100" 05:15:56 DEBUG "data": { 05:15:56 DEBUG "value": [ 05:15:56 INFO [http_cmd]: http status code OK 05:15:56 DEBUG "instance": "10.142.0.107:9100" 05:15:56 DEBUG "value": [ 05:15:56 DEBUG }, 05:15:56 DEBUG "resultType": "vector", 05:15:56 DEBUG 1713500156, 05:15:56 DEBUG --- status code --- 05:15:56 DEBUG }, 05:15:56 DEBUG 1713500156, 05:15:56 DEBUG "value": [ 05:15:56 DEBUG "result": [ 05:15:56 DEBUG "55.172" 05:15:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:15:56 DEBUG "value": [ 05:15:56 DEBUG "81.73333333333333" 05:15:56 DEBUG 05:15:56 DEBUG { 05:15:56 DEBUG ] 05:15:56 DEBUG --- http response --- 05:15:56 DEBUG 05:15:56 DEBUG ] 05:15:56 DEBUG 05:15:56 DEBUG "metric": { 05:15:56 DEBUG } 05:15:56 DEBUG 05:15:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:15:56 DEBUG } 05:15:56 DEBUG 05:15:56 DEBUG "instance": "10.142.0.107:9100" 05:15:56 DEBUG ] 05:15:56 DEBUG 05:15:56 DEBUG ----- output ----- 05:15:56 DEBUG ] 05:15:56 DEBUG [...] 05:15:56 DEBUG }, 05:15:56 DEBUG } 05:15:56 DEBUG [...] 05:15:56 DEBUG { 05:15:56 DEBUG } 05:15:56 DEBUG 05:15:56 DEBUG "value": [ 05:15:56 DEBUG } 05:15:56 DEBUG 05:15:56 DEBUG "status": "success", 05:15:56 DEBUG } 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG "data": { 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG "resultType": "vector", 05:15:56 DEBUG "instance": "10.142.0.43:9100" 05:15:56 DEBUG 05:15:56 DEBUG "instance": "10.142.0.43:9100" 05:15:56 DEBUG "result": [ 05:15:56 DEBUG }, 05:15:56 DEBUG [...] 05:15:56 DEBUG }, 05:15:56 DEBUG { 05:15:56 DEBUG "value": [ 05:15:56 DEBUG 05:15:56 DEBUG "value": [ 05:15:56 DEBUG "metric": { 05:15:56 DEBUG 1713500156, 05:15:56 DEBUG 05:15:56 DEBUG 1713500156, 05:15:56 DEBUG "pod": "ds-cts-0" 05:15:56 DEBUG "457750035456" 05:15:56 DEBUG 05:15:56 DEBUG "15827004.399999999" 05:15:56 DEBUG }, 05:15:56 DEBUG ] 05:15:56 DEBUG "instance": "10.142.0.43:9100" 05:15:56 DEBUG ] 05:15:56 DEBUG "value": [ 05:15:56 DEBUG } 05:15:56 DEBUG }, 05:15:56 DEBUG } 05:15:56 DEBUG 05:15:56 DEBUG ] 05:15:56 DEBUG "value": [ 05:15:56 DEBUG ] 05:15:56 DEBUG 05:15:56 DEBUG } 05:15:56 DEBUG 1713500156, 05:15:56 DEBUG } 05:15:56 DEBUG 05:15:56 DEBUG } 05:15:56 DEBUG "15833189.166666666" 05:15:56 DEBUG } 05:15:56 DEBUG [...] 05:15:56 DEBUG ] 05:15:56 DEBUG 05:15:56 DEBUG } 05:15:56 DEBUG 05:15:56 DEBUG ] 05:15:56 DEBUG 05:15:56 DEBUG } 05:15:56 DEBUG "pod": "ds-cts-2" 05:15:56 DEBUG } 05:15:56 DEBUG }, 05:15:56 DEBUG "value": [ 05:15:56 DEBUG 1713500156, 05:15:56 DEBUG "532.9333333333333" 05:15:56 DEBUG ] 05:15:56 DEBUG } 05:15:56 DEBUG ] 05:15:56 DEBUG } 05:15:56 DEBUG } 05:15:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500156" 05:15:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500156" 05:15:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500156" 05:15:56 INFO [http_cmd]: http status code OK 05:15:56 DEBUG --- status code --- 05:15:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:15:56 DEBUG --- http response --- 05:15:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:15:56 DEBUG ----- output ----- 05:15:56 DEBUG { 05:15:56 DEBUG "status": "success", 05:15:56 DEBUG "data": { 05:15:56 DEBUG "resultType": "vector", 05:15:56 DEBUG "result": [ 05:15:56 DEBUG { 05:15:56 DEBUG "metric": { 05:15:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:15:56 DEBUG }, 05:15:56 DEBUG "value": [ 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG [...] 05:15:56 INFO [http_cmd]: http status code OK 05:15:56 DEBUG 05:15:56 DEBUG --- status code --- 05:15:56 DEBUG 05:15:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:15:56 DEBUG 05:15:56 DEBUG --- http response --- 05:15:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:15:56 DEBUG }, 05:15:56 DEBUG "value": [ 05:15:56 DEBUG 1713500156, 05:15:56 DEBUG "2000.5" 05:15:56 DEBUG ] 05:15:56 DEBUG } 05:15:56 DEBUG ] 05:15:56 DEBUG } 05:15:56 DEBUG } 05:15:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:15:56 DEBUG ----- output ----- 05:15:56 DEBUG { 05:15:56 DEBUG "status": "success", 05:15:56 DEBUG "data": { 05:15:56 DEBUG "resultType": "vector", 05:15:56 DEBUG "result": [ 05:15:56 DEBUG { 05:15:56 DEBUG "metric": { 05:15:56 DEBUG "pod": "ds-cts-0" 05:15:56 DEBUG }, 05:15:56 DEBUG "value": [ 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG [...] 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG "pod": "ds-cts-2" 05:15:56 DEBUG }, 05:15:56 DEBUG "value": [ 05:15:56 DEBUG 1713500156, 05:15:56 DEBUG "0" 05:15:56 DEBUG ] 05:15:56 DEBUG } 05:15:56 DEBUG ] 05:15:56 DEBUG } 05:15:56 DEBUG } 05:15:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713500156" 05:15:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500156" 05:15:56 INFO [http_cmd]: http status code OK 05:15:56 DEBUG --- status code --- 05:15:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:15:56 DEBUG --- http response --- 05:15:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:15:56 DEBUG ----- output ----- 05:15:56 DEBUG { 05:15:56 DEBUG "status": "success", 05:15:56 DEBUG "data": { 05:15:56 DEBUG "resultType": "vector", 05:15:56 DEBUG "result": [ 05:15:56 DEBUG { 05:15:56 DEBUG "metric": { 05:15:56 DEBUG "pod": "ds-idrepo-0" 05:15:56 DEBUG }, 05:15:56 DEBUG "value": [ 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG [...] 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG "pod": "ds-idrepo-2" 05:15:56 DEBUG }, 05:15:56 DEBUG "value": [ 05:15:56 DEBUG 1713500156, 05:15:56 DEBUG "0" 05:15:56 DEBUG ] 05:15:56 DEBUG } 05:15:56 DEBUG ] 05:15:56 DEBUG } 05:15:56 DEBUG } 05:15:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500156" 05:15:56 INFO [http_cmd]: http status code OK 05:15:56 DEBUG --- status code --- 05:15:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:15:56 DEBUG --- http response --- 05:15:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:15:56 DEBUG ----- output ----- 05:15:56 DEBUG { 05:15:56 DEBUG "status": "success", 05:15:56 DEBUG "data": { 05:15:56 DEBUG "resultType": "vector", 05:15:56 DEBUG "result": [ 05:15:56 DEBUG { 05:15:56 DEBUG "metric": { 05:15:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:15:56 DEBUG }, 05:15:56 DEBUG "value": [ 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG [...] 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:15:56 DEBUG }, 05:15:56 DEBUG "value": [ 05:15:56 DEBUG 1713500156, 05:15:56 DEBUG "2103853056" 05:15:56 DEBUG ] 05:15:56 DEBUG } 05:15:56 DEBUG ] 05:15:56 DEBUG } 05:15:56 DEBUG } 05:15:56 INFO [http_cmd]: http status code OK 05:15:56 DEBUG --- status code --- 05:15:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:15:56 DEBUG --- http response --- 05:15:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:15:56 DEBUG ----- output ----- 05:15:56 DEBUG { 05:15:56 DEBUG "status": "success", 05:15:56 DEBUG "data": { 05:15:56 DEBUG "resultType": "vector", 05:15:56 DEBUG "result": [ 05:15:56 DEBUG { 05:15:56 DEBUG "metric": { 05:15:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:15:56 DEBUG }, 05:15:56 DEBUG "value": [ 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG [...] 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:15:56 DEBUG }, 05:15:56 DEBUG "value": [ 05:15:56 DEBUG 1713500156, 05:15:56 DEBUG "81.83333333333333" 05:15:56 DEBUG ] 05:15:56 DEBUG } 05:15:56 DEBUG ] 05:15:56 DEBUG } 05:15:56 DEBUG } 05:15:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500156" 05:15:56 INFO [http_cmd]: http status code OK 05:15:56 DEBUG --- status code --- 05:15:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:15:56 DEBUG --- http response --- 05:15:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:15:56 DEBUG ----- output ----- 05:15:56 DEBUG { 05:15:56 DEBUG "status": "success", 05:15:56 DEBUG "data": { 05:15:56 DEBUG "resultType": "vector", 05:15:56 DEBUG "result": [ 05:15:56 DEBUG { 05:15:56 DEBUG "metric": { 05:15:56 DEBUG "pod": "ds-idrepo-0" 05:15:56 DEBUG }, 05:15:56 DEBUG "value": [ 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG [...] 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG "pod": "ds-idrepo-2" 05:15:56 DEBUG }, 05:15:56 DEBUG "value": [ 05:15:56 DEBUG 1713500156, 05:15:56 DEBUG "0" 05:15:56 DEBUG ] 05:15:56 DEBUG } 05:15:56 DEBUG ] 05:15:56 DEBUG } 05:15:56 DEBUG } 05:15:56 INFO [http_cmd]: http status code OK 05:15:56 DEBUG --- status code --- 05:15:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:15:56 DEBUG --- http response --- 05:15:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:15:56 DEBUG ----- output ----- 05:15:56 DEBUG { 05:15:56 DEBUG "status": "success", 05:15:56 DEBUG "data": { 05:15:56 DEBUG "resultType": "vector", 05:15:56 DEBUG "result": [ 05:15:56 DEBUG { 05:15:56 DEBUG "metric": { 05:15:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:15:56 DEBUG }, 05:15:56 DEBUG "value": [ 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG [...] 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 05:15:56 DEBUG }, 05:15:56 DEBUG "value": [ 05:15:56 DEBUG 1713500156, 05:15:56 DEBUG "0" 05:15:56 DEBUG ] 05:15:56 DEBUG } 05:15:56 DEBUG ] 05:15:56 DEBUG } 05:15:56 DEBUG } 05:15:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500156" 05:15:56 INFO [http_cmd]: http status code OK 05:15:56 DEBUG --- status code --- 05:15:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:15:56 DEBUG --- http response --- 05:15:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:15:56 DEBUG ----- output ----- 05:15:56 DEBUG { 05:15:56 DEBUG "status": "success", 05:15:56 DEBUG "data": { 05:15:56 DEBUG "resultType": "vector", 05:15:56 DEBUG "result": [ 05:15:56 DEBUG { 05:15:56 DEBUG "metric": { 05:15:56 DEBUG "instance": "10.142.0.107:9100" 05:15:56 DEBUG }, 05:15:56 DEBUG "value": [ 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG [...] 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG 05:15:56 DEBUG "instance": "10.142.0.43:9100" 05:15:56 DEBUG }, 05:15:56 DEBUG "value": [ 05:15:56 DEBUG 1713500156, 05:15:56 DEBUG "0.4466666666666654" 05:15:56 DEBUG ] 05:15:56 DEBUG } 05:15:56 DEBUG ] 05:15:56 DEBUG } 05:15:56 DEBUG } 05:15:57 INFO 05:15:57 INFO [loop_until]: kubectl --namespace=xlou top pods 05:15:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:15:57 INFO [loop_until]: OK (rc = 0) 05:15:57 DEBUG --- stdout --- 05:15:57 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 4702m 9139Mi am-6fd554f8f7-hvvkw 3466m 8887Mi am-6fd554f8f7-pmj4b 5610m 8783Mi ds-cts-0 9319m 13256Mi ds-cts-1 10046m 13225Mi ds-cts-2 9767m 13182Mi ds-idrepo-0 1811m 3266Mi ds-idrepo-1 1277m 3199Mi ds-idrepo-2 1198m 3065Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 4m 1215Mi lodemon-6b89d774d5-rf7sk 1m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 2921m 1945Mi 05:15:57 DEBUG --- stderr --- 05:15:57 DEBUG 05:15:57 INFO 05:15:57 INFO [loop_until]: kubectl --namespace=xlou top node 05:15:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:15:57 INFO [loop_until]: OK (rc = 0) 05:15:57 DEBUG --- stdout --- 05:15:57 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 72m 0% 6257Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 2261m 7% 6713Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 5289m 17% 11602Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 86m 0% 3237Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 11754m 39% 15871Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 134m 0% 5173Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 5560m 18% 11920Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 2257m 7% 2727Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 4486m 15% 10990Mi 9% gke-xlou-cdm-ds-0f601751-j488 2094m 7% 6164Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 9832m 32% 17205Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 2016m 6% 6874Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 9471m 31% 15134Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 3306m 20% 5680Mi 9% 05:15:57 DEBUG --- stderr --- 05:15:57 DEBUG 05:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500215" 05:16:55 INFO [http_cmd]: http status code OK 05:16:55 DEBUG --- status code --- 05:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:16:55 DEBUG --- http response --- 05:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:16:55 DEBUG ----- output ----- 05:16:55 DEBUG { 05:16:55 DEBUG "status": "success", 05:16:55 DEBUG "data": { 05:16:55 DEBUG "resultType": "vector", 05:16:55 DEBUG "result": [ 05:16:55 DEBUG { 05:16:55 DEBUG "metric": { 05:16:55 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:16:55 DEBUG }, 05:16:55 DEBUG "value": [ 05:16:55 DEBUG 05:16:55 DEBUG 05:16:55 DEBUG 05:16:55 DEBUG [...] 05:16:55 DEBUG 05:16:55 DEBUG 05:16:55 DEBUG 05:16:55 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:16:55 DEBUG }, 05:16:55 DEBUG "value": [ 05:16:55 DEBUG 1713500215, 05:16:55 DEBUG "3493686" 05:16:55 DEBUG ] 05:16:55 DEBUG } 05:16:55 DEBUG ] 05:16:55 DEBUG } 05:16:55 DEBUG } 05:16:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713500216" 05:16:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500216" 05:16:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713500216" 05:16:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500216" 05:16:56 INFO [http_cmd]: http status code OK 05:16:56 DEBUG --- status code --- 05:16:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:16:56 DEBUG --- http response --- 05:16:56 INFO [http_cmd]: http status code OK 05:16:56 DEBUG --- status code --- 05:16:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:16:56 DEBUG --- http response --- 05:16:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500216" 05:16:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:16:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500216" 05:16:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:16:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713500216" 05:16:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500216" 05:16:56 DEBUG ----- output ----- 05:16:56 DEBUG ----- output ----- 05:16:56 DEBUG { 05:16:56 DEBUG { 05:16:56 DEBUG "status": "success", 05:16:56 DEBUG "status": "success", 05:16:56 DEBUG "data": { 05:16:56 DEBUG "data": { 05:16:56 DEBUG "resultType": "vector", 05:16:56 DEBUG "resultType": "vector", 05:16:56 DEBUG "result": [ 05:16:56 DEBUG "result": [ 05:16:56 DEBUG { 05:16:56 DEBUG { 05:16:56 DEBUG "metric": { 05:16:56 DEBUG "metric": { 05:16:56 DEBUG "pod": "ds-idrepo-0" 05:16:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:16:56 DEBUG }, 05:16:56 DEBUG }, 05:16:56 DEBUG "value": [ 05:16:56 DEBUG "value": [ 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG [...] 05:16:56 DEBUG [...] 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG "pod": "ds-cts-2" 05:16:56 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 05:16:56 DEBUG }, 05:16:56 DEBUG }, 05:16:56 DEBUG "value": [ 05:16:56 DEBUG "value": [ 05:16:56 DEBUG 1713500216, 05:16:56 DEBUG 1713500216, 05:16:56 DEBUG "0" 05:16:56 DEBUG "0" 05:16:56 DEBUG ] 05:16:56 DEBUG ] 05:16:56 DEBUG } 05:16:56 DEBUG } 05:16:56 DEBUG ] 05:16:56 DEBUG ] 05:16:56 DEBUG } 05:16:56 DEBUG } 05:16:56 DEBUG } 05:16:56 DEBUG } 05:16:56 INFO [http_cmd]: http status code OK 05:16:56 ERROR [http_cmd]: ERROR 05:16:56 INFO [http_cmd]: http status code OK 05:16:56 INFO [http_cmd]: http status code OK 05:16:56 DEBUG --- status code --- 05:16:56 ERROR --- status code --- 05:16:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500216" 05:16:56 DEBUG --- status code --- 05:16:56 DEBUG --- status code --- 05:16:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:16:56 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:16:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:16:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:16:56 INFO [http_cmd]: http status code OK 05:16:56 DEBUG --- http response --- 05:16:56 ERROR --- http response --- 05:16:56 INFO [http_cmd]: http status code OK 05:16:56 DEBUG --- http response --- 05:16:56 DEBUG --- http response --- 05:16:56 DEBUG --- status code --- 05:16:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:16:56 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:16:56 DEBUG --- status code --- 05:16:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:16:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:16:56 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:16:56 DEBUG ----- output ----- 05:16:56 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:16:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:16:56 DEBUG ----- output ----- 05:16:56 DEBUG --- http response --- 05:16:56 DEBUG { 05:16:56 DEBUG --- http response --- 05:16:56 DEBUG { 05:16:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:16:56 DEBUG "status": "success", 05:16:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:16:56 DEBUG "status": "success", 05:16:56 DEBUG ----- output ----- 05:16:56 DEBUG "data": { 05:16:56 DEBUG ----- output ----- 05:16:56 DEBUG "data": { 05:16:56 DEBUG { 05:16:56 DEBUG "resultType": "vector", 05:16:56 DEBUG { 05:16:56 DEBUG "resultType": "vector", 05:16:56 DEBUG "status": "success", 05:16:56 DEBUG "result": [ 05:16:56 DEBUG "status": "success", 05:16:56 DEBUG "result": [ 05:16:56 DEBUG "data": { 05:16:56 DEBUG { 05:16:56 DEBUG "data": { 05:16:56 DEBUG { 05:16:56 DEBUG "resultType": "vector", 05:16:56 DEBUG "metric": { 05:16:56 DEBUG "resultType": "vector", 05:16:56 DEBUG "metric": { 05:16:56 DEBUG "result": [ 05:16:56 DEBUG "pod": "ds-cts-0" 05:16:56 DEBUG "result": [ 05:16:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:16:56 DEBUG { 05:16:56 DEBUG }, 05:16:56 DEBUG { 05:16:56 DEBUG }, 05:16:56 DEBUG "metric": { 05:16:56 DEBUG "value": [ 05:16:56 DEBUG "metric": { 05:16:56 DEBUG "value": [ 05:16:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:16:56 DEBUG 05:16:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:16:56 DEBUG 05:16:56 DEBUG }, 05:16:56 DEBUG "value": [ 05:16:56 DEBUG }, 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG "value": [ 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG [...] 05:16:56 DEBUG [...] 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG [...] 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG [...] 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:16:56 DEBUG "pod": "ds-cts-2" 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG }, 05:16:56 DEBUG }, 05:16:56 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 05:16:56 DEBUG 05:16:56 DEBUG "value": [ 05:16:56 DEBUG "value": [ 05:16:56 DEBUG }, 05:16:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:16:56 DEBUG 1713500216, 05:16:56 DEBUG 1713500216, 05:16:56 DEBUG "value": [ 05:16:56 DEBUG }, 05:16:56 DEBUG "0" 05:16:56 DEBUG "0.0006333333333333334" 05:16:56 DEBUG 1713500216, 05:16:56 DEBUG "value": [ 05:16:56 DEBUG ] 05:16:56 DEBUG ] 05:16:56 DEBUG "0.0000294366248943461" 05:16:56 DEBUG 1713500216, 05:16:56 DEBUG } 05:16:56 DEBUG } 05:16:56 DEBUG ] 05:16:56 DEBUG "7.197519528302878" 05:16:56 DEBUG ] 05:16:56 DEBUG ] 05:16:56 DEBUG } 05:16:56 DEBUG ] 05:16:56 DEBUG } 05:16:56 DEBUG } 05:16:56 DEBUG ] 05:16:56 DEBUG } 05:16:56 DEBUG } 05:16:56 DEBUG } 05:16:56 DEBUG } 05:16:56 DEBUG ] 05:16:56 DEBUG } 05:16:56 DEBUG } 05:16:56 DEBUG } 05:16:56 INFO [http_cmd]: http status code OK 05:16:56 DEBUG --- status code --- 05:16:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:16:56 DEBUG --- http response --- 05:16:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:16:56 DEBUG ----- output ----- 05:16:56 DEBUG { 05:16:56 DEBUG "status": "success", 05:16:56 DEBUG "data": { 05:16:56 DEBUG "resultType": "vector", 05:16:56 DEBUG "result": [ 05:16:56 DEBUG { 05:16:56 DEBUG "metric": { 05:16:56 DEBUG "pod": "ds-idrepo-0" 05:16:56 DEBUG }, 05:16:56 DEBUG "value": [ 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG [...] 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG "pod": "ds-idrepo-2" 05:16:56 DEBUG }, 05:16:56 DEBUG "value": [ 05:16:56 DEBUG 1713500216, 05:16:56 DEBUG "0" 05:16:56 DEBUG ] 05:16:56 DEBUG } 05:16:56 DEBUG ] 05:16:56 DEBUG } 05:16:56 DEBUG } 05:16:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500216" 05:16:56 INFO [http_cmd]: http status code OK 05:16:56 DEBUG --- status code --- 05:16:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:16:56 DEBUG --- http response --- 05:16:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:16:56 DEBUG ----- output ----- 05:16:56 DEBUG { 05:16:56 DEBUG "status": "success", 05:16:56 DEBUG "data": { 05:16:56 DEBUG "resultType": "vector", 05:16:56 DEBUG "result": [ 05:16:56 DEBUG { 05:16:56 DEBUG "metric": { 05:16:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:16:56 DEBUG }, 05:16:56 DEBUG "value": [ 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG [...] 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:16:56 DEBUG }, 05:16:56 DEBUG "value": [ 05:16:56 DEBUG 1713500216, 05:16:56 DEBUG "2168.133333333333" 05:16:56 DEBUG ] 05:16:56 DEBUG } 05:16:56 DEBUG ] 05:16:56 DEBUG } 05:16:56 DEBUG } 05:16:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500216" 05:16:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500216" 05:16:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500216" 05:16:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500216" 05:16:56 INFO [http_cmd]: http status code OK 05:16:56 INFO [http_cmd]: http status code OK 05:16:56 DEBUG --- status code --- 05:16:56 DEBUG --- status code --- 05:16:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:16:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:16:56 DEBUG --- http response --- 05:16:56 DEBUG --- http response --- 05:16:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:16:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:16:56 DEBUG ----- output ----- 05:16:56 DEBUG ----- output ----- 05:16:56 DEBUG { 05:16:56 DEBUG { 05:16:56 DEBUG "status": "success", 05:16:56 DEBUG "status": "success", 05:16:56 DEBUG "data": { 05:16:56 DEBUG "data": { 05:16:56 DEBUG "resultType": "vector", 05:16:56 DEBUG "resultType": "vector", 05:16:56 DEBUG "result": [ 05:16:56 DEBUG "result": [ 05:16:56 DEBUG { 05:16:56 DEBUG { 05:16:56 DEBUG "metric": { 05:16:56 DEBUG "metric": { 05:16:56 DEBUG "instance": "10.142.0.107:9100" 05:16:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:16:56 DEBUG }, 05:16:56 DEBUG }, 05:16:56 DEBUG "value": [ 05:16:56 DEBUG "value": [ 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG [...] 05:16:56 DEBUG [...] 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG "instance": "10.142.0.43:9100" 05:16:56 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 05:16:56 DEBUG }, 05:16:56 DEBUG }, 05:16:56 DEBUG "value": [ 05:16:56 DEBUG "value": [ 05:16:56 DEBUG 1713500216, 05:16:56 DEBUG 1713500216, 05:16:56 DEBUG "0.686333333333341" 05:16:56 DEBUG "0" 05:16:56 DEBUG ] 05:16:56 DEBUG ] 05:16:56 DEBUG } 05:16:56 DEBUG } 05:16:56 DEBUG ] 05:16:56 DEBUG ] 05:16:56 DEBUG } 05:16:56 INFO [http_cmd]: http status code OK 05:16:56 DEBUG } 05:16:56 DEBUG } 05:16:56 DEBUG --- status code --- 05:16:56 DEBUG } 05:16:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:16:56 DEBUG --- http response --- 05:16:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500216" 05:16:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500216" 05:16:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:16:56 DEBUG ----- output ----- 05:16:56 DEBUG { 05:16:56 DEBUG "status": "success", 05:16:56 DEBUG "data": { 05:16:56 DEBUG "resultType": "vector", 05:16:56 DEBUG "result": [ 05:16:56 DEBUG { 05:16:56 DEBUG "metric": { 05:16:56 DEBUG "pod": "ds-cts-0" 05:16:56 DEBUG }, 05:16:56 DEBUG "value": [ 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG [...] 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG "pod": "ds-cts-2" 05:16:56 DEBUG }, 05:16:56 DEBUG "value": [ 05:16:56 DEBUG 1713500216, 05:16:56 DEBUG "0" 05:16:56 DEBUG ] 05:16:56 DEBUG } 05:16:56 DEBUG ] 05:16:56 DEBUG } 05:16:56 DEBUG } 05:16:56 INFO [http_cmd]: http status code OK 05:16:56 DEBUG --- status code --- 05:16:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:16:56 DEBUG --- http response --- 05:16:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:16:56 DEBUG ----- output ----- 05:16:56 DEBUG { 05:16:56 DEBUG "status": "success", 05:16:56 DEBUG "data": { 05:16:56 DEBUG "resultType": "vector", 05:16:56 DEBUG "result": [ 05:16:56 DEBUG { 05:16:56 DEBUG "metric": { 05:16:56 DEBUG "pod": "ds-cts-0" 05:16:56 DEBUG }, 05:16:56 DEBUG "value": [ 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG [...] 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG "pod": "ds-cts-2" 05:16:56 DEBUG }, 05:16:56 DEBUG "value": [ 05:16:56 DEBUG 1713500216, 05:16:56 DEBUG "465.26666666666665" 05:16:56 DEBUG ] 05:16:56 DEBUG } 05:16:56 DEBUG ] 05:16:56 DEBUG } 05:16:56 DEBUG } 05:16:56 INFO [http_cmd]: http status code OK 05:16:56 INFO [http_cmd]: http status code OK 05:16:56 DEBUG --- status code --- 05:16:56 DEBUG --- status code --- 05:16:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:16:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:16:56 DEBUG --- http response --- 05:16:56 DEBUG --- http response --- 05:16:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:16:56 DEBUG ----- output ----- 05:16:56 DEBUG { 05:16:56 DEBUG "status": "success", 05:16:56 DEBUG "data": { 05:16:56 DEBUG "resultType": "vector", 05:16:56 DEBUG "result": [ 05:16:56 DEBUG { 05:16:56 DEBUG "metric": { 05:16:56 DEBUG "pod": "ds-idrepo-0" 05:16:56 DEBUG }, 05:16:56 DEBUG "value": [ 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG [...] 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG "pod": "ds-idrepo-2" 05:16:56 DEBUG }, 05:16:56 DEBUG "value": [ 05:16:56 DEBUG 1713500216, 05:16:56 DEBUG "0" 05:16:56 DEBUG ] 05:16:56 DEBUG } 05:16:56 DEBUG ] 05:16:56 DEBUG } 05:16:56 DEBUG } 05:16:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:16:56 DEBUG ----- output ----- 05:16:56 DEBUG { 05:16:56 DEBUG "status": "success", 05:16:56 DEBUG "data": { 05:16:56 DEBUG "resultType": "vector", 05:16:56 DEBUG "result": [ 05:16:56 DEBUG { 05:16:56 DEBUG "metric": { 05:16:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:16:56 DEBUG }, 05:16:56 DEBUG "value": [ 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG [...] 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:16:56 DEBUG }, 05:16:56 DEBUG "value": [ 05:16:56 DEBUG 1713500216, 05:16:56 DEBUG "2168.3333333333335" 05:16:56 DEBUG ] 05:16:56 DEBUG } 05:16:56 DEBUG ] 05:16:56 DEBUG } 05:16:56 DEBUG } 05:16:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713500216" 05:16:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713500216" 05:16:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500216" 05:16:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500216" 05:16:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500216" 05:16:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500216" 05:16:56 INFO [http_cmd]: http status code OK 05:16:56 DEBUG --- status code --- 05:16:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:16:56 DEBUG --- http response --- 05:16:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:16:56 DEBUG ----- output ----- 05:16:56 DEBUG { 05:16:56 DEBUG "status": "success", 05:16:56 DEBUG "data": { 05:16:56 DEBUG "resultType": "vector", 05:16:56 DEBUG "result": [ 05:16:56 DEBUG { 05:16:56 DEBUG "metric": { 05:16:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:16:56 DEBUG }, 05:16:56 DEBUG "value": [ 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG [...] 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:16:56 DEBUG }, 05:16:56 DEBUG "value": [ 05:16:56 DEBUG 1713500216, 05:16:56 DEBUG "55.172" 05:16:56 DEBUG ] 05:16:56 DEBUG } 05:16:56 DEBUG ] 05:16:56 DEBUG } 05:16:56 DEBUG } 05:16:56 INFO [http_cmd]: http status code OK 05:16:56 DEBUG --- status code --- 05:16:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:16:56 DEBUG --- http response --- 05:16:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:16:56 DEBUG ----- output ----- 05:16:56 DEBUG { 05:16:56 DEBUG "status": "success", 05:16:56 DEBUG "data": { 05:16:56 DEBUG "resultType": "vector", 05:16:56 DEBUG "result": [ 05:16:56 DEBUG { 05:16:56 DEBUG "metric": { 05:16:56 DEBUG "instance": "10.142.0.107:9100" 05:16:56 DEBUG }, 05:16:56 DEBUG "value": [ 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG [...] 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG "instance": "10.142.0.43:9100" 05:16:56 DEBUG }, 05:16:56 DEBUG "value": [ 05:16:56 DEBUG 1713500216, 05:16:56 DEBUG "459955704832" 05:16:56 DEBUG ] 05:16:56 DEBUG } 05:16:56 DEBUG ] 05:16:56 DEBUG } 05:16:56 DEBUG } 05:16:56 INFO [http_cmd]: http status code OK 05:16:56 DEBUG --- status code --- 05:16:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:16:56 DEBUG --- http response --- 05:16:56 INFO [http_cmd]: http status code OK 05:16:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:16:56 DEBUG --- status code --- 05:16:56 DEBUG ----- output ----- 05:16:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:16:56 DEBUG { 05:16:56 DEBUG --- http response --- 05:16:56 INFO [http_cmd]: http status code OK 05:16:56 DEBUG "status": "success", 05:16:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:16:56 DEBUG --- status code --- 05:16:56 DEBUG "data": { 05:16:56 DEBUG ----- output ----- 05:16:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:16:56 DEBUG "resultType": "vector", 05:16:56 DEBUG { 05:16:56 DEBUG --- http response --- 05:16:56 DEBUG "result": [ 05:16:56 DEBUG "status": "success", 05:16:56 DEBUG { 05:16:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:16:56 DEBUG "data": { 05:16:56 DEBUG "metric": { 05:16:56 DEBUG ----- output ----- 05:16:56 DEBUG "resultType": "vector", 05:16:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:16:56 DEBUG { 05:16:56 DEBUG "result": [ 05:16:56 DEBUG }, 05:16:56 DEBUG "status": "success", 05:16:56 DEBUG { 05:16:56 DEBUG "value": [ 05:16:56 DEBUG "data": { 05:16:56 DEBUG "metric": { 05:16:56 DEBUG 05:16:56 DEBUG "resultType": "vector", 05:16:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:16:56 DEBUG 05:16:56 DEBUG "result": [ 05:16:56 DEBUG }, 05:16:56 DEBUG 05:16:56 DEBUG { 05:16:56 DEBUG "value": [ 05:16:56 DEBUG [...] 05:16:56 DEBUG "metric": { 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG "instance": "10.142.0.107:9100" 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG }, 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG "value": [ 05:16:56 DEBUG [...] 05:16:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG }, 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG "value": [ 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 1713500216, 05:16:56 DEBUG [...] 05:16:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:16:56 DEBUG "2174210048" 05:16:56 DEBUG 05:16:56 DEBUG }, 05:16:56 DEBUG ] 05:16:56 DEBUG 05:16:56 DEBUG "value": [ 05:16:56 DEBUG } 05:16:56 DEBUG 05:16:56 DEBUG 1713500216, 05:16:56 DEBUG ] 05:16:56 DEBUG "instance": "10.142.0.43:9100" 05:16:56 DEBUG "1997.5" 05:16:56 DEBUG } 05:16:56 DEBUG }, 05:16:56 DEBUG ] 05:16:56 DEBUG } 05:16:56 DEBUG "value": [ 05:16:56 DEBUG } 05:16:56 DEBUG 1713500216, 05:16:56 DEBUG ] 05:16:56 DEBUG "64000731.36666666" 05:16:56 DEBUG } 05:16:56 DEBUG ] 05:16:56 DEBUG } 05:16:56 INFO [http_cmd]: http status code OK 05:16:56 DEBUG } 05:16:56 DEBUG --- status code --- 05:16:56 DEBUG ] 05:16:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:16:56 DEBUG } 05:16:56 DEBUG --- http response --- 05:16:56 DEBUG } 05:16:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:16:56 DEBUG ----- output ----- 05:16:56 DEBUG { 05:16:56 DEBUG "status": "success", 05:16:56 DEBUG "data": { 05:16:56 DEBUG "resultType": "vector", 05:16:56 DEBUG "result": [ 05:16:56 DEBUG { 05:16:56 DEBUG "metric": { 05:16:56 DEBUG "instance": "10.142.0.107:9100" 05:16:56 DEBUG }, 05:16:56 DEBUG "value": [ 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG [...] 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG "instance": "10.142.0.43:9100" 05:16:56 DEBUG }, 05:16:56 DEBUG "value": [ 05:16:56 DEBUG 1713500216, 05:16:56 DEBUG "64002366.36666667" 05:16:56 DEBUG ] 05:16:56 DEBUG } 05:16:56 DEBUG ] 05:16:56 DEBUG } 05:16:56 DEBUG } 05:16:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500216" 05:16:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500216" 05:16:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500216" 05:16:56 INFO [http_cmd]: http status code OK 05:16:56 DEBUG --- status code --- 05:16:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:16:56 DEBUG --- http response --- 05:16:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:16:56 DEBUG ----- output ----- 05:16:56 DEBUG { 05:16:56 DEBUG "status": "success", 05:16:56 DEBUG "data": { 05:16:56 DEBUG "resultType": "vector", 05:16:56 DEBUG "result": [ 05:16:56 DEBUG { 05:16:56 DEBUG "metric": { 05:16:56 DEBUG "pod": "ds-idrepo-0" 05:16:56 DEBUG }, 05:16:56 DEBUG "value": [ 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG [...] 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG "pod": "ds-idrepo-2" 05:16:56 DEBUG }, 05:16:56 DEBUG "value": [ 05:16:56 DEBUG 1713500216, 05:16:56 DEBUG "0" 05:16:56 DEBUG ] 05:16:56 DEBUG } 05:16:56 DEBUG ] 05:16:56 DEBUG } 05:16:56 DEBUG } 05:16:56 INFO [http_cmd]: http status code OK 05:16:56 INFO [http_cmd]: http status code OK 05:16:56 DEBUG --- status code --- 05:16:56 DEBUG --- status code --- 05:16:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:16:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:16:56 DEBUG --- http response --- 05:16:56 DEBUG --- http response --- 05:16:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:16:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:16:56 DEBUG ----- output ----- 05:16:56 DEBUG ----- output ----- 05:16:56 DEBUG { 05:16:56 DEBUG { 05:16:56 DEBUG "status": "success", 05:16:56 DEBUG "status": "success", 05:16:56 DEBUG "data": { 05:16:56 DEBUG "data": { 05:16:56 DEBUG "resultType": "vector", 05:16:56 DEBUG "resultType": "vector", 05:16:56 DEBUG "result": [ 05:16:56 DEBUG "result": [ 05:16:56 DEBUG { 05:16:56 DEBUG { 05:16:56 DEBUG "metric": { 05:16:56 DEBUG "metric": { 05:16:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:16:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:16:56 DEBUG }, 05:16:56 DEBUG }, 05:16:56 DEBUG "value": [ 05:16:56 DEBUG "value": [ 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG [...] 05:16:56 DEBUG [...] 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 05:16:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:16:56 DEBUG }, 05:16:56 DEBUG }, 05:16:56 DEBUG "value": [ 05:16:56 DEBUG "value": [ 05:16:56 DEBUG 1713500216, 05:16:56 DEBUG 1713500216, 05:16:56 DEBUG "0" 05:16:56 DEBUG "2168.5333333333333" 05:16:56 DEBUG ] 05:16:56 DEBUG ] 05:16:56 DEBUG } 05:16:56 DEBUG } 05:16:56 DEBUG ] 05:16:56 DEBUG ] 05:16:56 DEBUG } 05:16:56 DEBUG } 05:16:56 DEBUG } 05:16:56 DEBUG } 05:16:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500216" 05:16:56 INFO [http_cmd]: http status code OK 05:16:56 DEBUG --- status code --- 05:16:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:16:56 DEBUG --- http response --- 05:16:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:16:56 DEBUG ----- output ----- 05:16:56 DEBUG { 05:16:56 DEBUG "status": "success", 05:16:56 DEBUG "data": { 05:16:56 DEBUG "resultType": "vector", 05:16:56 DEBUG "result": [ 05:16:56 DEBUG { 05:16:56 DEBUG "metric": { 05:16:56 DEBUG "instance": "10.142.0.107:9100" 05:16:56 DEBUG }, 05:16:56 DEBUG "value": [ 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG [...] 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG 05:16:56 DEBUG "instance": "10.142.0.43:9100" 05:16:56 DEBUG }, 05:16:56 DEBUG "value": [ 05:16:56 DEBUG 1713500216, 05:16:56 DEBUG "0.3726666666666697" 05:16:56 DEBUG ] 05:16:56 DEBUG } 05:16:56 DEBUG ] 05:16:56 DEBUG } 05:16:56 DEBUG } 05:16:57 INFO 05:16:57 INFO [loop_until]: kubectl --namespace=xlou top pods 05:16:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:16:57 INFO [loop_until]: OK (rc = 0) 05:16:57 DEBUG --- stdout --- 05:16:57 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 13671m 9140Mi am-6fd554f8f7-hvvkw 12906m 8890Mi am-6fd554f8f7-pmj4b 14422m 8785Mi ds-cts-0 18215m 12714Mi ds-cts-1 17088m 12615Mi ds-cts-2 18562m 12551Mi ds-idrepo-0 5726m 3267Mi ds-idrepo-1 5812m 3198Mi ds-idrepo-2 5462m 3066Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 4m 1215Mi lodemon-6b89d774d5-rf7sk 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7122m 1980Mi 05:16:57 DEBUG --- stderr --- 05:16:57 DEBUG 05:16:57 INFO 05:16:57 INFO [loop_until]: kubectl --namespace=xlou top node 05:16:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:16:57 INFO [loop_until]: OK (rc = 0) 05:16:57 DEBUG --- stdout --- 05:16:57 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 70m 0% 6253Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5789m 19% 6726Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13098m 43% 11601Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 78m 0% 3256Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 18091m 60% 15293Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 154m 0% 5162Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 13952m 46% 11927Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 11280m 37% 2730Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 15157m 50% 10981Mi 9% gke-xlou-cdm-ds-0f601751-j488 6138m 20% 6166Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 17539m 58% 16622Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 6357m 21% 6876Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 18652m 62% 14527Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 7340m 46% 5715Mi 9% 05:16:57 DEBUG --- stderr --- 05:16:57 DEBUG 05:17:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500276" 05:17:56 INFO [http_cmd]: http status code OK 05:17:56 DEBUG --- status code --- 05:17:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:17:56 DEBUG --- http response --- 05:17:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:17:56 DEBUG ----- output ----- 05:17:56 DEBUG { 05:17:56 DEBUG "status": "success", 05:17:56 DEBUG "data": { 05:17:56 DEBUG "resultType": "vector", 05:17:56 DEBUG "result": [ 05:17:56 DEBUG { 05:17:56 DEBUG "metric": { 05:17:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:17:56 DEBUG }, 05:17:56 DEBUG "value": [ 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG [...] 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:17:56 DEBUG }, 05:17:56 DEBUG "value": [ 05:17:56 DEBUG 1713500276, 05:17:56 DEBUG "3504325" 05:17:56 DEBUG ] 05:17:56 DEBUG } 05:17:56 DEBUG ] 05:17:56 DEBUG } 05:17:56 DEBUG } 05:17:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500276" 05:17:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500276" 05:17:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713500276" 05:17:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500276" 05:17:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500276" 05:17:56 INFO [http_cmd]: http status code OK 05:17:56 INFO [http_cmd]: http status code OK 05:17:56 DEBUG --- status code --- 05:17:56 DEBUG --- status code --- 05:17:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:17:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:17:56 DEBUG --- http response --- 05:17:56 DEBUG --- http response --- 05:17:56 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:17:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:17:56 DEBUG ----- output ----- 05:17:56 DEBUG { 05:17:56 DEBUG "status": "success", 05:17:56 DEBUG "data": { 05:17:56 DEBUG "resultType": "vector", 05:17:56 DEBUG "result": [ 05:17:56 DEBUG { 05:17:56 DEBUG "metric": { 05:17:56 DEBUG "pod": "ds-cts-0" 05:17:56 DEBUG }, 05:17:56 DEBUG "value": [ 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG [...] 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG "pod": "ds-cts-2" 05:17:56 DEBUG }, 05:17:56 DEBUG "value": [ 05:17:56 DEBUG 1713500276, 05:17:56 DEBUG "0.00099566" 05:17:56 DEBUG ] 05:17:56 DEBUG } 05:17:56 DEBUG ] 05:17:56 DEBUG } 05:17:56 DEBUG } 05:17:56 INFO [http_cmd]: http status code OK 05:17:56 DEBUG --- status code --- 05:17:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:17:56 DEBUG --- http response --- 05:17:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:17:56 DEBUG ----- output ----- 05:17:56 DEBUG { 05:17:56 DEBUG "status": "success", 05:17:56 DEBUG "data": { 05:17:56 DEBUG "resultType": "vector", 05:17:56 DEBUG "result": [ 05:17:56 DEBUG { 05:17:56 DEBUG "metric": { 05:17:56 DEBUG "pod": "ds-idrepo-0" 05:17:56 DEBUG }, 05:17:56 DEBUG "value": [ 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG [...] 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG "pod": "ds-cts-2" 05:17:56 DEBUG }, 05:17:56 DEBUG "value": [ 05:17:56 DEBUG 1713500276, 05:17:56 DEBUG "0" 05:17:56 DEBUG ] 05:17:56 DEBUG } 05:17:56 DEBUG ] 05:17:56 DEBUG } 05:17:56 DEBUG } 05:17:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713500276" 05:17:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500276" 05:17:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713500276" 05:17:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500276" 05:17:56 INFO [http_cmd]: http status code OK 05:17:56 DEBUG --- status code --- 05:17:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:17:56 DEBUG --- http response --- 05:17:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:17:56 DEBUG ----- output ----- 05:17:56 DEBUG { 05:17:56 DEBUG "status": "success", 05:17:56 DEBUG "data": { 05:17:56 DEBUG "resultType": "vector", 05:17:56 DEBUG "result": [ 05:17:56 DEBUG { 05:17:56 DEBUG "metric": { 05:17:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:17:56 DEBUG }, 05:17:56 DEBUG "value": [ 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG [...] 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG "pod": "ds-cts-0" 05:17:56 DEBUG }, 05:17:56 DEBUG "value": [ 05:17:56 DEBUG 1713500276, 05:17:56 DEBUG "0" 05:17:56 DEBUG ] 05:17:56 DEBUG } 05:17:56 DEBUG ] 05:17:56 DEBUG } 05:17:56 DEBUG } 05:17:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500276" 05:17:56 ERROR [http_cmd]: ERROR 05:17:56 INFO [http_cmd]: http status code OK 05:17:56 ERROR --- status code --- 05:17:56 DEBUG --- status code --- 05:17:56 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:17:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:17:56 ERROR --- http response --- 05:17:56 DEBUG --- http response --- 05:17:56 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:17:56 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:17:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:17:56 DEBUG ----- output ----- 05:17:56 DEBUG { 05:17:56 DEBUG "status": "success", 05:17:56 DEBUG "data": { 05:17:56 DEBUG "resultType": "vector", 05:17:56 DEBUG "result": [ 05:17:56 DEBUG { 05:17:56 DEBUG "metric": { 05:17:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:17:56 DEBUG }, 05:17:56 DEBUG "value": [ 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG [...] 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:17:56 DEBUG }, 05:17:56 DEBUG "value": [ 05:17:56 DEBUG 1713500276, 05:17:56 DEBUG "2167.7" 05:17:56 DEBUG ] 05:17:56 DEBUG } 05:17:56 DEBUG ] 05:17:56 DEBUG } 05:17:56 DEBUG } 05:17:56 INFO [http_cmd]: http status code OK 05:17:56 DEBUG --- status code --- 05:17:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:17:56 DEBUG --- http response --- 05:17:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:17:56 DEBUG ----- output ----- 05:17:56 DEBUG { 05:17:56 DEBUG "status": "success", 05:17:56 DEBUG "data": { 05:17:56 DEBUG "resultType": "vector", 05:17:56 DEBUG "result": [ 05:17:56 DEBUG { 05:17:56 DEBUG "metric": { 05:17:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:17:56 DEBUG }, 05:17:56 DEBUG "value": [ 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG [...] 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG "pod": "ds-cts-0" 05:17:56 DEBUG }, 05:17:56 DEBUG "value": [ 05:17:56 DEBUG 1713500276, 05:17:56 DEBUG "0" 05:17:56 DEBUG ] 05:17:56 DEBUG } 05:17:56 DEBUG ] 05:17:56 DEBUG } 05:17:56 DEBUG } 05:17:56 INFO [http_cmd]: http status code OK 05:17:56 DEBUG --- status code --- 05:17:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:17:56 DEBUG --- http response --- 05:17:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:17:56 DEBUG ----- output ----- 05:17:56 DEBUG { 05:17:56 DEBUG "status": "success", 05:17:56 DEBUG "data": { 05:17:56 DEBUG "resultType": "vector", 05:17:56 DEBUG "result": [ 05:17:56 DEBUG { 05:17:56 DEBUG "metric": { 05:17:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:17:56 DEBUG }, 05:17:56 DEBUG "value": [ 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG [...] 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:17:56 DEBUG }, 05:17:56 DEBUG "value": [ 05:17:56 DEBUG 1713500276, 05:17:56 DEBUG "0" 05:17:56 DEBUG ] 05:17:56 DEBUG } 05:17:56 DEBUG ] 05:17:56 DEBUG } 05:17:56 DEBUG } 05:17:56 INFO [http_cmd]: http status code OK 05:17:56 DEBUG --- status code --- 05:17:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:17:56 DEBUG --- http response --- 05:17:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:17:56 DEBUG ----- output ----- 05:17:56 DEBUG { 05:17:56 DEBUG "status": "success", 05:17:56 DEBUG "data": { 05:17:56 DEBUG "resultType": "vector", 05:17:56 DEBUG "result": [ 05:17:56 DEBUG { 05:17:56 DEBUG "metric": { 05:17:56 DEBUG "pod": "ds-idrepo-0" 05:17:56 DEBUG }, 05:17:56 DEBUG "value": [ 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG [...] 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG "pod": "ds-idrepo-2" 05:17:56 DEBUG }, 05:17:56 DEBUG "value": [ 05:17:56 DEBUG 1713500276, 05:17:56 DEBUG "0" 05:17:56 DEBUG ] 05:17:56 DEBUG } 05:17:56 DEBUG ] 05:17:56 DEBUG } 05:17:56 DEBUG } 05:17:56 INFO [http_cmd]: http status code OK 05:17:56 DEBUG --- status code --- 05:17:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:17:56 DEBUG --- http response --- 05:17:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:17:56 DEBUG ----- output ----- 05:17:56 DEBUG { 05:17:56 DEBUG "status": "success", 05:17:56 DEBUG "data": { 05:17:56 DEBUG "resultType": "vector", 05:17:56 DEBUG "result": [ 05:17:56 DEBUG { 05:17:56 DEBUG "metric": { 05:17:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:17:56 DEBUG }, 05:17:56 DEBUG "value": [ 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG [...] 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:17:56 DEBUG }, 05:17:56 DEBUG "value": [ 05:17:56 DEBUG 1713500276, 05:17:56 DEBUG "7.130999689152463" 05:17:56 DEBUG ] 05:17:56 DEBUG } 05:17:56 DEBUG ] 05:17:56 DEBUG } 05:17:56 DEBUG } 05:17:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500276" 05:17:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500276" 05:17:56 INFO [http_cmd]: http status code OK 05:17:56 INFO [http_cmd]: http status code OK 05:17:56 DEBUG --- status code --- 05:17:56 DEBUG --- status code --- 05:17:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:17:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:17:56 DEBUG --- http response --- 05:17:56 DEBUG --- http response --- 05:17:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:17:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 05:17:56 DEBUG ----- output ----- 05:17:56 DEBUG ----- output ----- 05:17:56 DEBUG { 05:17:56 DEBUG { 05:17:56 DEBUG "status": "success", 05:17:56 DEBUG "status": "success", 05:17:56 DEBUG "data": { 05:17:56 DEBUG "data": { 05:17:56 DEBUG "resultType": "vector", 05:17:56 DEBUG "resultType": "vector", 05:17:56 DEBUG "result": [ 05:17:56 DEBUG "result": [ 05:17:56 DEBUG { 05:17:56 DEBUG { 05:17:56 DEBUG "metric": { 05:17:56 DEBUG "metric": { 05:17:56 DEBUG "instance": "10.142.0.107:9100" 05:17:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:17:56 DEBUG }, 05:17:56 DEBUG }, 05:17:56 DEBUG "value": [ 05:17:56 DEBUG "value": [ 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG [...] 05:17:56 DEBUG [...] 05:17:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500276" 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG "instance": "10.142.0.43:9100" 05:17:56 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 05:17:56 DEBUG }, 05:17:56 DEBUG }, 05:17:56 DEBUG "value": [ 05:17:56 DEBUG "value": [ 05:17:56 DEBUG 1713500276, 05:17:56 DEBUG 1713500276, 05:17:56 DEBUG "463061349376" 05:17:56 DEBUG "0" 05:17:56 DEBUG ] 05:17:56 DEBUG ] 05:17:56 DEBUG } 05:17:56 DEBUG } 05:17:56 DEBUG ] 05:17:56 DEBUG ] 05:17:56 DEBUG } 05:17:56 DEBUG } 05:17:56 DEBUG } 05:17:56 DEBUG } 05:17:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500276" 05:17:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500276" 05:17:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713500276" 05:17:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500276" 05:17:56 INFO [http_cmd]: http status code OK 05:17:56 DEBUG --- status code --- 05:17:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:17:56 DEBUG --- http response --- 05:17:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:17:56 DEBUG ----- output ----- 05:17:56 DEBUG { 05:17:56 DEBUG "status": "success", 05:17:56 DEBUG "data": { 05:17:56 DEBUG "resultType": "vector", 05:17:56 DEBUG "result": [ 05:17:56 DEBUG { 05:17:56 DEBUG "metric": { 05:17:56 DEBUG "instance": "10.142.0.107:9100" 05:17:56 DEBUG }, 05:17:56 DEBUG "value": [ 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG [...] 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG "instance": "10.142.0.43:9100" 05:17:56 DEBUG }, 05:17:56 DEBUG "value": [ 05:17:56 DEBUG 1713500276, 05:17:56 DEBUG "64156828.93333333" 05:17:56 DEBUG ] 05:17:56 DEBUG } 05:17:56 DEBUG ] 05:17:56 DEBUG } 05:17:56 DEBUG } 05:17:56 INFO [http_cmd]: http status code OK 05:17:56 DEBUG --- status code --- 05:17:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:17:56 DEBUG --- http response --- 05:17:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:17:56 DEBUG ----- output ----- 05:17:56 DEBUG { 05:17:56 INFO [http_cmd]: http status code OK 05:17:56 DEBUG "status": "success", 05:17:56 DEBUG --- status code --- 05:17:56 DEBUG "data": { 05:17:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:17:56 DEBUG "resultType": "vector", 05:17:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500276" 05:17:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500276" 05:17:56 DEBUG --- http response --- 05:17:56 DEBUG "result": [ 05:17:56 DEBUG { 05:17:56 DEBUG "metric": { 05:17:56 DEBUG "pod": "ds-cts-0" 05:17:56 DEBUG }, 05:17:56 DEBUG "value": [ 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG [...] 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG "pod": "ds-cts-2" 05:17:56 DEBUG }, 05:17:56 DEBUG "value": [ 05:17:56 DEBUG 1713500276, 05:17:56 DEBUG "0" 05:17:56 DEBUG ] 05:17:56 DEBUG } 05:17:56 DEBUG ] 05:17:56 DEBUG } 05:17:56 DEBUG } 05:17:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:17:56 INFO [http_cmd]: http status code OK 05:17:56 DEBUG ----- output ----- 05:17:56 DEBUG --- status code --- 05:17:56 DEBUG { 05:17:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:17:56 DEBUG "status": "success", 05:17:56 DEBUG --- http response --- 05:17:56 DEBUG "data": { 05:17:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:17:56 DEBUG "resultType": "vector", 05:17:56 DEBUG ----- output ----- 05:17:56 DEBUG "result": [ 05:17:56 DEBUG { 05:17:56 DEBUG { 05:17:56 DEBUG "status": "success", 05:17:56 DEBUG "metric": { 05:17:56 DEBUG "data": { 05:17:56 DEBUG "instance": "10.142.0.107:9100" 05:17:56 DEBUG "resultType": "vector", 05:17:56 DEBUG }, 05:17:56 DEBUG "result": [ 05:17:56 DEBUG "value": [ 05:17:56 DEBUG { 05:17:56 DEBUG 05:17:56 DEBUG "metric": { 05:17:56 DEBUG 05:17:56 INFO [http_cmd]: http status code OK 05:17:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:17:56 DEBUG 05:17:56 DEBUG --- status code --- 05:17:56 DEBUG }, 05:17:56 DEBUG [...] 05:17:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:17:56 DEBUG "value": [ 05:17:56 DEBUG 05:17:56 DEBUG --- http response --- 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG ----- output ----- 05:17:56 DEBUG "instance": "10.142.0.43:9100" 05:17:56 DEBUG [...] 05:17:56 DEBUG { 05:17:56 DEBUG }, 05:17:56 DEBUG 05:17:56 DEBUG "status": "success", 05:17:56 DEBUG "value": [ 05:17:56 DEBUG 05:17:56 DEBUG "data": { 05:17:56 DEBUG 1713500276, 05:17:56 DEBUG 05:17:56 DEBUG "resultType": "vector", 05:17:56 DEBUG "0.656433333333328" 05:17:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:17:56 DEBUG "result": [ 05:17:56 DEBUG ] 05:17:56 DEBUG }, 05:17:56 DEBUG { 05:17:56 DEBUG } 05:17:56 DEBUG "value": [ 05:17:56 DEBUG "metric": { 05:17:56 DEBUG ] 05:17:56 DEBUG 1713500276, 05:17:56 DEBUG "pod": "ds-cts-0" 05:17:56 DEBUG } 05:17:56 DEBUG "2202701824" 05:17:56 DEBUG }, 05:17:56 DEBUG } 05:17:56 DEBUG ] 05:17:56 DEBUG "value": [ 05:17:56 DEBUG } 05:17:56 DEBUG 05:17:56 DEBUG ] 05:17:56 DEBUG 05:17:56 DEBUG } 05:17:56 DEBUG 05:17:56 DEBUG } 05:17:56 DEBUG [...] 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG "pod": "ds-cts-2" 05:17:56 DEBUG }, 05:17:56 DEBUG "value": [ 05:17:56 DEBUG 1713500276, 05:17:56 DEBUG "471.8666666666667" 05:17:56 DEBUG ] 05:17:56 DEBUG } 05:17:56 DEBUG ] 05:17:56 DEBUG } 05:17:56 DEBUG } 05:17:56 INFO [http_cmd]: http status code OK 05:17:56 INFO [http_cmd]: http status code OK 05:17:56 DEBUG --- status code --- 05:17:56 DEBUG --- status code --- 05:17:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:17:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:17:56 DEBUG --- http response --- 05:17:56 DEBUG --- http response --- 05:17:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:17:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:17:56 DEBUG ----- output ----- 05:17:56 DEBUG ----- output ----- 05:17:56 DEBUG { 05:17:56 DEBUG { 05:17:56 DEBUG "status": "success", 05:17:56 DEBUG "status": "success", 05:17:56 DEBUG "data": { 05:17:56 DEBUG "data": { 05:17:56 DEBUG "resultType": "vector", 05:17:56 DEBUG "resultType": "vector", 05:17:56 DEBUG "result": [ 05:17:56 DEBUG "result": [ 05:17:56 DEBUG { 05:17:56 DEBUG { 05:17:56 DEBUG "metric": { 05:17:56 DEBUG "metric": { 05:17:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:17:56 DEBUG "pod": "ds-idrepo-0" 05:17:56 DEBUG }, 05:17:56 DEBUG }, 05:17:56 DEBUG "value": [ 05:17:56 DEBUG "value": [ 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG [...] 05:17:56 DEBUG [...] 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:17:56 DEBUG "pod": "ds-idrepo-2" 05:17:56 DEBUG }, 05:17:56 DEBUG }, 05:17:56 DEBUG "value": [ 05:17:56 DEBUG "value": [ 05:17:56 DEBUG 1713500276, 05:17:56 DEBUG 1713500276, 05:17:56 DEBUG "2168.0333333333333" 05:17:56 DEBUG "0" 05:17:56 DEBUG ] 05:17:56 DEBUG ] 05:17:56 DEBUG } 05:17:56 DEBUG } 05:17:56 DEBUG ] 05:17:56 DEBUG ] 05:17:56 DEBUG } 05:17:56 DEBUG } 05:17:56 DEBUG } 05:17:56 DEBUG } 05:17:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500276" 05:17:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713500276" 05:17:56 INFO [http_cmd]: http status code OK 05:17:56 DEBUG --- status code --- 05:17:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:17:56 DEBUG --- http response --- 05:17:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 05:17:56 DEBUG ----- output ----- 05:17:56 DEBUG { 05:17:56 DEBUG "status": "success", 05:17:56 DEBUG "data": { 05:17:56 DEBUG "resultType": "vector", 05:17:56 DEBUG "result": [ 05:17:56 DEBUG { 05:17:56 DEBUG "metric": { 05:17:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:17:56 DEBUG }, 05:17:56 DEBUG "value": [ 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG [...] 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 05:17:56 DEBUG }, 05:17:56 DEBUG "value": [ 05:17:56 DEBUG 1713500276, 05:17:56 DEBUG "0" 05:17:56 DEBUG ] 05:17:56 DEBUG } 05:17:56 DEBUG ] 05:17:56 DEBUG } 05:17:56 DEBUG } 05:17:56 INFO [http_cmd]: http status code OK 05:17:56 DEBUG --- status code --- 05:17:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:17:56 DEBUG --- http response --- 05:17:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:17:56 DEBUG ----- output ----- 05:17:56 DEBUG { 05:17:56 DEBUG "status": "success", 05:17:56 DEBUG "data": { 05:17:56 DEBUG "resultType": "vector", 05:17:56 DEBUG "result": [ 05:17:56 DEBUG { 05:17:56 DEBUG "metric": { 05:17:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:17:56 DEBUG }, 05:17:56 DEBUG "value": [ 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG [...] 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:17:56 DEBUG }, 05:17:56 DEBUG "value": [ 05:17:56 DEBUG 1713500276, 05:17:56 DEBUG "55.172" 05:17:56 DEBUG ] 05:17:56 DEBUG } 05:17:56 DEBUG ] 05:17:56 DEBUG } 05:17:56 DEBUG } 05:17:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500276" 05:17:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500276" 05:17:56 INFO [http_cmd]: http status code OK 05:17:56 INFO [http_cmd]: http status code OK 05:17:56 DEBUG --- status code --- 05:17:56 DEBUG --- status code --- 05:17:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:17:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:17:56 DEBUG --- http response --- 05:17:56 DEBUG --- http response --- 05:17:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:17:56 DEBUG ----- output ----- 05:17:56 DEBUG { 05:17:56 DEBUG "status": "success", 05:17:56 DEBUG "data": { 05:17:56 DEBUG "resultType": "vector", 05:17:56 DEBUG "result": [ 05:17:56 DEBUG { 05:17:56 DEBUG "metric": { 05:17:56 DEBUG "instance": "10.142.0.107:9100" 05:17:56 DEBUG }, 05:17:56 DEBUG "value": [ 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG [...] 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG "instance": "10.142.0.43:9100" 05:17:56 DEBUG }, 05:17:56 DEBUG "value": [ 05:17:56 DEBUG 1713500276, 05:17:56 DEBUG "64157399.666666664" 05:17:56 DEBUG ] 05:17:56 DEBUG } 05:17:56 DEBUG ] 05:17:56 DEBUG } 05:17:56 DEBUG } 05:17:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:17:56 DEBUG ----- output ----- 05:17:56 DEBUG { 05:17:56 DEBUG "status": "success", 05:17:56 DEBUG "data": { 05:17:56 DEBUG "resultType": "vector", 05:17:56 DEBUG "result": [ 05:17:56 DEBUG { 05:17:56 DEBUG "metric": { 05:17:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:17:56 DEBUG }, 05:17:56 DEBUG "value": [ 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG [...] 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:17:56 DEBUG }, 05:17:56 DEBUG "value": [ 05:17:56 DEBUG 1713500276, 05:17:56 DEBUG "1992.8" 05:17:56 DEBUG ] 05:17:56 DEBUG } 05:17:56 DEBUG ] 05:17:56 DEBUG } 05:17:56 DEBUG } 05:17:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500276" 05:17:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500276" 05:17:56 INFO [http_cmd]: http status code OK 05:17:56 DEBUG --- status code --- 05:17:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:17:56 DEBUG --- http response --- 05:17:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:17:56 DEBUG ----- output ----- 05:17:56 DEBUG { 05:17:56 DEBUG "status": "success", 05:17:56 DEBUG "data": { 05:17:56 DEBUG "resultType": "vector", 05:17:56 DEBUG "result": [ 05:17:56 DEBUG { 05:17:56 DEBUG "metric": { 05:17:56 DEBUG "pod": "ds-idrepo-0" 05:17:56 DEBUG }, 05:17:56 DEBUG "value": [ 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG [...] 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG "pod": "ds-idrepo-2" 05:17:56 DEBUG }, 05:17:56 DEBUG "value": [ 05:17:56 DEBUG 1713500276, 05:17:56 DEBUG "0" 05:17:56 DEBUG ] 05:17:56 DEBUG } 05:17:56 DEBUG ] 05:17:56 DEBUG } 05:17:56 DEBUG } 05:17:56 INFO [http_cmd]: http status code OK 05:17:56 DEBUG --- status code --- 05:17:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:17:56 DEBUG --- http response --- 05:17:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:17:56 DEBUG ----- output ----- 05:17:56 DEBUG { 05:17:56 DEBUG "status": "success", 05:17:56 DEBUG "data": { 05:17:56 DEBUG "resultType": "vector", 05:17:56 DEBUG "result": [ 05:17:56 DEBUG { 05:17:56 DEBUG "metric": { 05:17:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:17:56 DEBUG }, 05:17:56 DEBUG "value": [ 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG [...] 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:17:56 DEBUG }, 05:17:56 DEBUG "value": [ 05:17:56 DEBUG 1713500276, 05:17:56 DEBUG "2168.1" 05:17:56 DEBUG ] 05:17:56 DEBUG } 05:17:56 DEBUG ] 05:17:56 DEBUG } 05:17:56 DEBUG } 05:17:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500276" 05:17:56 INFO [http_cmd]: http status code OK 05:17:56 DEBUG --- status code --- 05:17:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:17:56 DEBUG --- http response --- 05:17:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:17:56 DEBUG ----- output ----- 05:17:56 DEBUG { 05:17:56 DEBUG "status": "success", 05:17:56 DEBUG "data": { 05:17:56 DEBUG "resultType": "vector", 05:17:56 DEBUG "result": [ 05:17:56 DEBUG { 05:17:56 DEBUG "metric": { 05:17:56 DEBUG "instance": "10.142.0.107:9100" 05:17:56 DEBUG }, 05:17:56 DEBUG "value": [ 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG [...] 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG 05:17:56 DEBUG "instance": "10.142.0.43:9100" 05:17:56 DEBUG }, 05:17:56 DEBUG "value": [ 05:17:56 DEBUG 1713500276, 05:17:56 DEBUG "0.3753333333333351" 05:17:56 DEBUG ] 05:17:56 DEBUG } 05:17:56 DEBUG ] 05:17:56 DEBUG } 05:17:56 DEBUG } 05:17:57 INFO 05:17:57 INFO [loop_until]: kubectl --namespace=xlou top pods 05:17:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:17:57 INFO [loop_until]: OK (rc = 0) 05:17:57 DEBUG --- stdout --- 05:17:57 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 13586m 9143Mi am-6fd554f8f7-hvvkw 13516m 8895Mi am-6fd554f8f7-pmj4b 14128m 8785Mi ds-cts-0 17677m 12731Mi ds-cts-1 18127m 12708Mi ds-cts-2 18951m 12729Mi ds-idrepo-0 5936m 3267Mi ds-idrepo-1 5789m 3198Mi ds-idrepo-2 5776m 3066Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 3m 1215Mi lodemon-6b89d774d5-rf7sk 1m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7003m 2009Mi 05:17:57 DEBUG --- stderr --- 05:17:57 DEBUG 05:17:57 INFO 05:17:57 INFO [loop_until]: kubectl --namespace=xlou top node 05:17:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:17:57 INFO [loop_until]: OK (rc = 0) 05:17:57 DEBUG --- stdout --- 05:17:57 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 66m 0% 6225Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5850m 19% 6714Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13182m 44% 11619Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 81m 0% 3268Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 18538m 62% 15500Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 159m 0% 5173Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 13716m 45% 11922Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 11276m 37% 2736Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 15182m 50% 10988Mi 9% gke-xlou-cdm-ds-0f601751-j488 6132m 20% 6167Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 17772m 59% 16739Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 6421m 21% 6880Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 18493m 61% 14678Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 7279m 45% 5739Mi 9% 05:17:57 DEBUG --- stderr --- 05:17:57 DEBUG 05:18:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500336" 05:18:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713500336" 05:18:56 INFO [http_cmd]: http status code OK 05:18:56 DEBUG --- status code --- 05:18:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:18:56 DEBUG --- http response --- 05:18:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:18:56 DEBUG ----- output ----- 05:18:56 DEBUG { 05:18:56 DEBUG "status": "success", 05:18:56 DEBUG "data": { 05:18:56 DEBUG "resultType": "vector", 05:18:56 DEBUG "result": [ 05:18:56 DEBUG { 05:18:56 DEBUG "metric": { 05:18:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:18:56 DEBUG }, 05:18:56 DEBUG "value": [ 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG [...] 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:18:56 DEBUG }, 05:18:56 DEBUG "value": [ 05:18:56 DEBUG 1713500336, 05:18:56 DEBUG "3514648" 05:18:56 DEBUG ] 05:18:56 DEBUG } 05:18:56 DEBUG ] 05:18:56 DEBUG } 05:18:56 DEBUG } 05:18:56 ERROR [http_cmd]: ERROR 05:18:56 ERROR --- status code --- 05:18:56 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:18:56 ERROR --- http response --- 05:18:56 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:18:56 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:18:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500336" 05:18:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500336" 05:18:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500336" 05:18:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713500336" 05:18:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500336" 05:18:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500336" 05:18:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713500336" 05:18:56 INFO [http_cmd]: http status code OK 05:18:56 DEBUG --- status code --- 05:18:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:18:56 DEBUG --- http response --- 05:18:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:18:56 DEBUG ----- output ----- 05:18:56 DEBUG { 05:18:56 DEBUG "status": "success", 05:18:56 DEBUG "data": { 05:18:56 DEBUG "resultType": "vector", 05:18:56 DEBUG "result": [ 05:18:56 DEBUG { 05:18:56 DEBUG "metric": { 05:18:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:18:56 DEBUG }, 05:18:56 DEBUG "value": [ 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG [...] 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG "pod": "idm-b85f9d978-wnhxb" 05:18:56 DEBUG }, 05:18:56 DEBUG "value": [ 05:18:56 DEBUG 1713500336, 05:18:56 DEBUG "0.005883642287696616" 05:18:56 DEBUG ] 05:18:56 DEBUG } 05:18:56 DEBUG ] 05:18:56 DEBUG } 05:18:56 DEBUG } 05:18:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500336" 05:18:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500336" 05:18:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500336" 05:18:56 INFO [http_cmd]: http status code OK 05:18:56 INFO [http_cmd]: http status code OK 05:18:56 INFO [http_cmd]: http status code OK 05:18:56 INFO [http_cmd]: http status code OK 05:18:56 DEBUG --- status code --- 05:18:56 DEBUG --- status code --- 05:18:56 DEBUG --- status code --- 05:18:56 DEBUG --- status code --- 05:18:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:18:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:18:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:18:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:18:56 DEBUG --- http response --- 05:18:56 DEBUG --- http response --- 05:18:56 DEBUG --- http response --- 05:18:56 DEBUG --- http response --- 05:18:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:18:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:18:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:18:56 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:18:56 DEBUG ----- output ----- 05:18:56 DEBUG ----- output ----- 05:18:56 DEBUG ----- output ----- 05:18:56 DEBUG { 05:18:56 DEBUG { 05:18:56 DEBUG { 05:18:56 DEBUG "status": "success", 05:18:56 DEBUG "status": "success", 05:18:56 DEBUG "status": "success", 05:18:56 DEBUG "data": { 05:18:56 DEBUG "data": { 05:18:56 DEBUG "data": { 05:18:56 DEBUG "resultType": "vector", 05:18:56 DEBUG "resultType": "vector", 05:18:56 DEBUG "resultType": "vector", 05:18:56 DEBUG "result": [ 05:18:56 DEBUG "result": [ 05:18:56 INFO [http_cmd]: http status code OK 05:18:56 DEBUG "result": [ 05:18:56 DEBUG { 05:18:56 INFO [http_cmd]: http status code OK 05:18:56 DEBUG { 05:18:56 INFO [http_cmd]: http status code OK 05:18:56 DEBUG --- status code --- 05:18:56 DEBUG { 05:18:56 INFO [http_cmd]: http status code OK 05:18:56 DEBUG "metric": { 05:18:56 DEBUG --- status code --- 05:18:56 DEBUG "metric": { 05:18:56 DEBUG --- status code --- 05:18:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:18:56 DEBUG "metric": { 05:18:56 DEBUG --- status code --- 05:18:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:18:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:18:56 DEBUG "pod": "ds-idrepo-0" 05:18:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:18:56 DEBUG --- http response --- 05:18:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:18:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:18:56 DEBUG }, 05:18:56 DEBUG --- http response --- 05:18:56 DEBUG }, 05:18:56 DEBUG --- http response --- 05:18:56 DEBUG }, 05:18:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:18:56 DEBUG ----- output ----- 05:18:56 DEBUG { 05:18:56 DEBUG "value": [ 05:18:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:18:56 DEBUG "value": [ 05:18:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:18:56 DEBUG --- http response --- 05:18:56 DEBUG "value": [ 05:18:56 DEBUG "status": "success", 05:18:56 DEBUG 05:18:56 DEBUG ----- output ----- 05:18:56 DEBUG 05:18:56 DEBUG ----- output ----- 05:18:56 DEBUG 05:18:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:18:56 DEBUG "data": { 05:18:56 DEBUG 05:18:56 DEBUG { 05:18:56 DEBUG 05:18:56 DEBUG { 05:18:56 DEBUG 05:18:56 DEBUG ----- output ----- 05:18:56 DEBUG "resultType": "vector", 05:18:56 DEBUG 05:18:56 DEBUG "status": "success", 05:18:56 DEBUG 05:18:56 DEBUG "status": "success", 05:18:56 DEBUG 05:18:56 DEBUG { 05:18:56 DEBUG "result": [ 05:18:56 DEBUG [...] 05:18:56 DEBUG "data": { 05:18:56 DEBUG [...] 05:18:56 DEBUG "data": { 05:18:56 DEBUG [...] 05:18:56 DEBUG "status": "success", 05:18:56 DEBUG { 05:18:56 DEBUG 05:18:56 DEBUG "resultType": "vector", 05:18:56 DEBUG 05:18:56 DEBUG "resultType": "vector", 05:18:56 DEBUG 05:18:56 INFO [http_cmd]: http status code OK 05:18:56 DEBUG "data": { 05:18:56 DEBUG "metric": { 05:18:56 DEBUG 05:18:56 DEBUG "result": [ 05:18:56 DEBUG 05:18:56 DEBUG "result": [ 05:18:56 DEBUG 05:18:56 DEBUG --- status code --- 05:18:56 DEBUG "resultType": "vector", 05:18:56 DEBUG "pod": "ds-idrepo-0" 05:18:56 DEBUG 05:18:56 DEBUG { 05:18:56 DEBUG 05:18:56 DEBUG { 05:18:56 DEBUG 05:18:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:18:56 DEBUG "result": [ 05:18:56 DEBUG }, 05:18:56 DEBUG "pod": "ds-cts-2" 05:18:56 DEBUG "metric": { 05:18:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:18:56 DEBUG "metric": { 05:18:56 DEBUG "pod": "idm-b85f9d978-wnhxb" 05:18:56 DEBUG --- http response --- 05:18:56 DEBUG { 05:18:56 DEBUG "value": [ 05:18:56 DEBUG }, 05:18:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:18:56 DEBUG }, 05:18:56 DEBUG "pod": "ds-cts-0" 05:18:56 DEBUG }, 05:18:56 DEBUG "metric": { 05:18:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:18:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500336" 05:18:56 DEBUG 05:18:56 DEBUG "value": [ 05:18:56 DEBUG }, 05:18:56 DEBUG "value": [ 05:18:56 DEBUG }, 05:18:56 DEBUG "value": [ 05:18:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:18:56 DEBUG ----- output ----- 05:18:56 DEBUG 05:18:56 DEBUG 1713500336, 05:18:56 DEBUG "value": [ 05:18:56 DEBUG 1713500336, 05:18:56 DEBUG "value": [ 05:18:56 DEBUG 1713500336, 05:18:56 DEBUG }, 05:18:56 DEBUG { 05:18:56 DEBUG 05:18:56 DEBUG "0" 05:18:56 DEBUG 05:18:56 DEBUG "0" 05:18:56 DEBUG 05:18:56 DEBUG "1107.7485317046046" 05:18:56 DEBUG "value": [ 05:18:56 DEBUG "status": "success", 05:18:56 DEBUG [...] 05:18:56 DEBUG ] 05:18:56 DEBUG 05:18:56 DEBUG ] 05:18:56 DEBUG 05:18:56 DEBUG ] 05:18:56 DEBUG 05:18:56 DEBUG "data": { 05:18:56 DEBUG 05:18:56 DEBUG } 05:18:56 DEBUG 05:18:56 DEBUG } 05:18:56 DEBUG 05:18:56 DEBUG } 05:18:56 DEBUG 05:18:56 DEBUG "resultType": "vector", 05:18:56 DEBUG 05:18:56 DEBUG ] 05:18:56 DEBUG [...] 05:18:56 DEBUG ] 05:18:56 DEBUG [...] 05:18:56 DEBUG ] 05:18:56 DEBUG 05:18:56 DEBUG "result": [ 05:18:56 DEBUG 05:18:56 DEBUG } 05:18:56 DEBUG 05:18:56 DEBUG } 05:18:56 DEBUG 05:18:56 DEBUG } 05:18:56 DEBUG [...] 05:18:56 DEBUG { 05:18:56 DEBUG "pod": "ds-idrepo-2" 05:18:56 DEBUG } 05:18:56 DEBUG 05:18:56 DEBUG } 05:18:56 DEBUG 05:18:56 DEBUG } 05:18:56 DEBUG 05:18:56 DEBUG "metric": { 05:18:56 DEBUG }, 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:18:56 DEBUG "value": [ 05:18:56 DEBUG "pod": "idm-b85f9d978-wnhxb" 05:18:56 DEBUG "pod": "ds-cts-2" 05:18:56 DEBUG 05:18:56 DEBUG }, 05:18:56 DEBUG 1713500336, 05:18:56 DEBUG }, 05:18:56 DEBUG }, 05:18:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:18:56 DEBUG "value": [ 05:18:56 DEBUG "0" 05:18:56 DEBUG "value": [ 05:18:56 DEBUG "value": [ 05:18:56 DEBUG }, 05:18:56 DEBUG 05:18:56 DEBUG ] 05:18:56 DEBUG 1713500336, 05:18:56 DEBUG 1713500336, 05:18:56 DEBUG "value": [ 05:18:56 DEBUG 05:18:56 DEBUG } 05:18:56 DEBUG "40974771.76577829" 05:18:56 DEBUG "0.0009333333333333333" 05:18:56 DEBUG 1713500336, 05:18:56 DEBUG 05:18:56 DEBUG ] 05:18:56 DEBUG ] 05:18:56 DEBUG ] 05:18:56 DEBUG "7.100800210249198" 05:18:56 DEBUG [...] 05:18:56 DEBUG 05:18:56 DEBUG } 05:18:56 DEBUG } 05:18:56 DEBUG ] 05:18:56 DEBUG } 05:18:56 DEBUG 05:18:56 DEBUG ] 05:18:56 DEBUG ] 05:18:56 DEBUG } 05:18:56 DEBUG } 05:18:56 DEBUG 05:18:56 DEBUG } 05:18:56 DEBUG } 05:18:56 DEBUG ] 05:18:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:18:56 DEBUG } 05:18:56 DEBUG } 05:18:56 DEBUG } 05:18:56 DEBUG }, 05:18:56 DEBUG } 05:18:56 DEBUG "value": [ 05:18:56 DEBUG 1713500336, 05:18:56 DEBUG "2170.733333333333" 05:18:56 DEBUG ] 05:18:56 DEBUG } 05:18:56 DEBUG ] 05:18:56 DEBUG } 05:18:56 DEBUG } 05:18:56 INFO [http_cmd]: http status code OK 05:18:56 DEBUG --- status code --- 05:18:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:18:56 DEBUG --- http response --- 05:18:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:18:56 DEBUG ----- output ----- 05:18:56 DEBUG { 05:18:56 DEBUG "status": "success", 05:18:56 DEBUG "data": { 05:18:56 DEBUG "resultType": "vector", 05:18:56 DEBUG "result": [ 05:18:56 DEBUG { 05:18:56 DEBUG "metric": { 05:18:56 DEBUG "instance": "10.142.0.107:9100" 05:18:56 DEBUG }, 05:18:56 DEBUG "value": [ 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG [...] 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG "instance": "10.142.0.43:9100" 05:18:56 DEBUG }, 05:18:56 DEBUG "value": [ 05:18:56 DEBUG 1713500336, 05:18:56 DEBUG "64194411.46666667" 05:18:56 DEBUG ] 05:18:56 DEBUG } 05:18:56 DEBUG ] 05:18:56 DEBUG } 05:18:56 DEBUG } 05:18:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500336" 05:18:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500336" 05:18:56 INFO [http_cmd]: http status code OK 05:18:56 INFO [http_cmd]: http status code OK 05:18:56 DEBUG --- status code --- 05:18:56 DEBUG --- status code --- 05:18:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:18:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:18:56 DEBUG --- http response --- 05:18:56 DEBUG --- http response --- 05:18:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:18:56 DEBUG ----- output ----- 05:18:56 DEBUG { 05:18:56 DEBUG "status": "success", 05:18:56 DEBUG "data": { 05:18:56 DEBUG "resultType": "vector", 05:18:56 DEBUG "result": [ 05:18:56 DEBUG { 05:18:56 DEBUG "metric": { 05:18:56 DEBUG "pod": "ds-cts-0" 05:18:56 DEBUG }, 05:18:56 DEBUG "value": [ 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG [...] 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG "pod": "ds-cts-2" 05:18:56 DEBUG }, 05:18:56 DEBUG "value": [ 05:18:56 DEBUG 1713500336, 05:18:56 DEBUG "0" 05:18:56 DEBUG ] 05:18:56 DEBUG } 05:18:56 DEBUG ] 05:18:56 DEBUG } 05:18:56 DEBUG } 05:18:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:18:56 DEBUG ----- output ----- 05:18:56 DEBUG { 05:18:56 DEBUG "status": "success", 05:18:56 DEBUG "data": { 05:18:56 DEBUG "resultType": "vector", 05:18:56 DEBUG "result": [ 05:18:56 DEBUG { 05:18:56 DEBUG "metric": { 05:18:56 DEBUG "pod": "ds-idrepo-0" 05:18:56 DEBUG }, 05:18:56 DEBUG "value": [ 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG [...] 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500336" 05:18:56 DEBUG "pod": "ds-idrepo-2" 05:18:56 DEBUG }, 05:18:56 DEBUG "value": [ 05:18:56 DEBUG 1713500336, 05:18:56 DEBUG "0" 05:18:56 DEBUG ] 05:18:56 DEBUG } 05:18:56 DEBUG ] 05:18:56 DEBUG } 05:18:56 DEBUG } 05:18:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500336" 05:18:56 INFO [http_cmd]: http status code OK 05:18:56 DEBUG --- status code --- 05:18:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:18:56 DEBUG --- http response --- 05:18:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:18:56 DEBUG ----- output ----- 05:18:56 DEBUG { 05:18:56 DEBUG "status": "success", 05:18:56 DEBUG "data": { 05:18:56 DEBUG "resultType": "vector", 05:18:56 DEBUG "result": [ 05:18:56 DEBUG { 05:18:56 DEBUG "metric": { 05:18:56 DEBUG "instance": "10.142.0.107:9100" 05:18:56 DEBUG }, 05:18:56 DEBUG "value": [ 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG [...] 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG "instance": "10.142.0.43:9100" 05:18:56 DEBUG }, 05:18:56 DEBUG "value": [ 05:18:56 DEBUG 1713500336, 05:18:56 DEBUG "466165085184" 05:18:56 DEBUG ] 05:18:56 DEBUG } 05:18:56 DEBUG ] 05:18:56 DEBUG } 05:18:56 DEBUG } 05:18:56 INFO [http_cmd]: http status code OK 05:18:56 DEBUG --- status code --- 05:18:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:18:56 DEBUG --- http response --- 05:18:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:18:56 DEBUG ----- output ----- 05:18:56 DEBUG { 05:18:56 DEBUG "status": "success", 05:18:56 DEBUG "data": { 05:18:56 DEBUG "resultType": "vector", 05:18:56 DEBUG "result": [ 05:18:56 DEBUG { 05:18:56 DEBUG "metric": { 05:18:56 DEBUG "instance": "10.142.0.107:9100" 05:18:56 DEBUG }, 05:18:56 DEBUG "value": [ 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG [...] 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG "instance": "10.142.0.43:9100" 05:18:56 DEBUG }, 05:18:56 DEBUG "value": [ 05:18:56 DEBUG 1713500336, 05:18:56 DEBUG "64197806.400000006" 05:18:56 DEBUG ] 05:18:56 DEBUG } 05:18:56 DEBUG ] 05:18:56 DEBUG } 05:18:56 DEBUG } 05:18:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500336" 05:18:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500336" 05:18:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500336" 05:18:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713500336" 05:18:56 INFO [http_cmd]: http status code OK 05:18:56 DEBUG --- status code --- 05:18:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:18:56 DEBUG --- http response --- 05:18:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:18:56 DEBUG ----- output ----- 05:18:56 DEBUG { 05:18:56 DEBUG "status": "success", 05:18:56 DEBUG "data": { 05:18:56 DEBUG "resultType": "vector", 05:18:56 DEBUG "result": [ 05:18:56 DEBUG { 05:18:56 DEBUG "metric": { 05:18:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:18:56 DEBUG }, 05:18:56 DEBUG "value": [ 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG [...] 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:18:56 DEBUG }, 05:18:56 DEBUG "value": [ 05:18:56 DEBUG 1713500336, 05:18:56 DEBUG "2170.633333333333" 05:18:56 DEBUG ] 05:18:56 DEBUG } 05:18:56 DEBUG ] 05:18:56 DEBUG } 05:18:56 DEBUG } 05:18:56 INFO [http_cmd]: http status code OK 05:18:56 INFO [http_cmd]: http status code OK 05:18:56 DEBUG --- status code --- 05:18:56 DEBUG --- status code --- 05:18:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:18:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:18:56 DEBUG --- http response --- 05:18:56 DEBUG --- http response --- 05:18:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:18:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:18:56 DEBUG ----- output ----- 05:18:56 INFO [http_cmd]: http status code OK 05:18:56 DEBUG ----- output ----- 05:18:56 DEBUG { 05:18:56 DEBUG --- status code --- 05:18:56 DEBUG { 05:18:56 DEBUG "status": "success", 05:18:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:18:56 DEBUG "status": "success", 05:18:56 DEBUG "data": { 05:18:56 DEBUG --- http response --- 05:18:56 DEBUG "data": { 05:18:56 DEBUG "resultType": "vector", 05:18:56 DEBUG "resultType": "vector", 05:18:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:18:56 DEBUG "result": [ 05:18:56 DEBUG "result": [ 05:18:56 DEBUG ----- output ----- 05:18:56 DEBUG { 05:18:56 DEBUG { 05:18:56 DEBUG { 05:18:56 DEBUG "metric": { 05:18:56 DEBUG "metric": { 05:18:56 DEBUG "status": "success", 05:18:56 DEBUG "pod": "ds-cts-0" 05:18:56 DEBUG "instance": "10.142.0.107:9100" 05:18:56 DEBUG "data": { 05:18:56 DEBUG }, 05:18:56 DEBUG }, 05:18:56 DEBUG "resultType": "vector", 05:18:56 DEBUG "value": [ 05:18:56 DEBUG "value": [ 05:18:56 DEBUG "result": [ 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG { 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG "metric": { 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:18:56 DEBUG [...] 05:18:56 DEBUG [...] 05:18:56 DEBUG }, 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG "value": [ 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG "pod": "ds-cts-2" 05:18:56 DEBUG "instance": "10.142.0.43:9100" 05:18:56 DEBUG 05:18:56 DEBUG }, 05:18:56 DEBUG }, 05:18:56 DEBUG [...] 05:18:56 DEBUG "value": [ 05:18:56 DEBUG "value": [ 05:18:56 DEBUG 05:18:56 DEBUG 1713500336, 05:18:56 DEBUG 1713500336, 05:18:56 DEBUG 05:18:56 DEBUG "444.46666666666664" 05:18:56 DEBUG "0.6886666666666741" 05:18:56 DEBUG 05:18:56 DEBUG ] 05:18:56 DEBUG ] 05:18:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:18:56 DEBUG } 05:18:56 DEBUG } 05:18:56 DEBUG }, 05:18:56 DEBUG ] 05:18:56 DEBUG ] 05:18:56 DEBUG "value": [ 05:18:56 DEBUG } 05:18:56 DEBUG } 05:18:56 DEBUG 1713500336, 05:18:56 DEBUG } 05:18:56 DEBUG } 05:18:56 DEBUG "2207940608" 05:18:56 DEBUG ] 05:18:56 DEBUG } 05:18:56 DEBUG ] 05:18:56 DEBUG } 05:18:56 DEBUG } 05:18:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500336" 05:18:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713500336" 05:18:56 INFO [http_cmd]: http status code OK 05:18:56 DEBUG --- status code --- 05:18:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:18:56 DEBUG --- http response --- 05:18:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:18:56 DEBUG ----- output ----- 05:18:56 DEBUG { 05:18:56 DEBUG "status": "success", 05:18:56 DEBUG "data": { 05:18:56 DEBUG "resultType": "vector", 05:18:56 DEBUG "result": [ 05:18:56 DEBUG { 05:18:56 DEBUG "metric": { 05:18:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:18:56 DEBUG }, 05:18:56 DEBUG "value": [ 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG [...] 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG "pod": "idm-b85f9d978-wnhxb" 05:18:56 DEBUG }, 05:18:56 DEBUG "value": [ 05:18:56 DEBUG 1713500336, 05:18:56 DEBUG "0" 05:18:56 DEBUG ] 05:18:56 DEBUG } 05:18:56 DEBUG ] 05:18:56 DEBUG } 05:18:56 DEBUG } 05:18:56 INFO [http_cmd]: http status code OK 05:18:56 DEBUG --- status code --- 05:18:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:18:56 DEBUG --- http response --- 05:18:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:18:56 DEBUG ----- output ----- 05:18:56 DEBUG { 05:18:56 DEBUG "status": "success", 05:18:56 DEBUG "data": { 05:18:56 DEBUG "resultType": "vector", 05:18:56 DEBUG "result": [ 05:18:56 DEBUG { 05:18:56 DEBUG "metric": { 05:18:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:18:56 DEBUG }, 05:18:56 DEBUG "value": [ 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG [...] 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:18:56 DEBUG }, 05:18:56 DEBUG "value": [ 05:18:56 DEBUG 1713500336, 05:18:56 DEBUG "55.172" 05:18:56 DEBUG ] 05:18:56 DEBUG } 05:18:56 DEBUG ] 05:18:56 DEBUG } 05:18:56 DEBUG } 05:18:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500336" 05:18:56 INFO [http_cmd]: http status code OK 05:18:56 DEBUG --- status code --- 05:18:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:18:56 DEBUG --- http response --- 05:18:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:18:56 DEBUG ----- output ----- 05:18:56 DEBUG { 05:18:56 DEBUG "status": "success", 05:18:56 DEBUG "data": { 05:18:56 DEBUG "resultType": "vector", 05:18:56 DEBUG "result": [ 05:18:56 DEBUG { 05:18:56 DEBUG "metric": { 05:18:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:18:56 DEBUG }, 05:18:56 DEBUG "value": [ 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG [...] 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:18:56 DEBUG }, 05:18:56 DEBUG "value": [ 05:18:56 DEBUG 1713500336, 05:18:56 DEBUG "1996.2666666666667" 05:18:56 DEBUG ] 05:18:56 DEBUG } 05:18:56 DEBUG ] 05:18:56 DEBUG } 05:18:56 DEBUG } 05:18:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500336" 05:18:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500336" 05:18:56 INFO [http_cmd]: http status code OK 05:18:56 DEBUG --- status code --- 05:18:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:18:56 DEBUG --- http response --- 05:18:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:18:56 DEBUG ----- output ----- 05:18:56 DEBUG { 05:18:56 DEBUG "status": "success", 05:18:56 DEBUG "data": { 05:18:56 DEBUG "resultType": "vector", 05:18:56 DEBUG "result": [ 05:18:56 DEBUG { 05:18:56 DEBUG "metric": { 05:18:56 DEBUG "pod": "ds-idrepo-0" 05:18:56 DEBUG }, 05:18:56 DEBUG "value": [ 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG [...] 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG "pod": "ds-idrepo-2" 05:18:56 DEBUG }, 05:18:56 DEBUG "value": [ 05:18:56 DEBUG 1713500336, 05:18:56 DEBUG "0" 05:18:56 DEBUG ] 05:18:56 DEBUG } 05:18:56 DEBUG ] 05:18:56 DEBUG } 05:18:56 DEBUG } 05:18:56 INFO [http_cmd]: http status code OK 05:18:56 DEBUG --- status code --- 05:18:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:18:56 DEBUG --- http response --- 05:18:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:18:56 DEBUG ----- output ----- 05:18:56 DEBUG { 05:18:56 DEBUG "status": "success", 05:18:56 DEBUG "data": { 05:18:56 DEBUG "resultType": "vector", 05:18:56 DEBUG "result": [ 05:18:56 DEBUG { 05:18:56 DEBUG "metric": { 05:18:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:18:56 DEBUG }, 05:18:56 DEBUG "value": [ 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG [...] 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:18:56 DEBUG }, 05:18:56 DEBUG "value": [ 05:18:56 DEBUG 1713500336, 05:18:56 DEBUG "2171.266666666667" 05:18:56 DEBUG ] 05:18:56 DEBUG } 05:18:56 DEBUG ] 05:18:56 DEBUG } 05:18:56 DEBUG } 05:18:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500336" 05:18:56 INFO [http_cmd]: http status code OK 05:18:56 DEBUG --- status code --- 05:18:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:18:56 DEBUG --- http response --- 05:18:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:18:56 DEBUG ----- output ----- 05:18:56 DEBUG { 05:18:56 DEBUG "status": "success", 05:18:56 DEBUG "data": { 05:18:56 DEBUG "resultType": "vector", 05:18:56 DEBUG "result": [ 05:18:56 DEBUG { 05:18:56 DEBUG "metric": { 05:18:56 DEBUG "instance": "10.142.0.107:9100" 05:18:56 DEBUG }, 05:18:56 DEBUG "value": [ 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG [...] 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG 05:18:56 DEBUG "instance": "10.142.0.43:9100" 05:18:56 DEBUG }, 05:18:56 DEBUG "value": [ 05:18:56 DEBUG 1713500336, 05:18:56 DEBUG "0.3980000000000008" 05:18:56 DEBUG ] 05:18:56 DEBUG } 05:18:56 DEBUG ] 05:18:56 DEBUG } 05:18:56 DEBUG } 05:18:57 INFO 05:18:57 INFO [loop_until]: kubectl --namespace=xlou top pods 05:18:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:18:57 INFO [loop_until]: OK (rc = 0) 05:18:57 DEBUG --- stdout --- 05:18:57 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 14006m 9141Mi am-6fd554f8f7-hvvkw 13232m 8899Mi am-6fd554f8f7-pmj4b 14165m 8786Mi ds-cts-0 18319m 13023Mi ds-cts-1 17621m 12981Mi ds-cts-2 18849m 12971Mi ds-idrepo-0 5862m 3267Mi ds-idrepo-1 5882m 3199Mi ds-idrepo-2 5671m 3066Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 3m 1216Mi lodemon-6b89d774d5-rf7sk 4m 65Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 6989m 2010Mi 05:18:57 DEBUG --- stderr --- 05:18:57 DEBUG 05:18:58 INFO 05:18:58 INFO [loop_until]: kubectl --namespace=xlou top node 05:18:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:18:58 INFO [loop_until]: OK (rc = 0) 05:18:58 DEBUG --- stdout --- 05:18:58 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 69m 0% 6238Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5810m 19% 6704Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13118m 43% 11611Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 79m 0% 3247Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 17970m 60% 15015Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 135m 0% 5207Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 13645m 45% 11924Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 11292m 37% 2721Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 15471m 51% 10994Mi 9% gke-xlou-cdm-ds-0f601751-j488 6146m 20% 6171Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 17856m 59% 16953Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6433m 21% 6880Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 18906m 63% 14863Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 7279m 45% 5741Mi 9% 05:18:58 DEBUG --- stderr --- 05:18:58 DEBUG 05:19:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713500396" 05:19:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500396" 05:19:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500396" 05:19:56 ERROR [http_cmd]: ERROR 05:19:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713500396" 05:19:56 ERROR --- status code --- 05:19:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500396" 05:19:56 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:19:56 INFO [http_cmd]: http status code OK 05:19:56 ERROR --- http response --- 05:19:56 DEBUG --- status code --- 05:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:19:56 DEBUG --- http response --- 05:19:56 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:19:56 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:19:56 DEBUG ----- output ----- 05:19:56 DEBUG { 05:19:56 DEBUG "status": "success", 05:19:56 DEBUG "data": { 05:19:56 DEBUG "resultType": "vector", 05:19:56 DEBUG "result": [ 05:19:56 DEBUG { 05:19:56 DEBUG "metric": { 05:19:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:19:56 DEBUG }, 05:19:56 DEBUG "value": [ 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG [...] 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:19:56 DEBUG }, 05:19:56 DEBUG "value": [ 05:19:56 DEBUG 1713500396, 05:19:56 DEBUG "3524682" 05:19:56 DEBUG ] 05:19:56 DEBUG } 05:19:56 DEBUG ] 05:19:56 DEBUG } 05:19:56 DEBUG } 05:19:56 INFO [http_cmd]: http status code OK 05:19:56 DEBUG --- status code --- 05:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:19:56 DEBUG --- http response --- 05:19:56 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:19:56 INFO [http_cmd]: http status code OK 05:19:56 DEBUG --- status code --- 05:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:19:56 DEBUG --- http response --- 05:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:19:56 DEBUG ----- output ----- 05:19:56 DEBUG { 05:19:56 DEBUG "status": "success", 05:19:56 DEBUG "data": { 05:19:56 DEBUG "resultType": "vector", 05:19:56 DEBUG "result": [ 05:19:56 DEBUG { 05:19:56 DEBUG "metric": { 05:19:56 DEBUG "pod": "ds-idrepo-0" 05:19:56 DEBUG }, 05:19:56 DEBUG "value": [ 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG [...] 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG "pod": "ds-cts-2" 05:19:56 DEBUG }, 05:19:56 DEBUG "value": [ 05:19:56 DEBUG 1713500396, 05:19:56 DEBUG "0" 05:19:56 DEBUG ] 05:19:56 DEBUG } 05:19:56 DEBUG ] 05:19:56 DEBUG } 05:19:56 DEBUG } 05:19:56 INFO [http_cmd]: http status code OK 05:19:56 DEBUG --- status code --- 05:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:19:56 DEBUG --- http response --- 05:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:19:56 DEBUG ----- output ----- 05:19:56 DEBUG { 05:19:56 DEBUG "status": "success", 05:19:56 DEBUG "data": { 05:19:56 DEBUG "resultType": "vector", 05:19:56 DEBUG "result": [ 05:19:56 DEBUG { 05:19:56 DEBUG "metric": { 05:19:56 DEBUG "pod": "ds-idrepo-1" 05:19:56 DEBUG }, 05:19:56 DEBUG "value": [ 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG [...] 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG "pod": "ds-cts-0" 05:19:56 DEBUG }, 05:19:56 DEBUG "value": [ 05:19:56 DEBUG 1713500396, 05:19:56 DEBUG "7.82962188685084" 05:19:56 DEBUG ] 05:19:56 DEBUG } 05:19:56 DEBUG ] 05:19:56 DEBUG } 05:19:56 DEBUG } 05:19:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500396" 05:19:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713500396" 05:19:56 INFO [http_cmd]: http status code OK 05:19:56 DEBUG --- status code --- 05:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:19:56 DEBUG --- http response --- 05:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:19:56 DEBUG ----- output ----- 05:19:56 DEBUG { 05:19:56 DEBUG "status": "success", 05:19:56 DEBUG "data": { 05:19:56 DEBUG "resultType": "vector", 05:19:56 DEBUG "result": [ 05:19:56 DEBUG { 05:19:56 DEBUG "metric": { 05:19:56 DEBUG "pod": "ds-cts-0" 05:19:56 DEBUG }, 05:19:56 DEBUG "value": [ 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG [...] 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG "pod": "ds-cts-2" 05:19:56 DEBUG }, 05:19:56 DEBUG "value": [ 05:19:56 DEBUG 1713500396, 05:19:56 DEBUG "0" 05:19:56 DEBUG ] 05:19:56 DEBUG } 05:19:56 DEBUG ] 05:19:56 DEBUG } 05:19:56 DEBUG } 05:19:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500396" 05:19:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500396" 05:19:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500396" 05:19:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500396" 05:19:56 INFO [http_cmd]: http status code OK 05:19:56 INFO [http_cmd]: http status code OK 05:19:56 DEBUG --- status code --- 05:19:56 DEBUG --- status code --- 05:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:19:56 DEBUG --- http response --- 05:19:56 DEBUG --- http response --- 05:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:19:56 DEBUG ----- output ----- 05:19:56 DEBUG { 05:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:19:56 DEBUG "status": "success", 05:19:56 DEBUG ----- output ----- 05:19:56 INFO [http_cmd]: http status code OK 05:19:56 DEBUG "data": { 05:19:56 DEBUG { 05:19:56 DEBUG --- status code --- 05:19:56 DEBUG "resultType": "vector", 05:19:56 DEBUG "status": "success", 05:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:19:56 DEBUG "result": [ 05:19:56 DEBUG "data": { 05:19:56 DEBUG --- http response --- 05:19:56 DEBUG { 05:19:56 DEBUG "resultType": "vector", 05:19:56 DEBUG "metric": { 05:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:19:56 DEBUG "result": [ 05:19:56 DEBUG "pod": "ds-idrepo-0" 05:19:56 DEBUG ----- output ----- 05:19:56 DEBUG { 05:19:56 DEBUG }, 05:19:56 DEBUG { 05:19:56 DEBUG "metric": { 05:19:56 DEBUG "value": [ 05:19:56 DEBUG "status": "success", 05:19:56 DEBUG "pod": "ds-idrepo-1" 05:19:56 DEBUG 05:19:56 DEBUG "data": { 05:19:56 DEBUG }, 05:19:56 DEBUG 05:19:56 DEBUG "resultType": "vector", 05:19:56 DEBUG "value": [ 05:19:56 DEBUG 05:19:56 DEBUG "result": [ 05:19:56 DEBUG 05:19:56 DEBUG [...] 05:19:56 DEBUG { 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG "metric": { 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:19:56 DEBUG [...] 05:19:56 DEBUG 05:19:56 DEBUG }, 05:19:56 DEBUG 05:19:56 DEBUG "pod": "ds-idrepo-2" 05:19:56 DEBUG "value": [ 05:19:56 DEBUG 05:19:56 DEBUG }, 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG "value": [ 05:19:56 DEBUG 05:19:56 DEBUG "pod": "ds-cts-0" 05:19:56 DEBUG 1713500396, 05:19:56 DEBUG 05:19:56 DEBUG }, 05:19:56 DEBUG "0" 05:19:56 DEBUG [...] 05:19:56 DEBUG "value": [ 05:19:56 DEBUG ] 05:19:56 DEBUG 05:19:56 DEBUG 1713500396, 05:19:56 DEBUG } 05:19:56 DEBUG 05:19:56 DEBUG "5805587.471553278" 05:19:56 DEBUG ] 05:19:56 DEBUG 05:19:56 DEBUG ] 05:19:56 DEBUG } 05:19:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:19:56 DEBUG } 05:19:56 DEBUG } 05:19:56 DEBUG }, 05:19:56 DEBUG ] 05:19:56 DEBUG "value": [ 05:19:56 DEBUG } 05:19:56 DEBUG 1713500396, 05:19:56 DEBUG } 05:19:56 DEBUG "7.138420958866185" 05:19:56 DEBUG ] 05:19:56 DEBUG } 05:19:56 DEBUG ] 05:19:56 DEBUG } 05:19:56 DEBUG } 05:19:56 INFO [http_cmd]: http status code OK 05:19:56 DEBUG --- status code --- 05:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:19:56 DEBUG --- http response --- 05:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:19:56 DEBUG ----- output ----- 05:19:56 DEBUG { 05:19:56 DEBUG "status": "success", 05:19:56 DEBUG "data": { 05:19:56 DEBUG "resultType": "vector", 05:19:56 DEBUG "result": [ 05:19:56 DEBUG { 05:19:56 DEBUG "metric": { 05:19:56 DEBUG "instance": "10.142.0.107:9100" 05:19:56 DEBUG }, 05:19:56 DEBUG "value": [ 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG [...] 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG "instance": "10.142.0.43:9100" 05:19:56 DEBUG }, 05:19:56 DEBUG "value": [ 05:19:56 DEBUG 1713500396, 05:19:56 DEBUG "64403274.46666667" 05:19:56 DEBUG ] 05:19:56 DEBUG } 05:19:56 DEBUG ] 05:19:56 DEBUG } 05:19:56 DEBUG } 05:19:56 INFO [http_cmd]: http status code OK 05:19:56 DEBUG --- status code --- 05:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:19:56 DEBUG --- http response --- 05:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:19:56 DEBUG ----- output ----- 05:19:56 DEBUG { 05:19:56 DEBUG "status": "success", 05:19:56 DEBUG "data": { 05:19:56 DEBUG "resultType": "vector", 05:19:56 DEBUG "result": [ 05:19:56 DEBUG { 05:19:56 DEBUG "metric": { 05:19:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:19:56 DEBUG }, 05:19:56 DEBUG "value": [ 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG [...] 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:19:56 DEBUG }, 05:19:56 DEBUG "value": [ 05:19:56 DEBUG 1713500396, 05:19:56 DEBUG "2174.366666666667" 05:19:56 DEBUG ] 05:19:56 DEBUG } 05:19:56 DEBUG ] 05:19:56 DEBUG } 05:19:56 DEBUG } 05:19:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500396" 05:19:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500396" 05:19:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500396" 05:19:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500396" 05:19:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500396" 05:19:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500396" 05:19:56 INFO [http_cmd]: http status code OK 05:19:56 INFO [http_cmd]: http status code OK 05:19:56 DEBUG --- status code --- 05:19:56 DEBUG --- status code --- 05:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:19:56 DEBUG --- http response --- 05:19:56 DEBUG --- http response --- 05:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:19:56 DEBUG ----- output ----- 05:19:56 DEBUG { 05:19:56 DEBUG "status": "success", 05:19:56 DEBUG "data": { 05:19:56 DEBUG "resultType": "vector", 05:19:56 DEBUG "result": [ 05:19:56 DEBUG { 05:19:56 DEBUG "metric": { 05:19:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:19:56 DEBUG }, 05:19:56 DEBUG "value": [ 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG [...] 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:19:56 DEBUG }, 05:19:56 DEBUG "value": [ 05:19:56 DEBUG 1713500396, 05:19:56 DEBUG "0" 05:19:56 DEBUG ] 05:19:56 DEBUG } 05:19:56 DEBUG ] 05:19:56 DEBUG } 05:19:56 DEBUG } 05:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:19:56 DEBUG ----- output ----- 05:19:56 DEBUG { 05:19:56 DEBUG "status": "success", 05:19:56 DEBUG "data": { 05:19:56 DEBUG "resultType": "vector", 05:19:56 DEBUG "result": [ 05:19:56 DEBUG { 05:19:56 DEBUG "metric": { 05:19:56 DEBUG "pod": "ds-idrepo-1" 05:19:56 DEBUG }, 05:19:56 DEBUG "value": [ 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG [...] 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG "pod": "ds-cts-0" 05:19:56 DEBUG }, 05:19:56 DEBUG "value": [ 05:19:56 DEBUG 1713500396, 05:19:56 DEBUG "22051628.72466007" 05:19:56 DEBUG ] 05:19:56 DEBUG } 05:19:56 DEBUG ] 05:19:56 DEBUG } 05:19:56 DEBUG } 05:19:56 INFO [http_cmd]: http status code OK 05:19:56 DEBUG --- status code --- 05:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:19:56 DEBUG --- http response --- 05:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:19:56 DEBUG ----- output ----- 05:19:56 DEBUG { 05:19:56 DEBUG "status": "success", 05:19:56 DEBUG "data": { 05:19:56 DEBUG "resultType": "vector", 05:19:56 DEBUG "result": [ 05:19:56 DEBUG { 05:19:56 DEBUG "metric": { 05:19:56 DEBUG "pod": "ds-idrepo-0" 05:19:56 DEBUG }, 05:19:56 DEBUG "value": [ 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG [...] 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG "pod": "ds-idrepo-2" 05:19:56 DEBUG }, 05:19:56 DEBUG "value": [ 05:19:56 DEBUG 1713500396, 05:19:56 DEBUG "0" 05:19:56 DEBUG ] 05:19:56 DEBUG } 05:19:56 DEBUG ] 05:19:56 DEBUG } 05:19:56 DEBUG } 05:19:56 INFO [http_cmd]: http status code OK 05:19:56 DEBUG --- status code --- 05:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:19:56 DEBUG --- http response --- 05:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:19:56 DEBUG ----- output ----- 05:19:56 DEBUG { 05:19:56 DEBUG "status": "success", 05:19:56 DEBUG "data": { 05:19:56 DEBUG "resultType": "vector", 05:19:56 DEBUG "result": [ 05:19:56 DEBUG { 05:19:56 DEBUG "metric": { 05:19:56 DEBUG "pod": "ds-cts-0" 05:19:56 DEBUG }, 05:19:56 DEBUG "value": [ 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG [...] 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG "pod": "ds-cts-2" 05:19:56 DEBUG }, 05:19:56 DEBUG "value": [ 05:19:56 DEBUG 1713500396, 05:19:56 DEBUG "0.0007666666666666667" 05:19:56 DEBUG ] 05:19:56 DEBUG } 05:19:56 DEBUG ] 05:19:56 DEBUG } 05:19:56 DEBUG } 05:19:56 INFO [http_cmd]: http status code OK 05:19:56 DEBUG --- status code --- 05:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:19:56 DEBUG --- http response --- 05:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:19:56 DEBUG ----- output ----- 05:19:56 DEBUG { 05:19:56 DEBUG "status": "success", 05:19:56 DEBUG "data": { 05:19:56 DEBUG "resultType": "vector", 05:19:56 DEBUG "result": [ 05:19:56 DEBUG { 05:19:56 DEBUG "metric": { 05:19:56 DEBUG "instance": "10.142.0.107:9100" 05:19:56 DEBUG }, 05:19:56 DEBUG "value": [ 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG [...] 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG "instance": "10.142.0.43:9100" 05:19:56 DEBUG }, 05:19:56 DEBUG "value": [ 05:19:56 DEBUG 1713500396, 05:19:56 DEBUG "64399964.56666666" 05:19:56 DEBUG ] 05:19:56 DEBUG } 05:19:56 DEBUG ] 05:19:56 DEBUG } 05:19:56 DEBUG } 05:19:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713500396" 05:19:56 INFO [http_cmd]: http status code OK 05:19:56 DEBUG --- status code --- 05:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:19:56 DEBUG --- http response --- 05:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:19:56 DEBUG ----- output ----- 05:19:56 DEBUG { 05:19:56 DEBUG "status": "success", 05:19:56 DEBUG "data": { 05:19:56 DEBUG "resultType": "vector", 05:19:56 DEBUG "result": [ 05:19:56 DEBUG { 05:19:56 DEBUG "metric": { 05:19:56 DEBUG "pod": "ds-cts-0" 05:19:56 DEBUG }, 05:19:56 DEBUG "value": [ 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG [...] 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG "pod": "ds-cts-2" 05:19:56 DEBUG }, 05:19:56 DEBUG "value": [ 05:19:56 DEBUG 1713500396, 05:19:56 DEBUG "445.4" 05:19:56 DEBUG ] 05:19:56 DEBUG } 05:19:56 DEBUG ] 05:19:56 DEBUG } 05:19:56 DEBUG } 05:19:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500396" 05:19:56 INFO [http_cmd]: http status code OK 05:19:56 DEBUG --- status code --- 05:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:19:56 DEBUG --- http response --- 05:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:19:56 DEBUG ----- output ----- 05:19:56 DEBUG { 05:19:56 DEBUG "status": "success", 05:19:56 DEBUG "data": { 05:19:56 DEBUG "resultType": "vector", 05:19:56 DEBUG "result": [ 05:19:56 DEBUG { 05:19:56 DEBUG "metric": { 05:19:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:19:56 DEBUG }, 05:19:56 DEBUG "value": [ 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG [...] 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:19:56 DEBUG }, 05:19:56 DEBUG "value": [ 05:19:56 DEBUG 1713500396, 05:19:56 DEBUG "55.172" 05:19:56 DEBUG ] 05:19:56 DEBUG } 05:19:56 DEBUG ] 05:19:56 DEBUG } 05:19:56 DEBUG } 05:19:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500396" 05:19:56 INFO [http_cmd]: http status code OK 05:19:56 DEBUG --- status code --- 05:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:19:56 DEBUG --- http response --- 05:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:19:56 DEBUG ----- output ----- 05:19:56 DEBUG { 05:19:56 DEBUG "status": "success", 05:19:56 DEBUG "data": { 05:19:56 DEBUG "resultType": "vector", 05:19:56 DEBUG "result": [ 05:19:56 DEBUG { 05:19:56 DEBUG "metric": { 05:19:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:19:56 DEBUG }, 05:19:56 DEBUG "value": [ 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG [...] 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:19:56 DEBUG }, 05:19:56 DEBUG "value": [ 05:19:56 DEBUG 1713500396, 05:19:56 DEBUG "2174.2" 05:19:56 DEBUG ] 05:19:56 DEBUG } 05:19:56 DEBUG ] 05:19:56 DEBUG } 05:19:56 DEBUG } 05:19:56 INFO [http_cmd]: http status code OK 05:19:56 DEBUG --- status code --- 05:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:19:56 DEBUG --- http response --- 05:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:19:56 DEBUG ----- output ----- 05:19:56 DEBUG { 05:19:56 DEBUG "status": "success", 05:19:56 DEBUG "data": { 05:19:56 DEBUG "resultType": "vector", 05:19:56 DEBUG "result": [ 05:19:56 DEBUG { 05:19:56 DEBUG "metric": { 05:19:56 DEBUG "instance": "10.142.0.107:9100" 05:19:56 DEBUG }, 05:19:56 DEBUG "value": [ 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG [...] 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG "instance": "10.142.0.43:9100" 05:19:56 DEBUG }, 05:19:56 DEBUG "value": [ 05:19:56 DEBUG 1713500396, 05:19:56 DEBUG "469275595776" 05:19:56 DEBUG ] 05:19:56 DEBUG } 05:19:56 DEBUG ] 05:19:56 DEBUG } 05:19:56 DEBUG } 05:19:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500396" 05:19:56 INFO [http_cmd]: http status code OK 05:19:56 DEBUG --- status code --- 05:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:19:56 DEBUG --- http response --- 05:19:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500396" 05:19:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713500396" 05:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:19:56 DEBUG ----- output ----- 05:19:56 DEBUG { 05:19:56 DEBUG "status": "success", 05:19:56 DEBUG "data": { 05:19:56 DEBUG "resultType": "vector", 05:19:56 DEBUG "result": [ 05:19:56 DEBUG { 05:19:56 DEBUG "metric": { 05:19:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:19:56 DEBUG }, 05:19:56 DEBUG "value": [ 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG [...] 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:19:56 DEBUG }, 05:19:56 DEBUG "value": [ 05:19:56 DEBUG 1713500396, 05:19:56 DEBUG "2009.1666666666667" 05:19:56 DEBUG ] 05:19:56 DEBUG } 05:19:56 DEBUG ] 05:19:56 DEBUG } 05:19:56 DEBUG } 05:19:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500396" 05:19:56 INFO [http_cmd]: http status code OK 05:19:56 DEBUG --- status code --- 05:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:19:56 DEBUG --- http response --- 05:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:19:56 DEBUG ----- output ----- 05:19:56 DEBUG { 05:19:56 DEBUG "status": "success", 05:19:56 DEBUG "data": { 05:19:56 DEBUG "resultType": "vector", 05:19:56 DEBUG "result": [ 05:19:56 DEBUG { 05:19:56 DEBUG "metric": { 05:19:56 DEBUG "instance": "10.142.0.107:9100" 05:19:56 DEBUG }, 05:19:56 DEBUG "value": [ 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG [...] 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG "instance": "10.142.0.43:9100" 05:19:56 DEBUG }, 05:19:56 DEBUG "value": [ 05:19:56 DEBUG 1713500396, 05:19:56 DEBUG "0.7023000000000034" 05:19:56 DEBUG ] 05:19:56 DEBUG } 05:19:56 DEBUG ] 05:19:56 DEBUG } 05:19:56 DEBUG } 05:19:56 INFO [http_cmd]: http status code OK 05:19:56 DEBUG --- status code --- 05:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:19:56 DEBUG --- http response --- 05:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:19:56 DEBUG ----- output ----- 05:19:56 DEBUG { 05:19:56 DEBUG "status": "success", 05:19:56 DEBUG "data": { 05:19:56 DEBUG "resultType": "vector", 05:19:56 DEBUG "result": [ 05:19:56 DEBUG { 05:19:56 DEBUG "metric": { 05:19:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:19:56 DEBUG }, 05:19:56 DEBUG "value": [ 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG [...] 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:19:56 DEBUG }, 05:19:56 DEBUG "value": [ 05:19:56 DEBUG 1713500396, 05:19:56 DEBUG "2206838784" 05:19:56 DEBUG ] 05:19:56 DEBUG } 05:19:56 DEBUG ] 05:19:56 DEBUG } 05:19:56 DEBUG } 05:19:56 INFO [http_cmd]: http status code OK 05:19:56 DEBUG --- status code --- 05:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:19:56 DEBUG --- http response --- 05:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:19:56 DEBUG ----- output ----- 05:19:56 DEBUG { 05:19:56 DEBUG "status": "success", 05:19:56 DEBUG "data": { 05:19:56 DEBUG "resultType": "vector", 05:19:56 DEBUG "result": [ 05:19:56 DEBUG { 05:19:56 DEBUG "metric": { 05:19:56 DEBUG "pod": "ds-idrepo-1" 05:19:56 DEBUG }, 05:19:56 DEBUG "value": [ 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG [...] 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG "pod": "ds-cts-0" 05:19:56 DEBUG }, 05:19:56 DEBUG "value": [ 05:19:56 DEBUG 1713500396, 05:19:56 DEBUG "8118755.038311218" 05:19:56 DEBUG ] 05:19:56 DEBUG } 05:19:56 DEBUG ] 05:19:56 DEBUG } 05:19:56 DEBUG } 05:19:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500396" 05:19:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500396" 05:19:56 INFO [http_cmd]: http status code OK 05:19:56 DEBUG --- status code --- 05:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:19:56 DEBUG --- http response --- 05:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:19:56 DEBUG ----- output ----- 05:19:56 DEBUG { 05:19:56 DEBUG "status": "success", 05:19:56 DEBUG "data": { 05:19:56 DEBUG "resultType": "vector", 05:19:56 DEBUG "result": [ 05:19:56 DEBUG { 05:19:56 DEBUG "metric": { 05:19:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:19:56 DEBUG }, 05:19:56 DEBUG "value": [ 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG [...] 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:19:56 DEBUG }, 05:19:56 DEBUG "value": [ 05:19:56 DEBUG 1713500396, 05:19:56 DEBUG "2171.8333333333335" 05:19:56 DEBUG ] 05:19:56 DEBUG } 05:19:56 DEBUG ] 05:19:56 DEBUG } 05:19:56 DEBUG } 05:19:56 INFO [http_cmd]: http status code OK 05:19:56 DEBUG --- status code --- 05:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:19:56 DEBUG --- http response --- 05:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:19:56 DEBUG ----- output ----- 05:19:56 DEBUG { 05:19:56 DEBUG "status": "success", 05:19:56 DEBUG "data": { 05:19:56 DEBUG "resultType": "vector", 05:19:56 DEBUG "result": [ 05:19:56 DEBUG { 05:19:56 DEBUG "metric": { 05:19:56 DEBUG "pod": "ds-idrepo-0" 05:19:56 DEBUG }, 05:19:56 DEBUG "value": [ 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG [...] 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG "pod": "ds-idrepo-2" 05:19:56 DEBUG }, 05:19:56 DEBUG "value": [ 05:19:56 DEBUG 1713500396, 05:19:56 DEBUG "0" 05:19:56 DEBUG ] 05:19:56 DEBUG } 05:19:56 DEBUG ] 05:19:56 DEBUG } 05:19:56 DEBUG } 05:19:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500396" 05:19:56 INFO [http_cmd]: http status code OK 05:19:56 DEBUG --- status code --- 05:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:19:56 DEBUG --- http response --- 05:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:19:56 DEBUG ----- output ----- 05:19:56 DEBUG { 05:19:56 DEBUG "status": "success", 05:19:56 DEBUG "data": { 05:19:56 DEBUG "resultType": "vector", 05:19:56 DEBUG "result": [ 05:19:56 DEBUG { 05:19:56 DEBUG "metric": { 05:19:56 DEBUG "instance": "10.142.0.107:9100" 05:19:56 DEBUG }, 05:19:56 DEBUG "value": [ 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG [...] 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG 05:19:56 DEBUG "instance": "10.142.0.43:9100" 05:19:56 DEBUG }, 05:19:56 DEBUG "value": [ 05:19:56 DEBUG 1713500396, 05:19:56 DEBUG "0.4126666666666665" 05:19:56 DEBUG ] 05:19:56 DEBUG } 05:19:56 DEBUG ] 05:19:56 DEBUG } 05:19:56 DEBUG } 05:19:57 INFO 05:19:57 INFO [loop_until]: kubectl --namespace=xlou top pods 05:19:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:19:57 INFO [loop_until]: OK (rc = 0) 05:19:57 DEBUG --- stdout --- 05:19:57 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 13742m 9142Mi am-6fd554f8f7-hvvkw 13336m 8901Mi am-6fd554f8f7-pmj4b 14526m 8786Mi ds-cts-0 18361m 12662Mi ds-cts-1 18413m 13066Mi ds-cts-2 19113m 13000Mi ds-idrepo-0 6081m 3267Mi ds-idrepo-1 5841m 3199Mi ds-idrepo-2 5564m 3066Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 4m 1216Mi lodemon-6b89d774d5-rf7sk 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7011m 2013Mi 05:19:57 DEBUG --- stderr --- 05:19:57 DEBUG 05:19:58 INFO 05:19:58 INFO [loop_until]: kubectl --namespace=xlou top node 05:19:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:19:58 INFO [loop_until]: OK (rc = 0) 05:19:58 DEBUG --- stdout --- 05:19:58 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 69m 0% 6247Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5805m 19% 6716Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13250m 44% 11607Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 80m 0% 3254Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 18546m 62% 15065Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 150m 0% 5192Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 13733m 45% 11926Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 11335m 37% 2726Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 15307m 51% 11000Mi 9% gke-xlou-cdm-ds-0f601751-j488 6163m 20% 6163Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 18007m 60% 16493Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 6405m 21% 6867Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 18783m 62% 15022Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 7297m 45% 5734Mi 9% 05:19:58 DEBUG --- stderr --- 05:19:58 DEBUG 05:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713500456" 05:20:56 INFO [http_cmd]: http status code OK 05:20:56 DEBUG --- status code --- 05:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:20:56 DEBUG --- http response --- 05:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:20:56 DEBUG ----- output ----- 05:20:56 DEBUG { 05:20:56 DEBUG "status": "success", 05:20:56 DEBUG "data": { 05:20:56 DEBUG "resultType": "vector", 05:20:56 DEBUG "result": [ 05:20:56 DEBUG { 05:20:56 DEBUG "metric": { 05:20:56 DEBUG "pod": "ds-idrepo-0" 05:20:56 DEBUG }, 05:20:56 DEBUG "value": [ 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG [...] 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG "pod": "ds-cts-2" 05:20:56 DEBUG }, 05:20:56 DEBUG "value": [ 05:20:56 DEBUG 1713500456, 05:20:56 DEBUG "0" 05:20:56 DEBUG ] 05:20:56 DEBUG } 05:20:56 DEBUG ] 05:20:56 DEBUG } 05:20:56 DEBUG } 05:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713500456" 05:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500456" 05:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500456" 05:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500456" 05:20:56 ERROR [http_cmd]: ERROR 05:20:56 INFO [http_cmd]: http status code OK 05:20:56 ERROR --- status code --- 05:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500456" 05:20:56 DEBUG --- status code --- 05:20:56 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:20:56 ERROR --- http response --- 05:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500456" 05:20:56 DEBUG --- http response --- 05:20:56 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:20:56 INFO [http_cmd]: http status code OK 05:20:56 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:20:56 DEBUG ----- output ----- 05:20:56 DEBUG --- status code --- 05:20:56 DEBUG { 05:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:20:56 DEBUG "status": "success", 05:20:56 DEBUG --- http response --- 05:20:56 DEBUG "data": { 05:20:56 DEBUG "resultType": "vector", 05:20:56 DEBUG "result": [ 05:20:56 DEBUG { 05:20:56 DEBUG "metric": { 05:20:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:20:56 DEBUG }, 05:20:56 DEBUG "value": [ 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG [...] 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:20:56 DEBUG }, 05:20:56 DEBUG "value": [ 05:20:56 DEBUG 1713500456, 05:20:56 DEBUG "3535228" 05:20:56 DEBUG ] 05:20:56 DEBUG } 05:20:56 DEBUG ] 05:20:56 DEBUG } 05:20:56 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:20:56 DEBUG } 05:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500456" 05:20:56 INFO [http_cmd]: http status code OK 05:20:56 DEBUG --- status code --- 05:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:20:56 DEBUG --- http response --- 05:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:20:56 DEBUG ----- output ----- 05:20:56 DEBUG { 05:20:56 DEBUG "status": "success", 05:20:56 DEBUG "data": { 05:20:56 DEBUG "resultType": "vector", 05:20:56 DEBUG "result": [ 05:20:56 DEBUG { 05:20:56 DEBUG "metric": { 05:20:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:20:56 DEBUG }, 05:20:56 DEBUG "value": [ 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG [...] 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:20:56 DEBUG }, 05:20:56 DEBUG "value": [ 05:20:56 DEBUG 1713500456, 05:20:56 DEBUG "3.6166595341089924" 05:20:56 DEBUG ] 05:20:56 DEBUG } 05:20:56 DEBUG ] 05:20:56 DEBUG } 05:20:56 DEBUG } 05:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500456" 05:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500456" 05:20:56 INFO [http_cmd]: http status code OK 05:20:56 DEBUG --- status code --- 05:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:20:56 DEBUG --- http response --- 05:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:20:56 DEBUG ----- output ----- 05:20:56 DEBUG { 05:20:56 DEBUG "status": "success", 05:20:56 DEBUG "data": { 05:20:56 DEBUG "resultType": "vector", 05:20:56 DEBUG "result": [ 05:20:56 DEBUG { 05:20:56 DEBUG "metric": { 05:20:56 DEBUG "pod": "ds-idrepo-0" 05:20:56 DEBUG }, 05:20:56 DEBUG "value": [ 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG [...] 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG "pod": "ds-idrepo-2" 05:20:56 DEBUG }, 05:20:56 DEBUG "value": [ 05:20:56 DEBUG 1713500456, 05:20:56 DEBUG "0" 05:20:56 DEBUG ] 05:20:56 DEBUG } 05:20:56 DEBUG ] 05:20:56 DEBUG } 05:20:56 DEBUG } 05:20:56 INFO [http_cmd]: http status code OK 05:20:56 DEBUG --- status code --- 05:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:20:56 DEBUG --- http response --- 05:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500456" 05:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:20:56 DEBUG ----- output ----- 05:20:56 DEBUG { 05:20:56 DEBUG "status": "success", 05:20:56 DEBUG "data": { 05:20:56 DEBUG "resultType": "vector", 05:20:56 DEBUG "result": [ 05:20:56 DEBUG { 05:20:56 DEBUG "metric": { 05:20:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:20:56 DEBUG }, 05:20:56 DEBUG "value": [ 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG [...] 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:20:56 DEBUG }, 05:20:56 DEBUG "value": [ 05:20:56 DEBUG 1713500456, 05:20:56 DEBUG "0" 05:20:56 DEBUG ] 05:20:56 DEBUG } 05:20:56 DEBUG ] 05:20:56 DEBUG } 05:20:56 DEBUG } 05:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500456" 05:20:56 INFO [http_cmd]: http status code OK 05:20:56 DEBUG --- status code --- 05:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:20:56 DEBUG --- http response --- 05:20:56 INFO [http_cmd]: http status code OK 05:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:20:56 DEBUG --- status code --- 05:20:56 DEBUG ----- output ----- 05:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:20:56 DEBUG { 05:20:56 DEBUG --- http response --- 05:20:56 DEBUG "status": "success", 05:20:56 DEBUG "data": { 05:20:56 DEBUG "resultType": "vector", 05:20:56 DEBUG "result": [ 05:20:56 DEBUG { 05:20:56 DEBUG "metric": { 05:20:56 DEBUG "pod": "ds-cts-0" 05:20:56 DEBUG }, 05:20:56 DEBUG "value": [ 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG [...] 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG "pod": "ds-cts-2" 05:20:56 DEBUG }, 05:20:56 DEBUG "value": [ 05:20:56 DEBUG 1713500456, 05:20:56 DEBUG "0.0003333333333333334" 05:20:56 DEBUG ] 05:20:56 DEBUG } 05:20:56 DEBUG ] 05:20:56 DEBUG } 05:20:56 DEBUG } 05:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:20:56 DEBUG ----- output ----- 05:20:56 DEBUG { 05:20:56 DEBUG "status": "success", 05:20:56 DEBUG "data": { 05:20:56 DEBUG "resultType": "vector", 05:20:56 DEBUG "result": [ 05:20:56 DEBUG { 05:20:56 DEBUG "metric": { 05:20:56 DEBUG "instance": "10.142.0.107:9100" 05:20:56 DEBUG }, 05:20:56 DEBUG "value": [ 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG [...] 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG "instance": "10.142.0.43:9100" 05:20:56 DEBUG }, 05:20:56 DEBUG "value": [ 05:20:56 DEBUG 1713500456, 05:20:56 DEBUG "64140830.06666666" 05:20:56 DEBUG ] 05:20:56 DEBUG } 05:20:56 DEBUG ] 05:20:56 DEBUG } 05:20:56 DEBUG } 05:20:56 INFO [http_cmd]: http status code OK 05:20:56 DEBUG --- status code --- 05:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:20:56 DEBUG --- http response --- 05:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:20:56 DEBUG ----- output ----- 05:20:56 DEBUG { 05:20:56 DEBUG "status": "success", 05:20:56 DEBUG "data": { 05:20:56 DEBUG "resultType": "vector", 05:20:56 DEBUG "result": [ 05:20:56 DEBUG { 05:20:56 DEBUG "metric": { 05:20:56 DEBUG "pod": "ds-cts-0" 05:20:56 DEBUG }, 05:20:56 DEBUG "value": [ 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG [...] 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG "pod": "ds-cts-2" 05:20:56 DEBUG }, 05:20:56 DEBUG "value": [ 05:20:56 INFO [http_cmd]: http status code OK 05:20:56 DEBUG 1713500456, 05:20:56 DEBUG --- status code --- 05:20:56 DEBUG "0.06666666666666667" 05:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:20:56 DEBUG ] 05:20:56 DEBUG --- http response --- 05:20:56 DEBUG } 05:20:56 DEBUG ] 05:20:56 DEBUG } 05:20:56 DEBUG } 05:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:20:56 DEBUG ----- output ----- 05:20:56 DEBUG { 05:20:56 DEBUG "status": "success", 05:20:56 DEBUG "data": { 05:20:56 DEBUG "resultType": "vector", 05:20:56 DEBUG "result": [ 05:20:56 DEBUG { 05:20:56 DEBUG "metric": { 05:20:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:20:56 DEBUG }, 05:20:56 DEBUG "value": [ 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG [...] 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:20:56 DEBUG }, 05:20:56 DEBUG "value": [ 05:20:56 DEBUG 1713500456, 05:20:56 DEBUG "1671.320058217002" 05:20:56 DEBUG ] 05:20:56 DEBUG } 05:20:56 DEBUG ] 05:20:56 DEBUG } 05:20:56 DEBUG } 05:20:56 INFO [http_cmd]: http status code OK 05:20:56 DEBUG --- status code --- 05:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:20:56 DEBUG --- http response --- 05:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:20:56 DEBUG ----- output ----- 05:20:56 DEBUG { 05:20:56 DEBUG "status": "success", 05:20:56 DEBUG "data": { 05:20:56 DEBUG "resultType": "vector", 05:20:56 DEBUG "result": [ 05:20:56 DEBUG { 05:20:56 DEBUG "metric": { 05:20:56 DEBUG "pod": "ds-idrepo-0" 05:20:56 DEBUG }, 05:20:56 DEBUG "value": [ 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG [...] 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG "pod": "ds-idrepo-2" 05:20:56 DEBUG }, 05:20:56 DEBUG "value": [ 05:20:56 DEBUG 1713500456, 05:20:56 DEBUG "0" 05:20:56 DEBUG ] 05:20:56 DEBUG } 05:20:56 DEBUG ] 05:20:56 DEBUG } 05:20:56 DEBUG } 05:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713500456" 05:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500456" 05:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500456" 05:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500456" 05:20:56 INFO [http_cmd]: http status code OK 05:20:56 INFO [http_cmd]: http status code OK 05:20:56 DEBUG --- status code --- 05:20:56 DEBUG --- status code --- 05:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:20:56 DEBUG --- http response --- 05:20:56 DEBUG --- http response --- 05:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:20:56 DEBUG ----- output ----- 05:20:56 DEBUG { 05:20:56 DEBUG "status": "success", 05:20:56 DEBUG "data": { 05:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:20:56 DEBUG "resultType": "vector", 05:20:56 DEBUG ----- output ----- 05:20:56 DEBUG "result": [ 05:20:56 DEBUG { 05:20:56 DEBUG { 05:20:56 DEBUG "status": "success", 05:20:56 DEBUG "metric": { 05:20:56 DEBUG "data": { 05:20:56 DEBUG "pod": "ds-cts-0" 05:20:56 DEBUG "resultType": "vector", 05:20:56 INFO [http_cmd]: http status code OK 05:20:56 DEBUG }, 05:20:56 DEBUG "result": [ 05:20:56 DEBUG --- status code --- 05:20:56 DEBUG "value": [ 05:20:56 DEBUG { 05:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:20:56 DEBUG 05:20:56 DEBUG "metric": { 05:20:56 DEBUG --- http response --- 05:20:56 DEBUG 05:20:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:20:56 DEBUG 05:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:20:56 DEBUG }, 05:20:56 DEBUG [...] 05:20:56 DEBUG ----- output ----- 05:20:56 DEBUG "value": [ 05:20:56 DEBUG 05:20:56 DEBUG { 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG "status": "success", 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG "data": { 05:20:56 DEBUG 05:20:56 DEBUG "pod": "ds-cts-2" 05:20:56 DEBUG "resultType": "vector", 05:20:56 DEBUG [...] 05:20:56 DEBUG }, 05:20:56 DEBUG "result": [ 05:20:56 DEBUG 05:20:56 DEBUG "value": [ 05:20:56 DEBUG { 05:20:56 DEBUG 05:20:56 DEBUG 1713500456, 05:20:56 DEBUG "metric": { 05:20:56 DEBUG 05:20:56 DEBUG "468.56666666666666" 05:20:56 DEBUG "instance": "10.142.0.107:9100" 05:20:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:20:56 DEBUG ] 05:20:56 DEBUG }, 05:20:56 DEBUG }, 05:20:56 DEBUG } 05:20:56 DEBUG "value": [ 05:20:56 DEBUG "value": [ 05:20:56 DEBUG ] 05:20:56 DEBUG 05:20:56 DEBUG 1713500456, 05:20:56 DEBUG } 05:20:56 DEBUG 05:20:56 DEBUG "7.135012953616561" 05:20:56 DEBUG } 05:20:56 DEBUG 05:20:56 DEBUG ] 05:20:56 DEBUG [...] 05:20:56 DEBUG } 05:20:56 DEBUG 05:20:56 DEBUG ] 05:20:56 DEBUG 05:20:56 DEBUG } 05:20:56 DEBUG 05:20:56 DEBUG } 05:20:56 DEBUG "instance": "10.142.0.43:9100" 05:20:56 DEBUG }, 05:20:56 DEBUG "value": [ 05:20:56 DEBUG 1713500456, 05:20:56 DEBUG "64144424.233333334" 05:20:56 DEBUG ] 05:20:56 DEBUG } 05:20:56 DEBUG ] 05:20:56 DEBUG } 05:20:56 DEBUG } 05:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500456" 05:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500456" 05:20:56 INFO [http_cmd]: http status code OK 05:20:56 DEBUG --- status code --- 05:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:20:56 DEBUG --- http response --- 05:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:20:56 DEBUG ----- output ----- 05:20:56 DEBUG { 05:20:56 DEBUG "status": "success", 05:20:56 DEBUG "data": { 05:20:56 DEBUG "resultType": "vector", 05:20:56 DEBUG "result": [ 05:20:56 DEBUG { 05:20:56 DEBUG "metric": { 05:20:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:20:56 DEBUG }, 05:20:56 DEBUG "value": [ 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG [...] 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:20:56 DEBUG }, 05:20:56 DEBUG "value": [ 05:20:56 DEBUG 1713500456, 05:20:56 DEBUG "2169.6666666666665" 05:20:56 DEBUG ] 05:20:56 DEBUG } 05:20:56 DEBUG ] 05:20:56 DEBUG } 05:20:56 DEBUG } 05:20:56 INFO [http_cmd]: http status code OK 05:20:56 DEBUG --- status code --- 05:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:20:56 DEBUG --- http response --- 05:20:56 INFO [http_cmd]: http status code OK 05:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:20:56 DEBUG --- status code --- 05:20:56 DEBUG ----- output ----- 05:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:20:56 DEBUG { 05:20:56 DEBUG --- http response --- 05:20:56 DEBUG "status": "success", 05:20:56 DEBUG "data": { 05:20:56 DEBUG "resultType": "vector", 05:20:56 DEBUG "result": [ 05:20:56 DEBUG { 05:20:56 DEBUG "metric": { 05:20:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:20:56 DEBUG }, 05:20:56 DEBUG "value": [ 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG [...] 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:20:56 DEBUG }, 05:20:56 DEBUG "value": [ 05:20:56 DEBUG 1713500456, 05:20:56 DEBUG "2169.633333333333" 05:20:56 DEBUG ] 05:20:56 DEBUG } 05:20:56 DEBUG ] 05:20:56 DEBUG } 05:20:56 DEBUG } 05:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:20:56 DEBUG ----- output ----- 05:20:56 DEBUG { 05:20:56 DEBUG "status": "success", 05:20:56 DEBUG "data": { 05:20:56 DEBUG "resultType": "vector", 05:20:56 DEBUG "result": [ 05:20:56 DEBUG { 05:20:56 DEBUG "metric": { 05:20:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:20:56 DEBUG }, 05:20:56 DEBUG "value": [ 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG [...] 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:20:56 DEBUG }, 05:20:56 DEBUG "value": [ 05:20:56 DEBUG 1713500456, 05:20:56 DEBUG "0" 05:20:56 DEBUG ] 05:20:56 DEBUG } 05:20:56 DEBUG ] 05:20:56 DEBUG } 05:20:56 DEBUG } 05:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713500456" 05:20:56 INFO [http_cmd]: http status code OK 05:20:56 DEBUG --- status code --- 05:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:20:56 DEBUG --- http response --- 05:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:20:56 DEBUG ----- output ----- 05:20:56 DEBUG { 05:20:56 DEBUG "status": "success", 05:20:56 DEBUG "data": { 05:20:56 DEBUG "resultType": "vector", 05:20:56 DEBUG "result": [ 05:20:56 DEBUG { 05:20:56 DEBUG "metric": { 05:20:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:20:56 DEBUG }, 05:20:56 DEBUG "value": [ 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG [...] 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:20:56 DEBUG }, 05:20:56 DEBUG "value": [ 05:20:56 DEBUG 1713500456, 05:20:56 DEBUG "55.172" 05:20:56 DEBUG ] 05:20:56 DEBUG } 05:20:56 DEBUG ] 05:20:56 DEBUG } 05:20:56 DEBUG } 05:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500456" 05:20:56 INFO [http_cmd]: http status code OK 05:20:56 DEBUG --- status code --- 05:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:20:56 DEBUG --- http response --- 05:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:20:56 DEBUG ----- output ----- 05:20:56 DEBUG { 05:20:56 DEBUG "status": "success", 05:20:56 DEBUG "data": { 05:20:56 DEBUG "resultType": "vector", 05:20:56 DEBUG "result": [ 05:20:56 DEBUG { 05:20:56 DEBUG "metric": { 05:20:56 DEBUG "instance": "10.142.0.107:9100" 05:20:56 DEBUG }, 05:20:56 DEBUG "value": [ 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG [...] 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG "instance": "10.142.0.43:9100" 05:20:56 DEBUG }, 05:20:56 DEBUG "value": [ 05:20:56 DEBUG 1713500456, 05:20:56 DEBUG "472395881472" 05:20:56 DEBUG ] 05:20:56 DEBUG } 05:20:56 DEBUG ] 05:20:56 DEBUG } 05:20:56 DEBUG } 05:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713500456" 05:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500456" 05:20:56 INFO [http_cmd]: http status code OK 05:20:56 DEBUG --- status code --- 05:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:20:56 DEBUG --- http response --- 05:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:20:56 DEBUG ----- output ----- 05:20:56 DEBUG { 05:20:56 DEBUG "status": "success", 05:20:56 DEBUG "data": { 05:20:56 DEBUG "resultType": "vector", 05:20:56 DEBUG "result": [ 05:20:56 DEBUG { 05:20:56 DEBUG "metric": { 05:20:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:20:56 DEBUG }, 05:20:56 DEBUG "value": [ 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG [...] 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:20:56 DEBUG }, 05:20:56 DEBUG "value": [ 05:20:56 DEBUG 1713500456, 05:20:56 DEBUG "2209267712" 05:20:56 DEBUG ] 05:20:56 DEBUG } 05:20:56 DEBUG ] 05:20:56 DEBUG } 05:20:56 DEBUG } 05:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500456" 05:20:56 INFO [http_cmd]: http status code OK 05:20:56 DEBUG --- status code --- 05:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:20:56 DEBUG --- http response --- 05:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:20:56 DEBUG ----- output ----- 05:20:56 DEBUG { 05:20:56 DEBUG "status": "success", 05:20:56 DEBUG "data": { 05:20:56 DEBUG "resultType": "vector", 05:20:56 DEBUG "result": [ 05:20:56 DEBUG { 05:20:56 DEBUG "metric": { 05:20:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:20:56 DEBUG }, 05:20:56 DEBUG "value": [ 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG [...] 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:20:56 DEBUG }, 05:20:56 DEBUG "value": [ 05:20:56 DEBUG 1713500456, 05:20:56 DEBUG "2000.3999999999999" 05:20:56 DEBUG ] 05:20:56 DEBUG } 05:20:56 DEBUG ] 05:20:56 DEBUG } 05:20:56 DEBUG } 05:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500456" 05:20:56 INFO [http_cmd]: http status code OK 05:20:56 DEBUG --- status code --- 05:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:20:56 DEBUG --- http response --- 05:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:20:56 DEBUG ----- output ----- 05:20:56 DEBUG { 05:20:56 DEBUG "status": "success", 05:20:56 DEBUG "data": { 05:20:56 DEBUG "resultType": "vector", 05:20:56 DEBUG "result": [ 05:20:56 DEBUG { 05:20:56 DEBUG "metric": { 05:20:56 DEBUG "instance": "10.142.0.107:9100" 05:20:56 DEBUG }, 05:20:56 DEBUG "value": [ 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG [...] 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG "instance": "10.142.0.43:9100" 05:20:56 DEBUG }, 05:20:56 DEBUG "value": [ 05:20:56 DEBUG 1713500456, 05:20:56 DEBUG "0.6918999999999926" 05:20:56 DEBUG ] 05:20:56 DEBUG } 05:20:56 DEBUG ] 05:20:56 DEBUG } 05:20:56 DEBUG } 05:20:56 INFO [http_cmd]: http status code OK 05:20:56 DEBUG --- status code --- 05:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:20:56 DEBUG --- http response --- 05:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:20:56 DEBUG ----- output ----- 05:20:56 DEBUG { 05:20:56 DEBUG "status": "success", 05:20:56 DEBUG "data": { 05:20:56 DEBUG "resultType": "vector", 05:20:56 DEBUG "result": [ 05:20:56 DEBUG { 05:20:56 DEBUG "metric": { 05:20:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:20:56 DEBUG }, 05:20:56 DEBUG "value": [ 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG [...] 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:20:56 DEBUG }, 05:20:56 DEBUG "value": [ 05:20:56 DEBUG 1713500456, 05:20:56 DEBUG "0" 05:20:56 DEBUG ] 05:20:56 DEBUG } 05:20:56 DEBUG ] 05:20:56 DEBUG } 05:20:56 DEBUG } 05:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500456" 05:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500456" 05:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500456" 05:20:56 INFO [http_cmd]: http status code OK 05:20:56 DEBUG --- status code --- 05:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:20:56 DEBUG --- http response --- 05:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:20:56 DEBUG ----- output ----- 05:20:56 DEBUG { 05:20:56 DEBUG "status": "success", 05:20:56 DEBUG "data": { 05:20:56 DEBUG "resultType": "vector", 05:20:56 DEBUG "result": [ 05:20:56 DEBUG { 05:20:56 DEBUG "metric": { 05:20:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:20:56 DEBUG }, 05:20:56 DEBUG "value": [ 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG [...] 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:20:56 DEBUG }, 05:20:56 DEBUG "value": [ 05:20:56 DEBUG 1713500456, 05:20:56 DEBUG "2169.6666666666665" 05:20:56 DEBUG ] 05:20:56 DEBUG } 05:20:56 DEBUG ] 05:20:56 DEBUG } 05:20:56 DEBUG } 05:20:56 INFO [http_cmd]: http status code OK 05:20:56 DEBUG --- status code --- 05:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:20:56 DEBUG --- http response --- 05:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:20:56 DEBUG ----- output ----- 05:20:56 DEBUG { 05:20:56 DEBUG "status": "success", 05:20:56 DEBUG "data": { 05:20:56 DEBUG "resultType": "vector", 05:20:56 DEBUG "result": [ 05:20:56 DEBUG { 05:20:56 DEBUG "metric": { 05:20:56 DEBUG "instance": "10.142.0.107:9100" 05:20:56 DEBUG }, 05:20:56 DEBUG "value": [ 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG [...] 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG "instance": "10.142.0.43:9100" 05:20:56 DEBUG }, 05:20:56 DEBUG "value": [ 05:20:56 DEBUG 1713500456, 05:20:56 DEBUG "0.400333333333335" 05:20:56 DEBUG ] 05:20:56 DEBUG } 05:20:56 DEBUG ] 05:20:56 DEBUG } 05:20:56 DEBUG } 05:20:56 INFO [http_cmd]: http status code OK 05:20:56 DEBUG --- status code --- 05:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:20:56 DEBUG --- http response --- 05:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:20:56 DEBUG ----- output ----- 05:20:56 DEBUG { 05:20:56 DEBUG "status": "success", 05:20:56 DEBUG "data": { 05:20:56 DEBUG "resultType": "vector", 05:20:56 DEBUG "result": [ 05:20:56 DEBUG { 05:20:56 DEBUG "metric": { 05:20:56 DEBUG "pod": "ds-idrepo-0" 05:20:56 DEBUG }, 05:20:56 DEBUG "value": [ 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG [...] 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG 05:20:56 DEBUG "pod": "ds-idrepo-2" 05:20:56 DEBUG }, 05:20:56 DEBUG "value": [ 05:20:56 DEBUG 1713500456, 05:20:56 DEBUG "0" 05:20:56 DEBUG ] 05:20:56 DEBUG } 05:20:56 DEBUG ] 05:20:56 DEBUG } 05:20:56 DEBUG } 05:20:57 INFO 05:20:57 INFO [loop_until]: kubectl --namespace=xlou top pods 05:20:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:20:57 INFO [loop_until]: OK (rc = 0) 05:20:57 DEBUG --- stdout --- 05:20:57 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 13422m 9142Mi am-6fd554f8f7-hvvkw 13167m 8897Mi am-6fd554f8f7-pmj4b 14596m 8789Mi ds-cts-0 18370m 12832Mi ds-cts-1 18052m 12659Mi ds-cts-2 18775m 12706Mi ds-idrepo-0 5715m 3267Mi ds-idrepo-1 5768m 3198Mi ds-idrepo-2 5651m 3066Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 4m 1215Mi lodemon-6b89d774d5-rf7sk 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7154m 2014Mi 05:20:57 DEBUG --- stderr --- 05:20:57 DEBUG 05:20:58 INFO 05:20:58 INFO [loop_until]: kubectl --namespace=xlou top node 05:20:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:20:58 INFO [loop_until]: OK (rc = 0) 05:20:58 DEBUG --- stdout --- 05:20:58 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 71m 0% 6247Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5831m 19% 6713Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13098m 43% 11613Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 85m 0% 3251Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 18208m 60% 15237Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 159m 0% 5182Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 13639m 45% 11915Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 11204m 37% 2735Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 15207m 50% 10998Mi 9% gke-xlou-cdm-ds-0f601751-j488 6117m 20% 6172Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 17604m 58% 16682Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 6363m 21% 6868Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 18945m 63% 14564Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 7333m 46% 5735Mi 9% 05:20:58 DEBUG --- stderr --- 05:20:58 DEBUG 05:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713500516" 05:21:56 INFO [http_cmd]: http status code OK 05:21:56 DEBUG --- status code --- 05:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:21:56 DEBUG --- http response --- 05:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:21:56 DEBUG ----- output ----- 05:21:56 DEBUG { 05:21:56 DEBUG "status": "success", 05:21:56 DEBUG "data": { 05:21:56 DEBUG "resultType": "vector", 05:21:56 DEBUG "result": [ 05:21:56 DEBUG { 05:21:56 DEBUG "metric": { 05:21:56 DEBUG "pod": "ds-idrepo-0" 05:21:56 DEBUG }, 05:21:56 DEBUG "value": [ 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG [...] 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG "pod": "ds-cts-2" 05:21:56 DEBUG }, 05:21:56 DEBUG "value": [ 05:21:56 DEBUG 1713500516, 05:21:56 DEBUG "0" 05:21:56 DEBUG ] 05:21:56 DEBUG } 05:21:56 DEBUG ] 05:21:56 DEBUG } 05:21:56 DEBUG } 05:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713500516" 05:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500516" 05:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500516" 05:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500516" 05:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500516" 05:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500516" 05:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500516" 05:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713500516" 05:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500516" 05:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500516" 05:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500516" 05:21:56 ERROR [http_cmd]: ERROR 05:21:56 INFO [http_cmd]: http status code OK 05:21:56 ERROR --- status code --- 05:21:56 INFO [http_cmd]: http status code OK 05:21:56 INFO [http_cmd]: http status code OK 05:21:56 DEBUG --- status code --- 05:21:56 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:21:56 DEBUG --- status code --- 05:21:56 DEBUG --- status code --- 05:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:21:56 ERROR --- http response --- 05:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:21:56 DEBUG --- http response --- 05:21:56 DEBUG --- http response --- 05:21:56 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:21:56 DEBUG --- http response --- 05:21:56 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:21:56 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500516" 05:21:56 DEBUG ----- output ----- 05:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500516" 05:21:56 DEBUG ----- output ----- 05:21:56 DEBUG { 05:21:56 INFO [http_cmd]: http status code OK 05:21:56 INFO [http_cmd]: http status code OK 05:21:56 INFO [http_cmd]: http status code OK 05:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500516" 05:21:56 INFO [http_cmd]: http status code OK 05:21:56 DEBUG --- status code --- 05:21:56 INFO [http_cmd]: http status code OK 05:21:56 DEBUG { 05:21:56 DEBUG "status": "success", 05:21:56 DEBUG --- status code --- 05:21:56 DEBUG --- status code --- 05:21:56 DEBUG --- status code --- 05:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500516" 05:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:21:56 INFO [http_cmd]: http status code OK 05:21:56 DEBUG --- status code --- 05:21:56 INFO [http_cmd]: http status code OK 05:21:56 DEBUG "status": "success", 05:21:56 DEBUG "data": { 05:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:21:56 DEBUG --- http response --- 05:21:56 DEBUG --- status code --- 05:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:21:56 DEBUG --- status code --- 05:21:56 DEBUG "data": { 05:21:56 DEBUG "resultType": "vector", 05:21:56 DEBUG "result": [ 05:21:56 DEBUG --- http response --- 05:21:56 DEBUG --- http response --- 05:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:21:56 DEBUG --- http response --- 05:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:21:56 DEBUG "resultType": "vector", 05:21:56 DEBUG --- http response --- 05:21:56 DEBUG { 05:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:21:56 DEBUG --- http response --- 05:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:21:56 INFO [http_cmd]: http status code OK 05:21:56 DEBUG ----- output ----- 05:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:21:56 DEBUG --- http response --- 05:21:56 DEBUG "result": [ 05:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:21:56 DEBUG "metric": { 05:21:56 DEBUG ----- output ----- 05:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:21:56 DEBUG ----- output ----- 05:21:56 DEBUG --- status code --- 05:21:56 DEBUG { 05:21:56 DEBUG ----- output ----- 05:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:21:56 DEBUG { 05:21:56 INFO [http_cmd]: http status code OK 05:21:56 DEBUG ----- output ----- 05:21:56 DEBUG "pod": "ds-idrepo-0" 05:21:56 DEBUG { 05:21:56 DEBUG ----- output ----- 05:21:56 DEBUG { 05:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:21:56 DEBUG "status": "success", 05:21:56 DEBUG { 05:21:56 DEBUG ----- output ----- 05:21:56 DEBUG "metric": { 05:21:56 DEBUG --- status code --- 05:21:56 DEBUG { 05:21:56 DEBUG }, 05:21:56 DEBUG "status": "success", 05:21:56 DEBUG { 05:21:56 DEBUG "status": "success", 05:21:56 DEBUG --- http response --- 05:21:56 DEBUG "data": { 05:21:56 DEBUG "status": "success", 05:21:56 DEBUG { 05:21:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:21:56 DEBUG "status": "success", 05:21:56 DEBUG "value": [ 05:21:56 DEBUG "data": { 05:21:56 DEBUG "status": "success", 05:21:56 DEBUG "data": { 05:21:56 DEBUG "resultType": "vector", 05:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:21:56 INFO [http_cmd]: http status code OK 05:21:56 DEBUG "data": { 05:21:56 DEBUG "status": "success", 05:21:56 DEBUG }, 05:21:56 DEBUG --- http response --- 05:21:56 DEBUG "data": { 05:21:56 DEBUG 05:21:56 DEBUG "resultType": "vector", 05:21:56 DEBUG "data": { 05:21:56 DEBUG "resultType": "vector", 05:21:56 DEBUG "result": [ 05:21:56 DEBUG ----- output ----- 05:21:56 DEBUG --- status code --- 05:21:56 DEBUG "resultType": "vector", 05:21:56 DEBUG "data": { 05:21:56 DEBUG "value": [ 05:21:56 DEBUG "resultType": "vector", 05:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:21:56 DEBUG 05:21:56 DEBUG "result": [ 05:21:56 DEBUG "resultType": "vector", 05:21:56 INFO [http_cmd]: http status code OK 05:21:56 DEBUG "result": [ 05:21:56 DEBUG { 05:21:56 DEBUG { 05:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:21:56 DEBUG "result": [ 05:21:56 DEBUG "resultType": "vector", 05:21:56 DEBUG 05:21:56 DEBUG "result": [ 05:21:56 DEBUG ----- output ----- 05:21:56 DEBUG 05:21:56 DEBUG { 05:21:56 DEBUG "result": [ 05:21:56 DEBUG --- status code --- 05:21:56 DEBUG { 05:21:56 DEBUG "metric": { 05:21:56 DEBUG "status": "success", 05:21:56 DEBUG --- http response --- 05:21:56 DEBUG { 05:21:56 DEBUG "result": [ 05:21:56 DEBUG { 05:21:56 DEBUG 05:21:56 DEBUG { 05:21:56 DEBUG [...] 05:21:56 DEBUG "metric": { 05:21:56 DEBUG { 05:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:21:56 DEBUG "metric": { 05:21:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:21:56 DEBUG "data": { 05:21:56 DEBUG "metric": { 05:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:21:56 DEBUG { 05:21:56 DEBUG "metric": { 05:21:56 DEBUG 05:21:56 DEBUG "status": "success", 05:21:56 DEBUG 05:21:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:21:56 DEBUG "metric": { 05:21:56 DEBUG --- http response --- 05:21:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:21:56 DEBUG }, 05:21:56 DEBUG "resultType": "vector", 05:21:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:21:56 DEBUG ----- output ----- 05:21:56 DEBUG "metric": { 05:21:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:21:56 DEBUG [...] 05:21:56 DEBUG "data": { 05:21:56 DEBUG 05:21:56 DEBUG }, 05:21:56 DEBUG "pod": "ds-idrepo-0" 05:21:56 DEBUG }, 05:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:21:56 DEBUG "value": [ 05:21:56 DEBUG "result": [ 05:21:56 DEBUG }, 05:21:56 DEBUG { 05:21:56 DEBUG "pod": "ds-cts-0" 05:21:56 DEBUG }, 05:21:56 DEBUG 05:21:56 DEBUG "resultType": "vector", 05:21:56 DEBUG 05:21:56 DEBUG "value": [ 05:21:56 DEBUG }, 05:21:56 DEBUG "value": [ 05:21:56 DEBUG ----- output ----- 05:21:56 DEBUG 05:21:56 DEBUG { 05:21:56 DEBUG "value": [ 05:21:56 DEBUG "status": "success", 05:21:56 DEBUG }, 05:21:56 DEBUG "value": [ 05:21:56 DEBUG 05:21:56 DEBUG "result": [ 05:21:56 DEBUG "pod": "ds-idrepo-2" 05:21:56 DEBUG 05:21:56 DEBUG "value": [ 05:21:56 DEBUG 05:21:56 DEBUG { 05:21:56 DEBUG 05:21:56 DEBUG "metric": { 05:21:56 DEBUG 05:21:56 DEBUG "data": { 05:21:56 DEBUG "value": [ 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG { 05:21:56 DEBUG "metric": { 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG "status": "success", 05:21:56 DEBUG 05:21:56 DEBUG "instance": "10.142.0.107:9100" 05:21:56 DEBUG 05:21:56 DEBUG "resultType": "vector", 05:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500516" 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:21:56 DEBUG "pod": "ds-cts-0" 05:21:56 DEBUG }, 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG "data": { 05:21:56 DEBUG [...] 05:21:56 DEBUG }, 05:21:56 DEBUG 05:21:56 DEBUG "result": [ 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG }, 05:21:56 DEBUG }, 05:21:56 DEBUG "value": [ 05:21:56 DEBUG [...] 05:21:56 DEBUG 05:21:56 DEBUG [...] 05:21:56 DEBUG "resultType": "vector", 05:21:56 DEBUG 05:21:56 DEBUG "value": [ 05:21:56 DEBUG [...] 05:21:56 DEBUG { 05:21:56 DEBUG 05:21:56 DEBUG [...] 05:21:56 DEBUG "value": [ 05:21:56 DEBUG "value": [ 05:21:56 DEBUG 1713500516, 05:21:56 DEBUG 05:21:56 DEBUG [...] 05:21:56 DEBUG 05:21:56 DEBUG "result": [ 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG "metric": { 05:21:56 DEBUG [...] 05:21:56 DEBUG 05:21:56 DEBUG 1713500516, 05:21:56 DEBUG 05:21:56 DEBUG "0" 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG { 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG "pod": "ds-cts-0" 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG "3545134" 05:21:56 DEBUG 05:21:56 DEBUG ] 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG "metric": { 05:21:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG }, 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG ] 05:21:56 DEBUG 05:21:56 DEBUG } 05:21:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:21:56 DEBUG 05:21:56 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:21:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:21:56 DEBUG }, 05:21:56 DEBUG [...] 05:21:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:21:56 DEBUG "value": [ 05:21:56 DEBUG 05:21:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:21:56 DEBUG } 05:21:56 DEBUG ] 05:21:56 DEBUG } 05:21:56 DEBUG }, 05:21:56 DEBUG "pod": "ds-idrepo-2" 05:21:56 DEBUG }, 05:21:56 DEBUG }, 05:21:56 DEBUG "value": [ 05:21:56 DEBUG 05:21:56 DEBUG }, 05:21:56 DEBUG 05:21:56 DEBUG "pod": "ds-cts-2" 05:21:56 DEBUG }, 05:21:56 DEBUG [...] 05:21:56 DEBUG ] 05:21:56 DEBUG } 05:21:56 DEBUG "value": [ 05:21:56 DEBUG }, 05:21:56 DEBUG "value": [ 05:21:56 DEBUG "value": [ 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG "value": [ 05:21:56 DEBUG 05:21:56 DEBUG }, 05:21:56 DEBUG "value": [ 05:21:56 DEBUG 05:21:56 DEBUG } 05:21:56 DEBUG 1713500516, 05:21:56 DEBUG "value": [ 05:21:56 DEBUG 1713500516, 05:21:56 DEBUG 1713500516, 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG [...] 05:21:56 DEBUG 05:21:56 DEBUG "value": [ 05:21:56 DEBUG 1713500516, 05:21:56 DEBUG 05:21:56 DEBUG } 05:21:56 DEBUG "2164.9666666666667" 05:21:56 DEBUG 1713500516, 05:21:56 DEBUG "7.068497396068955" 05:21:56 DEBUG ] 05:21:56 DEBUG "0" 05:21:56 DEBUG ] 05:21:56 DEBUG 05:21:56 DEBUG [...] 05:21:56 DEBUG 1713500516, 05:21:56 DEBUG "7.10776218855287" 05:21:56 DEBUG 05:21:56 DEBUG ] 05:21:56 DEBUG "0" 05:21:56 DEBUG ] 05:21:56 DEBUG } 05:21:56 DEBUG 05:21:56 DEBUG } 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG "453.7" 05:21:56 DEBUG ] 05:21:56 DEBUG "pod": "ds-cts-2" 05:21:56 DEBUG } 05:21:56 DEBUG } 05:21:56 DEBUG ] 05:21:56 DEBUG ] 05:21:56 DEBUG "instance": "10.142.0.43:9100" 05:21:56 DEBUG ] 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG ] 05:21:56 DEBUG } 05:21:56 DEBUG }, 05:21:56 DEBUG ] 05:21:56 DEBUG 1713500516, 05:21:56 DEBUG } 05:21:56 DEBUG } 05:21:56 DEBUG } 05:21:56 DEBUG }, 05:21:56 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:21:56 DEBUG 05:21:56 DEBUG } 05:21:56 DEBUG ] 05:21:56 DEBUG } 05:21:56 DEBUG } 05:21:56 DEBUG "2165.3" 05:21:56 DEBUG } 05:21:56 DEBUG } 05:21:56 DEBUG } 05:21:56 DEBUG }, 05:21:56 DEBUG "pod": "ds-cts-2" 05:21:56 DEBUG ] 05:21:56 DEBUG "value": [ 05:21:56 DEBUG } 05:21:56 DEBUG } 05:21:56 DEBUG ] 05:21:56 DEBUG "value": [ 05:21:56 DEBUG "value": [ 05:21:56 DEBUG }, 05:21:56 DEBUG } 05:21:56 DEBUG 1713500516, 05:21:56 INFO [http_cmd]: http status code OK 05:21:56 DEBUG } 05:21:56 DEBUG 1713500516, 05:21:56 DEBUG 1713500516, 05:21:56 DEBUG "value": [ 05:21:56 DEBUG } 05:21:56 DEBUG "0.00042788000000000006" 05:21:56 DEBUG --- status code --- 05:21:56 DEBUG ] 05:21:56 DEBUG "63987456.06666666" 05:21:56 DEBUG "28987.700105074484" 05:21:56 DEBUG 1713500516, 05:21:56 DEBUG ] 05:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:21:56 DEBUG } 05:21:56 DEBUG ] 05:21:56 DEBUG ] 05:21:56 DEBUG "0" 05:21:56 DEBUG } 05:21:56 DEBUG --- http response --- 05:21:56 DEBUG } 05:21:56 DEBUG } 05:21:56 DEBUG } 05:21:56 DEBUG ] 05:21:56 DEBUG } 05:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500516" 05:21:56 DEBUG ] 05:21:56 DEBUG ] 05:21:56 DEBUG ] 05:21:56 DEBUG } 05:21:56 DEBUG ----- output ----- 05:21:56 DEBUG } 05:21:56 DEBUG } 05:21:56 DEBUG } 05:21:56 DEBUG { 05:21:56 DEBUG } 05:21:56 DEBUG ] 05:21:56 DEBUG } 05:21:56 DEBUG "status": "success", 05:21:56 DEBUG } 05:21:56 DEBUG "data": { 05:21:56 DEBUG } 05:21:56 DEBUG "resultType": "vector", 05:21:56 DEBUG "result": [ 05:21:56 DEBUG { 05:21:56 DEBUG "metric": { 05:21:56 DEBUG "instance": "10.142.0.107:9100" 05:21:56 DEBUG }, 05:21:56 DEBUG "value": [ 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG [...] 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG "instance": "10.142.0.43:9100" 05:21:56 DEBUG }, 05:21:56 DEBUG "value": [ 05:21:56 DEBUG 1713500516, 05:21:56 DEBUG "63989240.56666666" 05:21:56 DEBUG ] 05:21:56 DEBUG } 05:21:56 DEBUG ] 05:21:56 DEBUG } 05:21:56 DEBUG } 05:21:56 INFO [http_cmd]: http status code OK 05:21:56 DEBUG --- status code --- 05:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:21:56 DEBUG --- http response --- 05:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:21:56 DEBUG ----- output ----- 05:21:56 DEBUG { 05:21:56 DEBUG "status": "success", 05:21:56 DEBUG "data": { 05:21:56 DEBUG "resultType": "vector", 05:21:56 DEBUG "result": [ 05:21:56 DEBUG { 05:21:56 DEBUG "metric": { 05:21:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:21:56 DEBUG }, 05:21:56 DEBUG "value": [ 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG [...] 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:21:56 DEBUG }, 05:21:56 DEBUG "value": [ 05:21:56 DEBUG 1713500516, 05:21:56 DEBUG "0" 05:21:56 DEBUG ] 05:21:56 DEBUG } 05:21:56 DEBUG ] 05:21:56 DEBUG } 05:21:56 DEBUG } 05:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713500516" 05:21:56 INFO [http_cmd]: http status code OK 05:21:56 DEBUG --- status code --- 05:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:21:56 DEBUG --- http response --- 05:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:21:56 DEBUG ----- output ----- 05:21:56 DEBUG { 05:21:56 DEBUG "status": "success", 05:21:56 DEBUG "data": { 05:21:56 DEBUG "resultType": "vector", 05:21:56 DEBUG "result": [ 05:21:56 DEBUG { 05:21:56 DEBUG "metric": { 05:21:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:21:56 DEBUG }, 05:21:56 DEBUG "value": [ 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG [...] 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:21:56 DEBUG }, 05:21:56 DEBUG "value": [ 05:21:56 DEBUG 1713500516, 05:21:56 DEBUG "55.172" 05:21:56 DEBUG ] 05:21:56 DEBUG } 05:21:56 DEBUG ] 05:21:56 DEBUG } 05:21:56 DEBUG } 05:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500516" 05:21:56 INFO [http_cmd]: http status code OK 05:21:56 DEBUG --- status code --- 05:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:21:56 DEBUG --- http response --- 05:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:21:56 DEBUG ----- output ----- 05:21:56 DEBUG { 05:21:56 DEBUG "status": "success", 05:21:56 DEBUG "data": { 05:21:56 DEBUG "resultType": "vector", 05:21:56 DEBUG "result": [ 05:21:56 DEBUG { 05:21:56 DEBUG "metric": { 05:21:56 DEBUG "instance": "10.142.0.107:9100" 05:21:56 DEBUG }, 05:21:56 DEBUG "value": [ 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG [...] 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG "instance": "10.142.0.43:9100" 05:21:56 DEBUG }, 05:21:56 DEBUG "value": [ 05:21:56 DEBUG 1713500516, 05:21:56 DEBUG "475506971648" 05:21:56 DEBUG ] 05:21:56 DEBUG } 05:21:56 DEBUG ] 05:21:56 DEBUG } 05:21:56 DEBUG } 05:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713500516" 05:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500516" 05:21:56 INFO [http_cmd]: http status code OK 05:21:56 DEBUG --- status code --- 05:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:21:56 DEBUG --- http response --- 05:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:21:56 DEBUG ----- output ----- 05:21:56 DEBUG { 05:21:56 DEBUG "status": "success", 05:21:56 DEBUG "data": { 05:21:56 DEBUG "resultType": "vector", 05:21:56 DEBUG "result": [ 05:21:56 DEBUG { 05:21:56 DEBUG "metric": { 05:21:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:21:56 DEBUG }, 05:21:56 DEBUG "value": [ 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG [...] 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:21:56 DEBUG }, 05:21:56 DEBUG "value": [ 05:21:56 DEBUG 1713500516, 05:21:56 DEBUG "2209443840" 05:21:56 DEBUG ] 05:21:56 DEBUG } 05:21:56 DEBUG ] 05:21:56 DEBUG } 05:21:56 DEBUG } 05:21:56 INFO [http_cmd]: http status code OK 05:21:56 DEBUG --- status code --- 05:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:21:56 DEBUG --- http response --- 05:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:21:56 DEBUG ----- output ----- 05:21:56 DEBUG { 05:21:56 DEBUG "status": "success", 05:21:56 DEBUG "data": { 05:21:56 DEBUG "resultType": "vector", 05:21:56 DEBUG "result": [ 05:21:56 DEBUG { 05:21:56 DEBUG "metric": { 05:21:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:21:56 DEBUG }, 05:21:56 DEBUG "value": [ 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG [...] 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:21:56 DEBUG }, 05:21:56 DEBUG "value": [ 05:21:56 DEBUG 1713500516, 05:21:56 DEBUG "2002.9333333333334" 05:21:56 DEBUG ] 05:21:56 DEBUG } 05:21:56 DEBUG ] 05:21:56 DEBUG } 05:21:56 DEBUG } 05:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500516" 05:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500516" 05:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500516" 05:21:56 INFO [http_cmd]: http status code OK 05:21:56 DEBUG --- status code --- 05:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:21:56 DEBUG --- http response --- 05:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:21:56 DEBUG ----- output ----- 05:21:56 DEBUG { 05:21:56 DEBUG "status": "success", 05:21:56 DEBUG "data": { 05:21:56 DEBUG "resultType": "vector", 05:21:56 DEBUG "result": [ 05:21:56 DEBUG { 05:21:56 DEBUG "metric": { 05:21:56 DEBUG "instance": "10.142.0.107:9100" 05:21:56 DEBUG }, 05:21:56 DEBUG "value": [ 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG [...] 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG "instance": "10.142.0.43:9100" 05:21:56 DEBUG }, 05:21:56 DEBUG "value": [ 05:21:56 DEBUG 1713500516, 05:21:56 DEBUG "0.6531333333333398" 05:21:56 DEBUG ] 05:21:56 DEBUG } 05:21:56 DEBUG ] 05:21:56 DEBUG } 05:21:56 DEBUG } 05:21:56 INFO [http_cmd]: http status code OK 05:21:56 INFO [http_cmd]: http status code OK 05:21:56 DEBUG --- status code --- 05:21:56 DEBUG --- status code --- 05:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:21:56 DEBUG --- http response --- 05:21:56 DEBUG --- http response --- 05:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:21:56 DEBUG ----- output ----- 05:21:56 DEBUG { 05:21:56 DEBUG "status": "success", 05:21:56 DEBUG "data": { 05:21:56 DEBUG "resultType": "vector", 05:21:56 DEBUG "result": [ 05:21:56 DEBUG { 05:21:56 DEBUG "metric": { 05:21:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:21:56 DEBUG }, 05:21:56 DEBUG "value": [ 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG [...] 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:21:56 DEBUG }, 05:21:56 DEBUG "value": [ 05:21:56 DEBUG 1713500516, 05:21:56 DEBUG "2165.266666666667" 05:21:56 DEBUG ] 05:21:56 DEBUG } 05:21:56 DEBUG ] 05:21:56 DEBUG } 05:21:56 DEBUG } 05:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:21:56 DEBUG ----- output ----- 05:21:56 DEBUG { 05:21:56 DEBUG "status": "success", 05:21:56 DEBUG "data": { 05:21:56 DEBUG "resultType": "vector", 05:21:56 DEBUG "result": [ 05:21:56 DEBUG { 05:21:56 DEBUG "metric": { 05:21:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:21:56 DEBUG }, 05:21:56 DEBUG "value": [ 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG [...] 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:21:56 DEBUG }, 05:21:56 DEBUG "value": [ 05:21:56 DEBUG 1713500516, 05:21:56 DEBUG "0" 05:21:56 DEBUG ] 05:21:56 DEBUG } 05:21:56 DEBUG ] 05:21:56 DEBUG } 05:21:56 DEBUG } 05:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500516" 05:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500516" 05:21:56 INFO [http_cmd]: http status code OK 05:21:56 DEBUG --- status code --- 05:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:21:56 DEBUG --- http response --- 05:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:21:56 DEBUG ----- output ----- 05:21:56 DEBUG { 05:21:56 DEBUG "status": "success", 05:21:56 DEBUG "data": { 05:21:56 DEBUG "resultType": "vector", 05:21:56 DEBUG "result": [ 05:21:56 DEBUG { 05:21:56 DEBUG "metric": { 05:21:56 DEBUG "pod": "ds-idrepo-0" 05:21:56 DEBUG }, 05:21:56 DEBUG "value": [ 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG [...] 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG "pod": "ds-idrepo-2" 05:21:56 DEBUG }, 05:21:56 DEBUG "value": [ 05:21:56 DEBUG 1713500516, 05:21:56 DEBUG "0" 05:21:56 DEBUG ] 05:21:56 DEBUG } 05:21:56 DEBUG ] 05:21:56 DEBUG } 05:21:56 DEBUG } 05:21:56 INFO [http_cmd]: http status code OK 05:21:56 DEBUG --- status code --- 05:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:21:56 DEBUG --- http response --- 05:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:21:56 DEBUG ----- output ----- 05:21:56 DEBUG { 05:21:56 DEBUG "status": "success", 05:21:56 DEBUG "data": { 05:21:56 DEBUG "resultType": "vector", 05:21:56 DEBUG "result": [ 05:21:56 DEBUG { 05:21:56 DEBUG "metric": { 05:21:56 DEBUG "instance": "10.142.0.107:9100" 05:21:56 DEBUG }, 05:21:56 DEBUG "value": [ 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG [...] 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG 05:21:56 DEBUG "instance": "10.142.0.43:9100" 05:21:56 DEBUG }, 05:21:56 DEBUG "value": [ 05:21:56 DEBUG 1713500516, 05:21:56 DEBUG "0.37133333333333285" 05:21:56 DEBUG ] 05:21:56 DEBUG } 05:21:56 DEBUG ] 05:21:56 DEBUG } 05:21:56 DEBUG } 05:21:58 INFO 05:21:58 INFO [loop_until]: kubectl --namespace=xlou top pods 05:21:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:21:58 INFO [loop_until]: OK (rc = 0) 05:21:58 DEBUG --- stdout --- 05:21:58 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 13857m 9140Mi am-6fd554f8f7-hvvkw 13596m 8898Mi am-6fd554f8f7-pmj4b 14252m 8786Mi ds-cts-0 19143m 13036Mi ds-cts-1 17586m 12899Mi ds-cts-2 18588m 12703Mi ds-idrepo-0 5912m 3267Mi ds-idrepo-1 5893m 3198Mi ds-idrepo-2 5575m 3066Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 3m 1215Mi lodemon-6b89d774d5-rf7sk 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7090m 2017Mi 05:21:58 DEBUG --- stderr --- 05:21:58 DEBUG 05:21:58 INFO 05:21:58 INFO [loop_until]: kubectl --namespace=xlou top node 05:21:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:21:58 INFO [loop_until]: OK (rc = 0) 05:21:58 DEBUG --- stdout --- 05:21:58 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 69m 0% 6250Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5833m 19% 6715Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13191m 44% 11612Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 3251Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 18792m 62% 15497Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 152m 0% 5198Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 13758m 46% 11921Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 11211m 37% 2732Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 15237m 51% 10990Mi 9% gke-xlou-cdm-ds-0f601751-j488 6053m 20% 6172Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 17813m 59% 16916Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6392m 21% 6874Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 18693m 62% 14742Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 7260m 45% 5741Mi 9% 05:21:58 DEBUG --- stderr --- 05:21:58 DEBUG 05:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713500576" 05:22:56 INFO [http_cmd]: http status code OK 05:22:56 DEBUG --- status code --- 05:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:22:56 DEBUG --- http response --- 05:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:22:56 DEBUG ----- output ----- 05:22:56 DEBUG { 05:22:56 DEBUG "status": "success", 05:22:56 DEBUG "data": { 05:22:56 DEBUG "resultType": "vector", 05:22:56 DEBUG "result": [ 05:22:56 DEBUG { 05:22:56 DEBUG "metric": { 05:22:56 DEBUG "pod": "ds-idrepo-0" 05:22:56 DEBUG }, 05:22:56 DEBUG "value": [ 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG [...] 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG "pod": "ds-cts-2" 05:22:56 DEBUG }, 05:22:56 DEBUG "value": [ 05:22:56 DEBUG 1713500576, 05:22:56 DEBUG "0" 05:22:56 DEBUG ] 05:22:56 DEBUG } 05:22:56 DEBUG ] 05:22:56 DEBUG } 05:22:56 DEBUG } 05:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713500576" 05:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500576" 05:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500576" 05:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500576" 05:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500576" 05:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500576" 05:22:56 INFO [http_cmd]: http status code OK 05:22:56 INFO [http_cmd]: http status code OK 05:22:56 DEBUG --- status code --- 05:22:56 INFO [http_cmd]: http status code OK 05:22:56 ERROR [http_cmd]: ERROR 05:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500576" 05:22:56 DEBUG --- status code --- 05:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:22:56 DEBUG --- status code --- 05:22:56 ERROR --- status code --- 05:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:22:56 DEBUG --- http response --- 05:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713500576" 05:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500576" 05:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:22:56 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:22:56 DEBUG --- http response --- 05:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:22:56 DEBUG --- http response --- 05:22:56 ERROR --- http response --- 05:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:22:56 DEBUG ----- output ----- 05:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:22:56 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:22:56 DEBUG ----- output ----- 05:22:56 DEBUG { 05:22:56 DEBUG ----- output ----- 05:22:56 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:22:56 DEBUG { 05:22:56 DEBUG "status": "success", 05:22:56 DEBUG { 05:22:56 DEBUG "status": "success", 05:22:56 DEBUG "data": { 05:22:56 DEBUG "status": "success", 05:22:56 DEBUG "data": { 05:22:56 DEBUG "resultType": "vector", 05:22:56 DEBUG "result": [ 05:22:56 DEBUG "resultType": "vector", 05:22:56 DEBUG "data": { 05:22:56 DEBUG { 05:22:56 DEBUG "result": [ 05:22:56 DEBUG "resultType": "vector", 05:22:56 DEBUG "metric": { 05:22:56 DEBUG { 05:22:56 DEBUG "result": [ 05:22:56 DEBUG "pod": "ds-cts-0" 05:22:56 DEBUG "metric": { 05:22:56 DEBUG { 05:22:56 DEBUG }, 05:22:56 DEBUG "pod": "ds-cts-0" 05:22:56 DEBUG "metric": { 05:22:56 DEBUG "value": [ 05:22:56 DEBUG }, 05:22:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:22:56 DEBUG 05:22:56 DEBUG "value": [ 05:22:56 DEBUG }, 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG "value": [ 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG [...] 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 INFO [http_cmd]: http status code OK 05:22:56 DEBUG 05:22:56 DEBUG [...] 05:22:56 DEBUG 05:22:56 DEBUG --- status code --- 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG [...] 05:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG --- http response --- 05:22:56 DEBUG "pod": "ds-cts-2" 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG }, 05:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:22:56 DEBUG "pod": "ds-cts-2" 05:22:56 DEBUG 05:22:56 DEBUG "value": [ 05:22:56 DEBUG ----- output ----- 05:22:56 DEBUG }, 05:22:56 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:22:56 DEBUG 1713500576, 05:22:56 DEBUG { 05:22:56 DEBUG "value": [ 05:22:56 DEBUG }, 05:22:56 DEBUG "0.0006433688888888889" 05:22:56 DEBUG "status": "success", 05:22:56 DEBUG 1713500576, 05:22:56 DEBUG "value": [ 05:22:56 DEBUG ] 05:22:56 DEBUG "data": { 05:22:56 DEBUG "443.8333333333333" 05:22:56 DEBUG 1713500576, 05:22:56 DEBUG } 05:22:56 DEBUG "resultType": "vector", 05:22:56 DEBUG ] 05:22:56 DEBUG "72289670.40174912" 05:22:56 DEBUG ] 05:22:56 DEBUG "result": [ 05:22:56 DEBUG } 05:22:56 DEBUG ] 05:22:56 DEBUG } 05:22:56 DEBUG { 05:22:56 DEBUG ] 05:22:56 DEBUG } 05:22:56 DEBUG } 05:22:56 DEBUG "metric": { 05:22:56 INFO [http_cmd]: http status code OK 05:22:56 INFO [http_cmd]: http status code OK 05:22:56 INFO [http_cmd]: http status code OK 05:22:56 DEBUG } 05:22:56 INFO [http_cmd]: http status code OK 05:22:56 DEBUG ] 05:22:56 DEBUG "instance": "10.142.0.107:9100" 05:22:56 DEBUG --- status code --- 05:22:56 DEBUG --- status code --- 05:22:56 DEBUG --- status code --- 05:22:56 DEBUG } 05:22:56 DEBUG --- status code --- 05:22:56 DEBUG } 05:22:56 DEBUG }, 05:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:22:56 DEBUG } 05:22:56 DEBUG "value": [ 05:22:56 DEBUG --- http response --- 05:22:56 DEBUG --- http response --- 05:22:56 DEBUG --- http response --- 05:22:56 DEBUG --- http response --- 05:22:56 DEBUG 05:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:22:56 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:22:56 DEBUG 05:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:22:56 DEBUG ----- output ----- 05:22:56 DEBUG ----- output ----- 05:22:56 DEBUG 05:22:56 DEBUG ----- output ----- 05:22:56 DEBUG { 05:22:56 DEBUG { 05:22:56 DEBUG [...] 05:22:56 DEBUG { 05:22:56 DEBUG "status": "success", 05:22:56 DEBUG "status": "success", 05:22:56 DEBUG 05:22:56 DEBUG "status": "success", 05:22:56 DEBUG "data": { 05:22:56 DEBUG "data": { 05:22:56 DEBUG 05:22:56 DEBUG "data": { 05:22:56 DEBUG "resultType": "vector", 05:22:56 DEBUG "resultType": "vector", 05:22:56 DEBUG 05:22:56 DEBUG "resultType": "vector", 05:22:56 DEBUG "result": [ 05:22:56 DEBUG "result": [ 05:22:56 DEBUG "instance": "10.142.0.43:9100" 05:22:56 DEBUG "result": [ 05:22:56 DEBUG { 05:22:56 DEBUG { 05:22:56 DEBUG }, 05:22:56 DEBUG { 05:22:56 DEBUG "metric": { 05:22:56 DEBUG "metric": { 05:22:56 DEBUG "value": [ 05:22:56 DEBUG "metric": { 05:22:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:22:56 DEBUG "pod": "ds-idrepo-0" 05:22:56 DEBUG 1713500576, 05:22:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:22:56 DEBUG }, 05:22:56 DEBUG }, 05:22:56 DEBUG "64321360.9" 05:22:56 DEBUG }, 05:22:56 DEBUG "value": [ 05:22:56 DEBUG "value": [ 05:22:56 DEBUG ] 05:22:56 DEBUG "value": [ 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG } 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG ] 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG } 05:22:56 DEBUG 05:22:56 DEBUG [...] 05:22:56 DEBUG [...] 05:22:56 DEBUG } 05:22:56 DEBUG [...] 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:22:56 DEBUG "pod": "ds-idrepo-2" 05:22:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:22:56 DEBUG }, 05:22:56 DEBUG }, 05:22:56 DEBUG }, 05:22:56 DEBUG "value": [ 05:22:56 DEBUG "value": [ 05:22:56 DEBUG "value": [ 05:22:56 DEBUG 1713500576, 05:22:56 DEBUG 1713500576, 05:22:56 DEBUG 1713500576, 05:22:56 DEBUG "0" 05:22:56 DEBUG "0" 05:22:56 DEBUG "7.117676188233565" 05:22:56 DEBUG ] 05:22:56 DEBUG ] 05:22:56 DEBUG ] 05:22:56 DEBUG } 05:22:56 DEBUG } 05:22:56 DEBUG } 05:22:56 DEBUG ] 05:22:56 DEBUG ] 05:22:56 DEBUG ] 05:22:56 DEBUG } 05:22:56 DEBUG } 05:22:56 DEBUG } 05:22:56 DEBUG } 05:22:56 DEBUG } 05:22:56 DEBUG } 05:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500576" 05:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500576" 05:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500576" 05:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500576" 05:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500576" 05:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500576" 05:22:56 INFO [http_cmd]: http status code OK 05:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500576" 05:22:56 INFO [http_cmd]: http status code OK 05:22:56 DEBUG --- status code --- 05:22:56 INFO [http_cmd]: http status code OK 05:22:56 DEBUG --- status code --- 05:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:22:56 DEBUG --- status code --- 05:22:56 INFO [http_cmd]: http status code OK 05:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:22:56 DEBUG --- http response --- 05:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:22:56 DEBUG --- status code --- 05:22:56 DEBUG --- http response --- 05:22:56 DEBUG --- http response --- 05:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:22:56 DEBUG ----- output ----- 05:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:22:56 DEBUG --- http response --- 05:22:56 DEBUG ----- output ----- 05:22:56 DEBUG { 05:22:56 DEBUG ----- output ----- 05:22:56 DEBUG { 05:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:22:56 DEBUG "status": "success", 05:22:56 DEBUG { 05:22:56 DEBUG "status": "success", 05:22:56 DEBUG ----- output ----- 05:22:56 DEBUG "data": { 05:22:56 DEBUG "status": "success", 05:22:56 DEBUG "data": { 05:22:56 DEBUG { 05:22:56 DEBUG "resultType": "vector", 05:22:56 DEBUG "data": { 05:22:56 DEBUG "resultType": "vector", 05:22:56 DEBUG "status": "success", 05:22:56 DEBUG "result": [ 05:22:56 DEBUG "resultType": "vector", 05:22:56 DEBUG "result": [ 05:22:56 DEBUG "data": { 05:22:56 DEBUG { 05:22:56 DEBUG "result": [ 05:22:56 DEBUG { 05:22:56 DEBUG "resultType": "vector", 05:22:56 DEBUG "metric": { 05:22:56 DEBUG { 05:22:56 DEBUG "metric": { 05:22:56 DEBUG "result": [ 05:22:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:22:56 INFO [http_cmd]: http status code OK 05:22:56 DEBUG "metric": { 05:22:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:22:56 DEBUG { 05:22:56 DEBUG }, 05:22:56 DEBUG --- status code --- 05:22:56 DEBUG "pod": "ds-idrepo-0" 05:22:56 DEBUG }, 05:22:56 DEBUG "metric": { 05:22:56 DEBUG "value": [ 05:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:22:56 DEBUG }, 05:22:56 DEBUG "value": [ 05:22:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:22:56 DEBUG 05:22:56 DEBUG --- http response --- 05:22:56 DEBUG "value": [ 05:22:56 DEBUG 05:22:56 DEBUG }, 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:22:56 DEBUG 05:22:56 DEBUG "value": [ 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG ----- output ----- 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG [...] 05:22:56 DEBUG 05:22:56 DEBUG { 05:22:56 DEBUG [...] 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG [...] 05:22:56 DEBUG "status": "success", 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500576" 05:22:56 DEBUG 05:22:56 DEBUG "data": { 05:22:56 INFO [http_cmd]: http status code OK 05:22:56 DEBUG 05:22:56 DEBUG [...] 05:22:56 INFO [http_cmd]: http status code OK 05:22:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:22:56 DEBUG 05:22:56 DEBUG "resultType": "vector", 05:22:56 DEBUG --- status code --- 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG --- status code --- 05:22:56 DEBUG }, 05:22:56 DEBUG 05:22:56 DEBUG "result": [ 05:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:22:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:22:56 DEBUG 05:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:22:56 DEBUG "value": [ 05:22:56 DEBUG "pod": "ds-idrepo-2" 05:22:56 DEBUG { 05:22:56 DEBUG --- http response --- 05:22:56 DEBUG }, 05:22:56 DEBUG 05:22:56 DEBUG --- http response --- 05:22:56 DEBUG 1713500576, 05:22:56 DEBUG }, 05:22:56 DEBUG "metric": { 05:22:56 DEBUG "value": [ 05:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:22:56 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:22:56 DEBUG "3554872" 05:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:22:56 DEBUG "value": [ 05:22:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:22:56 DEBUG 1713500576, 05:22:56 DEBUG ----- output ----- 05:22:56 DEBUG }, 05:22:56 DEBUG ] 05:22:56 DEBUG ----- output ----- 05:22:56 DEBUG 1713500576, 05:22:56 DEBUG }, 05:22:56 DEBUG "2172.7666666666664" 05:22:56 DEBUG { 05:22:56 DEBUG "value": [ 05:22:56 DEBUG } 05:22:56 DEBUG { 05:22:56 DEBUG "0" 05:22:56 DEBUG "value": [ 05:22:56 DEBUG ] 05:22:56 DEBUG "status": "success", 05:22:56 DEBUG 1713500576, 05:22:56 DEBUG ] 05:22:56 DEBUG "status": "success", 05:22:56 DEBUG ] 05:22:56 DEBUG 05:22:56 DEBUG } 05:22:56 DEBUG "data": { 05:22:56 DEBUG "7.083254502903824" 05:22:56 DEBUG } 05:22:56 DEBUG "data": { 05:22:56 DEBUG } 05:22:56 DEBUG 05:22:56 DEBUG ] 05:22:56 DEBUG "resultType": "vector", 05:22:56 DEBUG ] 05:22:56 DEBUG } 05:22:56 DEBUG "resultType": "vector", 05:22:56 DEBUG ] 05:22:56 DEBUG 05:22:56 DEBUG } 05:22:56 DEBUG "result": [ 05:22:56 DEBUG } 05:22:56 DEBUG "result": [ 05:22:56 DEBUG } 05:22:56 DEBUG [...] 05:22:56 DEBUG } 05:22:56 DEBUG { 05:22:56 DEBUG ] 05:22:56 DEBUG { 05:22:56 DEBUG } 05:22:56 DEBUG 05:22:56 DEBUG "metric": { 05:22:56 DEBUG } 05:22:56 DEBUG "metric": { 05:22:56 DEBUG 05:22:56 DEBUG "instance": "10.142.0.107:9100" 05:22:56 DEBUG } 05:22:56 DEBUG "pod": "ds-cts-0" 05:22:56 DEBUG 05:22:56 DEBUG }, 05:22:56 DEBUG }, 05:22:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:22:56 DEBUG "value": [ 05:22:56 DEBUG "value": [ 05:22:56 DEBUG }, 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG "value": [ 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG 1713500576, 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG "2172.8" 05:22:56 DEBUG [...] 05:22:56 DEBUG [...] 05:22:56 DEBUG ] 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG } 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG ] 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG } 05:22:56 DEBUG "instance": "10.142.0.43:9100" 05:22:56 DEBUG "pod": "ds-cts-2" 05:22:56 DEBUG } 05:22:56 DEBUG }, 05:22:56 DEBUG }, 05:22:56 DEBUG "value": [ 05:22:56 DEBUG "value": [ 05:22:56 DEBUG 1713500576, 05:22:56 DEBUG 1713500576, 05:22:56 DEBUG "64323318.33333333" 05:22:56 DEBUG "0" 05:22:56 DEBUG ] 05:22:56 DEBUG ] 05:22:56 DEBUG } 05:22:56 DEBUG } 05:22:56 DEBUG ] 05:22:56 DEBUG ] 05:22:56 DEBUG } 05:22:56 DEBUG } 05:22:56 DEBUG } 05:22:56 DEBUG } 05:22:56 INFO [http_cmd]: http status code OK 05:22:56 DEBUG --- status code --- 05:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:22:56 DEBUG --- http response --- 05:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:22:56 DEBUG ----- output ----- 05:22:56 DEBUG { 05:22:56 DEBUG "status": "success", 05:22:56 DEBUG "data": { 05:22:56 DEBUG "resultType": "vector", 05:22:56 DEBUG "result": [ 05:22:56 DEBUG { 05:22:56 DEBUG "metric": { 05:22:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:22:56 DEBUG }, 05:22:56 DEBUG "value": [ 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG [...] 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:22:56 DEBUG }, 05:22:56 DEBUG "value": [ 05:22:56 DEBUG 1713500576, 05:22:56 DEBUG "0" 05:22:56 DEBUG ] 05:22:56 DEBUG } 05:22:56 DEBUG ] 05:22:56 DEBUG } 05:22:56 DEBUG } 05:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500576" 05:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713500576" 05:22:56 INFO [http_cmd]: http status code OK 05:22:56 INFO [http_cmd]: http status code OK 05:22:56 DEBUG --- status code --- 05:22:56 DEBUG --- status code --- 05:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:22:56 DEBUG --- http response --- 05:22:56 DEBUG --- http response --- 05:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:22:56 DEBUG ----- output ----- 05:22:56 DEBUG { 05:22:56 DEBUG "status": "success", 05:22:56 DEBUG "data": { 05:22:56 DEBUG "resultType": "vector", 05:22:56 DEBUG "result": [ 05:22:56 DEBUG { 05:22:56 DEBUG "metric": { 05:22:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:22:56 DEBUG }, 05:22:56 DEBUG "value": [ 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG [...] 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:22:56 DEBUG }, 05:22:56 DEBUG "value": [ 05:22:56 DEBUG 1713500576, 05:22:56 DEBUG "55.172" 05:22:56 DEBUG ] 05:22:56 DEBUG } 05:22:56 DEBUG ] 05:22:56 DEBUG } 05:22:56 DEBUG } 05:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500576" 05:22:56 DEBUG ----- output ----- 05:22:56 DEBUG { 05:22:56 DEBUG "status": "success", 05:22:56 DEBUG "data": { 05:22:56 DEBUG "resultType": "vector", 05:22:56 DEBUG "result": [ 05:22:56 DEBUG { 05:22:56 DEBUG "metric": { 05:22:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:22:56 DEBUG }, 05:22:56 DEBUG "value": [ 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG [...] 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:22:56 DEBUG }, 05:22:56 DEBUG "value": [ 05:22:56 DEBUG 1713500576, 05:22:56 DEBUG "2172.9666666666667" 05:22:56 DEBUG ] 05:22:56 DEBUG } 05:22:56 DEBUG ] 05:22:56 DEBUG } 05:22:56 DEBUG } 05:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713500576" 05:22:56 INFO [http_cmd]: http status code OK 05:22:56 DEBUG --- status code --- 05:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:22:56 DEBUG --- http response --- 05:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:22:56 DEBUG ----- output ----- 05:22:56 DEBUG { 05:22:56 DEBUG "status": "success", 05:22:56 DEBUG "data": { 05:22:56 DEBUG "resultType": "vector", 05:22:56 DEBUG "result": [ 05:22:56 DEBUG { 05:22:56 DEBUG "metric": { 05:22:56 DEBUG "instance": "10.142.0.107:9100" 05:22:56 DEBUG }, 05:22:56 DEBUG "value": [ 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG [...] 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG "instance": "10.142.0.43:9100" 05:22:56 DEBUG }, 05:22:56 DEBUG "value": [ 05:22:56 DEBUG 1713500576, 05:22:56 DEBUG "478629405696" 05:22:56 DEBUG ] 05:22:56 DEBUG } 05:22:56 DEBUG ] 05:22:56 DEBUG } 05:22:56 DEBUG } 05:22:56 INFO [http_cmd]: http status code OK 05:22:56 DEBUG --- status code --- 05:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:22:56 DEBUG --- http response --- 05:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:22:56 DEBUG ----- output ----- 05:22:56 DEBUG { 05:22:56 DEBUG "status": "success", 05:22:56 DEBUG "data": { 05:22:56 DEBUG "resultType": "vector", 05:22:56 DEBUG "result": [ 05:22:56 DEBUG { 05:22:56 DEBUG "metric": { 05:22:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:22:56 DEBUG }, 05:22:56 DEBUG "value": [ 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG [...] 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:22:56 DEBUG }, 05:22:56 DEBUG "value": [ 05:22:56 DEBUG 1713500576, 05:22:56 DEBUG "2211373056" 05:22:56 DEBUG ] 05:22:56 DEBUG } 05:22:56 DEBUG ] 05:22:56 DEBUG } 05:22:56 DEBUG } 05:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500576" 05:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500576" 05:22:56 INFO [http_cmd]: http status code OK 05:22:56 DEBUG --- status code --- 05:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:22:56 DEBUG --- http response --- 05:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:22:56 DEBUG ----- output ----- 05:22:56 DEBUG { 05:22:56 DEBUG "status": "success", 05:22:56 DEBUG "data": { 05:22:56 DEBUG "resultType": "vector", 05:22:56 DEBUG "result": [ 05:22:56 DEBUG { 05:22:56 DEBUG "metric": { 05:22:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:22:56 DEBUG }, 05:22:56 DEBUG "value": [ 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG [...] 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:22:56 DEBUG }, 05:22:56 DEBUG "value": [ 05:22:56 DEBUG 1713500576, 05:22:56 DEBUG "2009.4666666666667" 05:22:56 DEBUG ] 05:22:56 DEBUG } 05:22:56 DEBUG ] 05:22:56 DEBUG } 05:22:56 DEBUG } 05:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500576" 05:22:56 INFO [http_cmd]: http status code OK 05:22:56 DEBUG --- status code --- 05:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:22:56 DEBUG --- http response --- 05:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:22:56 DEBUG ----- output ----- 05:22:56 DEBUG { 05:22:56 DEBUG "status": "success", 05:22:56 DEBUG "data": { 05:22:56 DEBUG "resultType": "vector", 05:22:56 DEBUG "result": [ 05:22:56 DEBUG { 05:22:56 DEBUG "metric": { 05:22:56 DEBUG "instance": "10.142.0.107:9100" 05:22:56 DEBUG }, 05:22:56 DEBUG "value": [ 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG [...] 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG "instance": "10.142.0.43:9100" 05:22:56 DEBUG }, 05:22:56 DEBUG "value": [ 05:22:56 DEBUG 1713500576, 05:22:56 DEBUG "0.6995333333333236" 05:22:56 DEBUG ] 05:22:56 DEBUG } 05:22:56 DEBUG ] 05:22:56 DEBUG } 05:22:56 DEBUG } 05:22:56 INFO [http_cmd]: http status code OK 05:22:56 DEBUG --- status code --- 05:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:22:56 DEBUG --- http response --- 05:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:22:56 DEBUG ----- output ----- 05:22:56 DEBUG { 05:22:56 DEBUG "status": "success", 05:22:56 DEBUG "data": { 05:22:56 DEBUG "resultType": "vector", 05:22:56 DEBUG "result": [ 05:22:56 DEBUG { 05:22:56 DEBUG "metric": { 05:22:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:22:56 DEBUG }, 05:22:56 DEBUG "value": [ 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG [...] 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:22:56 DEBUG }, 05:22:56 DEBUG "value": [ 05:22:56 DEBUG 1713500576, 05:22:56 DEBUG "0" 05:22:56 DEBUG ] 05:22:56 DEBUG } 05:22:56 DEBUG ] 05:22:56 DEBUG } 05:22:56 DEBUG } 05:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500576" 05:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500576" 05:22:56 INFO [http_cmd]: http status code OK 05:22:56 DEBUG --- status code --- 05:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:22:56 DEBUG --- http response --- 05:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:22:56 DEBUG ----- output ----- 05:22:56 DEBUG { 05:22:56 DEBUG "status": "success", 05:22:56 DEBUG "data": { 05:22:56 DEBUG "resultType": "vector", 05:22:56 DEBUG "result": [ 05:22:56 DEBUG { 05:22:56 DEBUG "metric": { 05:22:56 DEBUG "pod": "ds-idrepo-0" 05:22:56 DEBUG }, 05:22:56 DEBUG "value": [ 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG [...] 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG "pod": "ds-idrepo-2" 05:22:56 DEBUG }, 05:22:56 DEBUG "value": [ 05:22:56 DEBUG 1713500576, 05:22:56 DEBUG "0" 05:22:56 DEBUG ] 05:22:56 DEBUG } 05:22:56 DEBUG ] 05:22:56 DEBUG } 05:22:56 DEBUG } 05:22:56 INFO [http_cmd]: http status code OK 05:22:56 DEBUG --- status code --- 05:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:22:56 DEBUG --- http response --- 05:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:22:56 DEBUG ----- output ----- 05:22:56 DEBUG { 05:22:56 DEBUG "status": "success", 05:22:56 DEBUG "data": { 05:22:56 DEBUG "resultType": "vector", 05:22:56 DEBUG "result": [ 05:22:56 DEBUG { 05:22:56 DEBUG "metric": { 05:22:56 DEBUG "instance": "10.142.0.107:9100" 05:22:56 DEBUG }, 05:22:56 DEBUG "value": [ 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG [...] 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG 05:22:56 DEBUG "instance": "10.142.0.43:9100" 05:22:56 DEBUG }, 05:22:56 DEBUG "value": [ 05:22:56 DEBUG 1713500576, 05:22:56 DEBUG "0.39299999999999924" 05:22:56 DEBUG ] 05:22:56 DEBUG } 05:22:56 DEBUG ] 05:22:56 DEBUG } 05:22:56 DEBUG } 05:22:58 INFO 05:22:58 INFO [loop_until]: kubectl --namespace=xlou top pods 05:22:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:22:58 INFO [loop_until]: OK (rc = 0) 05:22:58 DEBUG --- stdout --- 05:22:58 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 13529m 9137Mi am-6fd554f8f7-hvvkw 13747m 8900Mi am-6fd554f8f7-pmj4b 14392m 8788Mi ds-cts-0 18493m 13088Mi ds-cts-1 18834m 12990Mi ds-cts-2 18912m 12945Mi ds-idrepo-0 5817m 3267Mi ds-idrepo-1 5910m 3198Mi ds-idrepo-2 5747m 3066Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 3m 1215Mi lodemon-6b89d774d5-rf7sk 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7476m 2018Mi 05:22:58 DEBUG --- stderr --- 05:22:58 DEBUG 05:22:58 INFO 05:22:58 INFO [loop_until]: kubectl --namespace=xlou top node 05:22:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:22:58 INFO [loop_until]: OK (rc = 0) 05:22:58 DEBUG --- stdout --- 05:22:58 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 71m 0% 6254Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5795m 19% 6716Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13174m 44% 11611Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 80m 0% 3247Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 18678m 62% 15034Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 147m 0% 5174Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 13676m 45% 11919Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 11221m 37% 2739Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 15335m 51% 10988Mi 9% gke-xlou-cdm-ds-0f601751-j488 6127m 20% 6154Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 18063m 60% 16473Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 6401m 21% 6876Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 18864m 63% 14962Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 7245m 45% 5742Mi 9% 05:22:58 DEBUG --- stderr --- 05:22:58 DEBUG 05:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713500636" 05:23:56 INFO [http_cmd]: http status code OK 05:23:56 DEBUG --- status code --- 05:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:23:56 DEBUG --- http response --- 05:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:23:56 DEBUG ----- output ----- 05:23:56 DEBUG { 05:23:56 DEBUG "status": "success", 05:23:56 DEBUG "data": { 05:23:56 DEBUG "resultType": "vector", 05:23:56 DEBUG "result": [ 05:23:56 DEBUG { 05:23:56 DEBUG "metric": { 05:23:56 DEBUG "pod": "ds-idrepo-0" 05:23:56 DEBUG }, 05:23:56 DEBUG "value": [ 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG [...] 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG "pod": "ds-cts-2" 05:23:56 DEBUG }, 05:23:56 DEBUG "value": [ 05:23:56 DEBUG 1713500636, 05:23:56 DEBUG "0" 05:23:56 DEBUG ] 05:23:56 DEBUG } 05:23:56 DEBUG ] 05:23:56 DEBUG } 05:23:56 DEBUG } 05:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713500636" 05:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500636" 05:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500636" 05:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500636" 05:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500636" 05:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500636" 05:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500636" 05:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500636" 05:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500636" 05:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500636" 05:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713500636" 05:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500636" 05:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500636" 05:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500636" 05:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713500636" 05:23:56 INFO [http_cmd]: http status code OK 05:23:56 INFO [http_cmd]: http status code OK 05:23:56 INFO [http_cmd]: http status code OK 05:23:56 ERROR [http_cmd]: ERROR 05:23:56 INFO [http_cmd]: http status code OK 05:23:56 INFO [http_cmd]: http status code OK 05:23:56 DEBUG --- status code --- 05:23:56 DEBUG --- status code --- 05:23:56 INFO [http_cmd]: http status code OK 05:23:56 INFO [http_cmd]: http status code OK 05:23:56 DEBUG --- status code --- 05:23:56 ERROR --- status code --- 05:23:56 INFO [http_cmd]: http status code OK 05:23:56 DEBUG --- status code --- 05:23:56 INFO [http_cmd]: http status code OK 05:23:56 DEBUG --- status code --- 05:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:23:56 DEBUG --- status code --- 05:23:56 INFO [http_cmd]: http status code OK 05:23:56 DEBUG --- status code --- 05:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:23:56 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:23:56 DEBUG --- status code --- 05:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:23:56 DEBUG --- status code --- 05:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:23:56 DEBUG --- http response --- 05:23:56 DEBUG --- http response --- 05:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:23:56 DEBUG --- status code --- 05:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:23:56 DEBUG --- http response --- 05:23:56 INFO [http_cmd]: http status code OK 05:23:56 ERROR --- http response --- 05:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:23:56 DEBUG --- http response --- 05:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:23:56 DEBUG --- http response --- 05:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:23:56 DEBUG --- http response --- 05:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:23:56 DEBUG --- http response --- 05:23:56 INFO [http_cmd]: http status code OK 05:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:23:56 DEBUG --- status code --- 05:23:56 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:23:56 DEBUG --- http response --- 05:23:56 DEBUG --- http response --- 05:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:23:56 DEBUG ----- output ----- 05:23:56 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:23:56 DEBUG ----- output ----- 05:23:56 DEBUG { 05:23:56 DEBUG --- status code --- 05:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:23:56 DEBUG ----- output ----- 05:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:23:56 INFO [http_cmd]: http status code OK 05:23:56 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:23:56 DEBUG ----- output ----- 05:23:56 DEBUG { 05:23:56 DEBUG ----- output ----- 05:23:56 DEBUG --- http response --- 05:23:56 DEBUG "status": "success", 05:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:23:56 DEBUG ----- output ----- 05:23:56 DEBUG { 05:23:56 DEBUG --- http response --- 05:23:56 DEBUG --- status code --- 05:23:56 DEBUG ----- output ----- 05:23:56 DEBUG ----- output ----- 05:23:56 INFO [http_cmd]: http status code OK 05:23:56 DEBUG { 05:23:56 DEBUG "status": "success", 05:23:56 DEBUG { 05:23:56 DEBUG "data": { 05:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:23:56 DEBUG --- http response --- 05:23:56 DEBUG { 05:23:56 DEBUG "status": "success", 05:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:23:56 DEBUG { 05:23:56 DEBUG { 05:23:56 DEBUG --- status code --- 05:23:56 DEBUG "status": "success", 05:23:56 DEBUG "data": { 05:23:56 DEBUG "status": "success", 05:23:56 DEBUG "resultType": "vector", 05:23:56 DEBUG ----- output ----- 05:23:56 DEBUG "status": "success", 05:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:23:56 DEBUG "data": { 05:23:56 DEBUG --- http response --- 05:23:56 DEBUG ----- output ----- 05:23:56 DEBUG "status": "success", 05:23:56 DEBUG "status": "success", 05:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:23:56 DEBUG "data": { 05:23:56 DEBUG "resultType": "vector", 05:23:56 DEBUG "data": { 05:23:56 DEBUG "result": [ 05:23:56 DEBUG { 05:23:56 DEBUG "data": { 05:23:56 DEBUG ----- output ----- 05:23:56 DEBUG "resultType": "vector", 05:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:23:56 DEBUG { 05:23:56 DEBUG "data": { 05:23:56 DEBUG "data": { 05:23:56 DEBUG --- http response --- 05:23:56 DEBUG "resultType": "vector", 05:23:56 DEBUG "result": [ 05:23:56 DEBUG "resultType": "vector", 05:23:56 DEBUG { 05:23:56 DEBUG "metric": { 05:23:56 DEBUG "resultType": "vector", 05:23:56 DEBUG { 05:23:56 DEBUG "result": [ 05:23:56 DEBUG ----- output ----- 05:23:56 DEBUG "status": "success", 05:23:56 DEBUG "resultType": "vector", 05:23:56 DEBUG "resultType": "vector", 05:23:56 DEBUG "result": [ 05:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:23:56 DEBUG { 05:23:56 DEBUG "result": [ 05:23:56 DEBUG "status": "success", 05:23:56 DEBUG "pod": "ds-cts-0" 05:23:56 DEBUG "result": [ 05:23:56 DEBUG "status": "success", 05:23:56 DEBUG { 05:23:56 DEBUG { 05:23:56 DEBUG "data": { 05:23:56 DEBUG "result": [ 05:23:56 DEBUG "result": [ 05:23:56 DEBUG { 05:23:56 DEBUG ----- output ----- 05:23:56 DEBUG "metric": { 05:23:56 DEBUG { 05:23:56 DEBUG "data": { 05:23:56 DEBUG }, 05:23:56 DEBUG { 05:23:56 DEBUG "data": { 05:23:56 DEBUG "metric": { 05:23:56 DEBUG "status": "success", 05:23:56 DEBUG "resultType": "vector", 05:23:56 DEBUG { 05:23:56 DEBUG { 05:23:56 DEBUG "metric": { 05:23:56 DEBUG { 05:23:56 DEBUG "pod": "ds-idrepo-0" 05:23:56 DEBUG "metric": { 05:23:56 DEBUG "resultType": "vector", 05:23:56 DEBUG "value": [ 05:23:56 DEBUG "metric": { 05:23:56 DEBUG "resultType": "vector", 05:23:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:23:56 DEBUG "data": { 05:23:56 DEBUG "result": [ 05:23:56 DEBUG "metric": { 05:23:56 DEBUG "metric": { 05:23:56 DEBUG "pod": "ds-cts-0" 05:23:56 DEBUG "status": "success", 05:23:56 DEBUG }, 05:23:56 DEBUG "instance": "10.142.0.107:9100" 05:23:56 DEBUG "result": [ 05:23:56 DEBUG 05:23:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:23:56 DEBUG "result": [ 05:23:56 DEBUG }, 05:23:56 DEBUG "resultType": "vector", 05:23:56 DEBUG { 05:23:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:23:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:23:56 DEBUG }, 05:23:56 DEBUG "data": { 05:23:56 DEBUG "value": [ 05:23:56 DEBUG }, 05:23:56 DEBUG { 05:23:56 DEBUG 05:23:56 DEBUG }, 05:23:56 DEBUG { 05:23:56 DEBUG "value": [ 05:23:56 DEBUG "result": [ 05:23:56 DEBUG "metric": { 05:23:56 DEBUG }, 05:23:56 DEBUG }, 05:23:56 DEBUG "value": [ 05:23:56 DEBUG "resultType": "vector", 05:23:56 DEBUG 05:23:56 DEBUG "value": [ 05:23:56 DEBUG "metric": { 05:23:56 DEBUG 05:23:56 DEBUG "value": [ 05:23:56 DEBUG "metric": { 05:23:56 DEBUG 05:23:56 DEBUG { 05:23:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:23:56 DEBUG "value": [ 05:23:56 DEBUG "value": [ 05:23:56 DEBUG 05:23:56 DEBUG "result": [ 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG "pod": "ds-cts-0" 05:23:56 DEBUG [...] 05:23:56 DEBUG 05:23:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:23:56 DEBUG 05:23:56 DEBUG "metric": { 05:23:56 DEBUG }, 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG { 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG }, 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG }, 05:23:56 DEBUG 05:23:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:23:56 DEBUG "value": [ 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG "metric": { 05:23:56 DEBUG [...] 05:23:56 DEBUG 05:23:56 DEBUG "value": [ 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG "value": [ 05:23:56 DEBUG [...] 05:23:56 DEBUG }, 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG [...] 05:23:56 DEBUG "pod": "ds-idrepo-0" 05:23:56 DEBUG 05:23:56 DEBUG [...] 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG [...] 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG "value": [ 05:23:56 DEBUG 05:23:56 DEBUG [...] 05:23:56 DEBUG [...] 05:23:56 DEBUG 05:23:56 DEBUG }, 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG "pod": "ds-cts-2" 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500636" 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG "value": [ 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG }, 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG [...] 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG "pod": "ds-idrepo-2" 05:23:56 DEBUG 05:23:56 DEBUG [...] 05:23:56 DEBUG "value": [ 05:23:56 DEBUG 05:23:56 DEBUG [...] 05:23:56 DEBUG "pod": "ds-cts-0" 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG "pod": "ds-cts-2" 05:23:56 DEBUG 05:23:56 DEBUG }, 05:23:56 DEBUG "instance": "10.142.0.43:9100" 05:23:56 DEBUG 05:23:56 DEBUG 1713500636, 05:23:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:23:56 DEBUG 05:23:56 DEBUG }, 05:23:56 DEBUG [...] 05:23:56 DEBUG 05:23:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:23:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:23:56 DEBUG }, 05:23:56 DEBUG 05:23:56 DEBUG "value": [ 05:23:56 DEBUG }, 05:23:56 DEBUG 05:23:56 DEBUG "0.0002666666666666667" 05:23:56 DEBUG }, 05:23:56 DEBUG 05:23:56 DEBUG "value": [ 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG }, 05:23:56 DEBUG }, 05:23:56 DEBUG "value": [ 05:23:56 DEBUG [...] 05:23:56 DEBUG 1713500636, 05:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500636" 05:23:56 DEBUG "value": [ 05:23:56 DEBUG 05:23:56 DEBUG ] 05:23:56 DEBUG "value": [ 05:23:56 DEBUG 05:23:56 DEBUG 1713500636, 05:23:56 DEBUG 05:23:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:23:56 DEBUG "value": [ 05:23:56 DEBUG "value": [ 05:23:56 DEBUG 1713500636, 05:23:56 DEBUG 05:23:56 DEBUG "0" 05:23:56 DEBUG 1713500636, 05:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500636" 05:23:56 DEBUG "pod": "ds-cts-2" 05:23:56 DEBUG } 05:23:56 DEBUG 1713500636, 05:23:56 DEBUG "2173.7666666666664" 05:23:56 DEBUG "pod": "ds-cts-0" 05:23:56 DEBUG }, 05:23:56 DEBUG }, 05:23:56 DEBUG 1713500636, 05:23:56 DEBUG 1713500636, 05:23:56 DEBUG "468.43333333333334" 05:23:56 DEBUG 05:23:56 DEBUG ] 05:23:56 DEBUG "64276246.46666667" 05:23:56 DEBUG }, 05:23:56 DEBUG ] 05:23:56 DEBUG "10509930.673011154" 05:23:56 DEBUG ] 05:23:56 DEBUG 05:23:56 DEBUG "value": [ 05:23:56 DEBUG "value": [ 05:23:56 DEBUG "3564660.5" 05:23:56 DEBUG "0" 05:23:56 DEBUG ] 05:23:56 DEBUG 05:23:56 DEBUG } 05:23:56 DEBUG ] 05:23:56 DEBUG "value": [ 05:23:56 DEBUG } 05:23:56 DEBUG ] 05:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713500636" 05:23:56 DEBUG } 05:23:56 DEBUG ] 05:23:56 DEBUG 1713500636, 05:23:56 DEBUG 1713500636, 05:23:56 DEBUG ] 05:23:56 DEBUG ] 05:23:56 DEBUG } 05:23:56 DEBUG "pod": "ds-idrepo-2" 05:23:56 DEBUG ] 05:23:56 DEBUG } 05:23:56 DEBUG 1713500636, 05:23:56 DEBUG } 05:23:56 DEBUG } 05:23:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:23:56 DEBUG } 05:23:56 DEBUG "11.095249987647058" 05:23:56 INFO [http_cmd]: http status code OK 05:23:56 DEBUG "7.09314164109492" 05:23:56 DEBUG } 05:23:56 DEBUG } 05:23:56 DEBUG ] 05:23:56 DEBUG } 05:23:56 DEBUG } 05:23:56 DEBUG ] 05:23:56 DEBUG } 05:23:56 DEBUG ] 05:23:56 DEBUG }, 05:23:56 DEBUG } 05:23:56 DEBUG ] 05:23:56 DEBUG --- status code --- 05:23:56 DEBUG ] 05:23:56 DEBUG ] 05:23:56 DEBUG ] 05:23:56 DEBUG }, 05:23:56 DEBUG } 05:23:56 DEBUG } 05:23:56 DEBUG "0" 05:23:56 DEBUG } 05:23:56 DEBUG } 05:23:56 DEBUG "value": [ 05:23:56 DEBUG } 05:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:23:56 DEBUG } 05:23:56 DEBUG } 05:23:56 DEBUG } 05:23:56 DEBUG "value": [ 05:23:56 DEBUG ] 05:23:56 DEBUG } 05:23:56 INFO [http_cmd]: http status code OK 05:23:56 DEBUG 1713500636, 05:23:56 DEBUG ] 05:23:56 DEBUG --- http response --- 05:23:56 DEBUG ] 05:23:56 DEBUG } 05:23:56 DEBUG } 05:23:56 DEBUG 1713500636, 05:23:56 DEBUG } 05:23:56 DEBUG --- status code --- 05:23:56 DEBUG "2216431616" 05:23:56 DEBUG } 05:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:23:56 INFO [http_cmd]: http status code OK 05:23:56 DEBUG } 05:23:56 DEBUG "0" 05:23:56 DEBUG ] 05:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:23:56 DEBUG ] 05:23:56 DEBUG } 05:23:56 DEBUG ----- output ----- 05:23:56 DEBUG --- status code --- 05:23:56 DEBUG } 05:23:56 DEBUG ] 05:23:56 INFO [http_cmd]: http status code OK 05:23:56 DEBUG } 05:23:56 DEBUG --- http response --- 05:23:56 DEBUG } 05:23:56 DEBUG { 05:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:23:56 DEBUG } 05:23:56 DEBUG --- status code --- 05:23:56 DEBUG } 05:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:23:56 DEBUG ] 05:23:56 DEBUG "status": "success", 05:23:56 DEBUG --- http response --- 05:23:56 DEBUG ] 05:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:23:56 DEBUG ----- output ----- 05:23:56 DEBUG } 05:23:56 DEBUG "data": { 05:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:23:56 DEBUG } 05:23:56 DEBUG --- http response --- 05:23:56 DEBUG { 05:23:56 DEBUG } 05:23:56 DEBUG "resultType": "vector", 05:23:56 DEBUG ----- output ----- 05:23:56 DEBUG } 05:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:23:56 DEBUG "status": "success", 05:23:56 DEBUG "result": [ 05:23:56 DEBUG { 05:23:56 DEBUG ----- output ----- 05:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500636" 05:23:56 DEBUG "data": { 05:23:56 DEBUG { 05:23:56 DEBUG "status": "success", 05:23:56 DEBUG { 05:23:56 DEBUG "resultType": "vector", 05:23:56 DEBUG "metric": { 05:23:56 DEBUG "data": { 05:23:56 DEBUG "status": "success", 05:23:56 DEBUG "result": [ 05:23:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:23:56 DEBUG "resultType": "vector", 05:23:56 DEBUG "data": { 05:23:56 DEBUG { 05:23:56 DEBUG }, 05:23:56 DEBUG "result": [ 05:23:56 DEBUG "resultType": "vector", 05:23:56 DEBUG "metric": { 05:23:56 DEBUG "value": [ 05:23:56 DEBUG { 05:23:56 DEBUG "result": [ 05:23:56 DEBUG "instance": "10.142.0.107:9100" 05:23:56 DEBUG 05:23:56 DEBUG "metric": { 05:23:56 DEBUG { 05:23:56 DEBUG }, 05:23:56 DEBUG 05:23:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:23:56 DEBUG "metric": { 05:23:56 DEBUG "value": [ 05:23:56 DEBUG 05:23:56 DEBUG }, 05:23:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:23:56 DEBUG 05:23:56 DEBUG [...] 05:23:56 DEBUG "value": [ 05:23:56 DEBUG }, 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG "value": [ 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG [...] 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:23:56 DEBUG [...] 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG }, 05:23:56 DEBUG 05:23:56 DEBUG [...] 05:23:56 DEBUG 05:23:56 DEBUG "value": [ 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG "instance": "10.142.0.43:9100" 05:23:56 DEBUG 1713500636, 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG }, 05:23:56 DEBUG "2173.7" 05:23:56 DEBUG "pod": "ds-cts-0" 05:23:56 DEBUG 05:23:56 DEBUG "value": [ 05:23:56 DEBUG ] 05:23:56 DEBUG }, 05:23:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:23:56 DEBUG 1713500636, 05:23:56 DEBUG } 05:23:56 DEBUG "value": [ 05:23:56 DEBUG }, 05:23:56 DEBUG "64280235.46666667" 05:23:56 DEBUG ] 05:23:56 DEBUG 1713500636, 05:23:56 DEBUG "value": [ 05:23:56 DEBUG ] 05:23:56 DEBUG } 05:23:56 DEBUG "30386000.565452296" 05:23:56 DEBUG 1713500636, 05:23:56 DEBUG } 05:23:56 DEBUG ] 05:23:56 DEBUG } 05:23:56 DEBUG "55.172" 05:23:56 DEBUG ] 05:23:56 DEBUG } 05:23:56 DEBUG ] 05:23:56 DEBUG } 05:23:56 DEBUG } 05:23:56 DEBUG } 05:23:56 DEBUG ] 05:23:56 DEBUG ] 05:23:56 DEBUG } 05:23:56 DEBUG } 05:23:56 DEBUG } 05:23:56 DEBUG } 05:23:56 INFO [http_cmd]: http status code OK 05:23:56 DEBUG --- status code --- 05:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:23:56 DEBUG --- http response --- 05:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:23:56 DEBUG ----- output ----- 05:23:56 DEBUG { 05:23:56 DEBUG "status": "success", 05:23:56 DEBUG "data": { 05:23:56 DEBUG "resultType": "vector", 05:23:56 DEBUG "result": [ 05:23:56 DEBUG { 05:23:56 DEBUG "metric": { 05:23:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:23:56 DEBUG }, 05:23:56 DEBUG "value": [ 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG [...] 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:23:56 DEBUG }, 05:23:56 DEBUG "value": [ 05:23:56 DEBUG 1713500636, 05:23:56 DEBUG "2173.4666666666667" 05:23:56 DEBUG ] 05:23:56 DEBUG } 05:23:56 DEBUG ] 05:23:56 DEBUG } 05:23:56 DEBUG } 05:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500636" 05:23:56 INFO [http_cmd]: http status code OK 05:23:56 DEBUG --- status code --- 05:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:23:56 DEBUG --- http response --- 05:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:23:56 DEBUG ----- output ----- 05:23:56 DEBUG { 05:23:56 DEBUG "status": "success", 05:23:56 DEBUG "data": { 05:23:56 DEBUG "resultType": "vector", 05:23:56 DEBUG "result": [ 05:23:56 DEBUG { 05:23:56 DEBUG "metric": { 05:23:56 DEBUG "instance": "10.142.0.107:9100" 05:23:56 DEBUG }, 05:23:56 DEBUG "value": [ 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG [...] 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG "instance": "10.142.0.43:9100" 05:23:56 DEBUG }, 05:23:56 DEBUG "value": [ 05:23:56 DEBUG 1713500636, 05:23:56 DEBUG "481772829696" 05:23:56 DEBUG ] 05:23:56 DEBUG } 05:23:56 DEBUG ] 05:23:56 DEBUG } 05:23:56 DEBUG } 05:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500636" 05:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500636" 05:23:56 INFO [http_cmd]: http status code OK 05:23:56 DEBUG --- status code --- 05:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:23:56 DEBUG --- http response --- 05:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:23:56 DEBUG ----- output ----- 05:23:56 DEBUG { 05:23:56 DEBUG "status": "success", 05:23:56 DEBUG "data": { 05:23:56 DEBUG "resultType": "vector", 05:23:56 DEBUG "result": [ 05:23:56 DEBUG { 05:23:56 DEBUG "metric": { 05:23:56 DEBUG "instance": "10.142.0.107:9100" 05:23:56 DEBUG }, 05:23:56 DEBUG "value": [ 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG [...] 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG "instance": "10.142.0.43:9100" 05:23:56 DEBUG }, 05:23:56 DEBUG "value": [ 05:23:56 DEBUG 1713500636, 05:23:56 DEBUG "0.7077333333333219" 05:23:56 DEBUG ] 05:23:56 DEBUG } 05:23:56 DEBUG ] 05:23:56 DEBUG } 05:23:56 DEBUG } 05:23:56 INFO [http_cmd]: http status code OK 05:23:56 DEBUG --- status code --- 05:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:23:56 DEBUG --- http response --- 05:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:23:56 DEBUG ----- output ----- 05:23:56 DEBUG { 05:23:56 DEBUG "status": "success", 05:23:56 DEBUG "data": { 05:23:56 DEBUG "resultType": "vector", 05:23:56 DEBUG "result": [ 05:23:56 DEBUG { 05:23:56 DEBUG "metric": { 05:23:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:23:56 DEBUG }, 05:23:56 DEBUG "value": [ 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG [...] 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:23:56 DEBUG }, 05:23:56 DEBUG "value": [ 05:23:56 DEBUG 1713500636, 05:23:56 DEBUG "1991.2666666666667" 05:23:56 DEBUG ] 05:23:56 DEBUG } 05:23:56 DEBUG ] 05:23:56 DEBUG } 05:23:56 DEBUG } 05:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500636" 05:23:56 INFO [http_cmd]: http status code OK 05:23:56 DEBUG --- status code --- 05:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:23:56 DEBUG --- http response --- 05:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:23:56 DEBUG ----- output ----- 05:23:56 DEBUG { 05:23:56 DEBUG "status": "success", 05:23:56 DEBUG "data": { 05:23:56 DEBUG "resultType": "vector", 05:23:56 DEBUG "result": [ 05:23:56 DEBUG { 05:23:56 DEBUG "metric": { 05:23:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:23:56 DEBUG }, 05:23:56 DEBUG "value": [ 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG [...] 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG "pod": "ds-cts-0" 05:23:56 DEBUG }, 05:23:56 DEBUG "value": [ 05:23:56 DEBUG 1713500636, 05:23:56 DEBUG "17371781.599008676" 05:23:56 DEBUG ] 05:23:56 DEBUG } 05:23:56 DEBUG ] 05:23:56 DEBUG } 05:23:56 DEBUG } 05:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500636" 05:23:56 INFO [http_cmd]: http status code OK 05:23:56 DEBUG --- status code --- 05:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:23:56 DEBUG --- http response --- 05:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:23:56 DEBUG ----- output ----- 05:23:56 DEBUG { 05:23:56 DEBUG "status": "success", 05:23:56 DEBUG "data": { 05:23:56 DEBUG "resultType": "vector", 05:23:56 DEBUG "result": [ 05:23:56 DEBUG { 05:23:56 DEBUG "metric": { 05:23:56 DEBUG "pod": "ds-idrepo-0" 05:23:56 DEBUG }, 05:23:56 DEBUG "value": [ 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG [...] 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG "pod": "ds-idrepo-2" 05:23:56 DEBUG }, 05:23:56 DEBUG "value": [ 05:23:56 DEBUG 1713500636, 05:23:56 DEBUG "0" 05:23:56 DEBUG ] 05:23:56 DEBUG } 05:23:56 DEBUG ] 05:23:56 DEBUG } 05:23:56 DEBUG } 05:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500636" 05:23:56 INFO [http_cmd]: http status code OK 05:23:56 DEBUG --- status code --- 05:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:23:56 DEBUG --- http response --- 05:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:23:56 DEBUG ----- output ----- 05:23:56 DEBUG { 05:23:56 DEBUG "status": "success", 05:23:56 DEBUG "data": { 05:23:56 DEBUG "resultType": "vector", 05:23:56 DEBUG "result": [ 05:23:56 DEBUG { 05:23:56 DEBUG "metric": { 05:23:56 DEBUG "instance": "10.142.0.107:9100" 05:23:56 DEBUG }, 05:23:56 DEBUG "value": [ 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG [...] 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG 05:23:56 DEBUG "instance": "10.142.0.43:9100" 05:23:56 DEBUG }, 05:23:56 DEBUG "value": [ 05:23:56 DEBUG 1713500636, 05:23:56 DEBUG "0.4326666666666634" 05:23:56 DEBUG ] 05:23:56 DEBUG } 05:23:56 DEBUG ] 05:23:56 DEBUG } 05:23:56 DEBUG } 05:23:58 INFO 05:23:58 INFO [loop_until]: kubectl --namespace=xlou top pods 05:23:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:23:58 INFO [loop_until]: OK (rc = 0) 05:23:58 DEBUG --- stdout --- 05:23:58 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 13767m 9139Mi am-6fd554f8f7-hvvkw 13603m 8901Mi am-6fd554f8f7-pmj4b 14400m 8787Mi ds-cts-0 18468m 12798Mi ds-cts-1 17727m 12684Mi ds-cts-2 18869m 12600Mi ds-idrepo-0 5800m 3267Mi ds-idrepo-1 5911m 3198Mi ds-idrepo-2 5618m 3066Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 4m 1215Mi lodemon-6b89d774d5-rf7sk 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7124m 2020Mi 05:23:58 DEBUG --- stderr --- 05:23:58 DEBUG 05:23:58 INFO 05:23:58 INFO [loop_until]: kubectl --namespace=xlou top node 05:23:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:23:58 INFO [loop_until]: OK (rc = 0) 05:23:58 DEBUG --- stdout --- 05:23:58 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 67m 0% 6247Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5831m 19% 6713Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13185m 44% 11612Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 78m 0% 3253Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 18348m 61% 15232Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 160m 0% 5175Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 13670m 45% 11916Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 11244m 37% 2724Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 15212m 50% 10983Mi 9% gke-xlou-cdm-ds-0f601751-j488 6150m 20% 6162Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 17653m 59% 16681Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 6408m 21% 6874Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 18662m 62% 14540Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 7027m 44% 5742Mi 9% 05:23:58 DEBUG --- stderr --- 05:23:58 DEBUG 05:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713500696" 05:24:56 INFO [http_cmd]: http status code OK 05:24:56 DEBUG --- status code --- 05:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:24:56 DEBUG --- http response --- 05:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:24:56 DEBUG ----- output ----- 05:24:56 DEBUG { 05:24:56 DEBUG "status": "success", 05:24:56 DEBUG "data": { 05:24:56 DEBUG "resultType": "vector", 05:24:56 DEBUG "result": [ 05:24:56 DEBUG { 05:24:56 DEBUG "metric": { 05:24:56 DEBUG "pod": "ds-idrepo-0" 05:24:56 DEBUG }, 05:24:56 DEBUG "value": [ 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG [...] 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG "pod": "ds-cts-2" 05:24:56 DEBUG }, 05:24:56 DEBUG "value": [ 05:24:56 DEBUG 1713500696, 05:24:56 DEBUG "0" 05:24:56 DEBUG ] 05:24:56 DEBUG } 05:24:56 DEBUG ] 05:24:56 DEBUG } 05:24:56 DEBUG } 05:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500696" 05:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500696" 05:24:56 INFO [http_cmd]: http status code OK 05:24:56 DEBUG --- status code --- 05:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:24:56 DEBUG --- http response --- 05:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:24:56 DEBUG ----- output ----- 05:24:56 DEBUG { 05:24:56 DEBUG "status": "success", 05:24:56 DEBUG "data": { 05:24:56 DEBUG "resultType": "vector", 05:24:56 DEBUG "result": [ 05:24:56 DEBUG { 05:24:56 DEBUG "metric": { 05:24:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:24:56 DEBUG }, 05:24:56 DEBUG "value": [ 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG [...] 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:24:56 DEBUG }, 05:24:56 DEBUG "value": [ 05:24:56 DEBUG 1713500696, 05:24:56 DEBUG "0" 05:24:56 DEBUG ] 05:24:56 DEBUG } 05:24:56 DEBUG ] 05:24:56 DEBUG } 05:24:56 DEBUG } 05:24:56 INFO [http_cmd]: http status code OK 05:24:56 DEBUG --- status code --- 05:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:24:56 DEBUG --- http response --- 05:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:24:56 DEBUG ----- output ----- 05:24:56 DEBUG { 05:24:56 DEBUG "status": "success", 05:24:56 DEBUG "data": { 05:24:56 DEBUG "resultType": "vector", 05:24:56 DEBUG "result": [ 05:24:56 DEBUG { 05:24:56 DEBUG "metric": { 05:24:56 DEBUG "pod": "ds-cts-0" 05:24:56 DEBUG }, 05:24:56 DEBUG "value": [ 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG [...] 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG "pod": "ds-cts-2" 05:24:56 DEBUG }, 05:24:56 DEBUG "value": [ 05:24:56 DEBUG 1713500696, 05:24:56 DEBUG "434.73333333333335" 05:24:56 DEBUG ] 05:24:56 DEBUG } 05:24:56 DEBUG ] 05:24:56 DEBUG } 05:24:56 DEBUG } 05:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500696" 05:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713500696" 05:24:56 INFO [http_cmd]: http status code OK 05:24:56 DEBUG --- status code --- 05:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:24:56 DEBUG --- http response --- 05:24:56 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500696" 05:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500696" 05:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500696" 05:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500696" 05:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713500696" 05:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713500696" 05:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500696" 05:24:56 ERROR [http_cmd]: ERROR 05:24:56 ERROR --- status code --- 05:24:56 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:24:56 ERROR --- http response --- 05:24:56 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:24:56 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500696" 05:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500696" 05:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500696" 05:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500696" 05:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500696" 05:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500696" 05:24:56 INFO [http_cmd]: http status code OK 05:24:56 INFO [http_cmd]: http status code OK 05:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500696" 05:24:56 INFO [http_cmd]: http status code OK 05:24:56 INFO [http_cmd]: http status code OK 05:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500696" 05:24:56 DEBUG --- status code --- 05:24:56 INFO [http_cmd]: http status code OK 05:24:56 INFO [http_cmd]: http status code OK 05:24:56 INFO [http_cmd]: http status code OK 05:24:56 DEBUG --- status code --- 05:24:56 INFO [http_cmd]: http status code OK 05:24:56 DEBUG --- status code --- 05:24:56 INFO [http_cmd]: http status code OK 05:24:56 INFO [http_cmd]: http status code OK 05:24:56 DEBUG --- status code --- 05:24:56 INFO [http_cmd]: http status code OK 05:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713500696" 05:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500696" 05:24:56 INFO [http_cmd]: http status code OK 05:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:24:56 DEBUG --- status code --- 05:24:56 DEBUG --- status code --- 05:24:56 DEBUG --- status code --- 05:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:24:56 DEBUG --- status code --- 05:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:24:56 DEBUG --- status code --- 05:24:56 DEBUG --- status code --- 05:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:24:56 DEBUG --- status code --- 05:24:56 INFO [http_cmd]: http status code OK 05:24:56 DEBUG --- status code --- 05:24:56 DEBUG --- http response --- 05:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:24:56 DEBUG --- http response --- 05:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:24:56 DEBUG --- http response --- 05:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:24:56 DEBUG --- http response --- 05:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:24:56 DEBUG --- status code --- 05:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:24:56 DEBUG --- http response --- 05:24:56 INFO [http_cmd]: http status code OK 05:24:56 DEBUG --- http response --- 05:24:56 DEBUG --- http response --- 05:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:24:56 DEBUG --- http response --- 05:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:24:56 DEBUG --- http response --- 05:24:56 DEBUG --- http response --- 05:24:56 DEBUG --- http response --- 05:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:24:56 DEBUG --- http response --- 05:24:56 DEBUG ----- output ----- 05:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:24:56 DEBUG --- status code --- 05:24:56 INFO [http_cmd]: http status code OK 05:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:24:56 DEBUG ----- output ----- 05:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:24:56 DEBUG ----- output ----- 05:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:24:56 DEBUG ----- output ----- 05:24:56 DEBUG --- http response --- 05:24:56 DEBUG { 05:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:24:56 DEBUG ----- output ----- 05:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:24:56 DEBUG --- status code --- 05:24:56 DEBUG ----- output ----- 05:24:56 DEBUG ----- output ----- 05:24:56 DEBUG { 05:24:56 DEBUG ----- output ----- 05:24:56 DEBUG { 05:24:56 DEBUG ----- output ----- 05:24:56 DEBUG ----- output ----- 05:24:56 DEBUG ----- output ----- 05:24:56 DEBUG { 05:24:56 DEBUG "status": "success", 05:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:24:56 DEBUG ----- output ----- 05:24:56 DEBUG { 05:24:56 DEBUG --- http response --- 05:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:24:56 DEBUG { 05:24:56 DEBUG { 05:24:56 DEBUG "status": "success", 05:24:56 DEBUG { 05:24:56 DEBUG "status": "success", 05:24:56 DEBUG { 05:24:56 DEBUG { 05:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500696" 05:24:56 DEBUG { 05:24:56 DEBUG "status": "success", 05:24:56 DEBUG "data": { 05:24:56 DEBUG ----- output ----- 05:24:56 DEBUG { 05:24:56 DEBUG "status": "success", 05:24:56 DEBUG --- http response --- 05:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:24:56 DEBUG "status": "success", 05:24:56 DEBUG "status": "success", 05:24:56 DEBUG "data": { 05:24:56 DEBUG "status": "success", 05:24:56 DEBUG "data": { 05:24:56 DEBUG "status": "success", 05:24:56 DEBUG "status": "success", 05:24:56 DEBUG "status": "success", 05:24:56 DEBUG "data": { 05:24:56 DEBUG "resultType": "vector", 05:24:56 DEBUG { 05:24:56 DEBUG "status": "success", 05:24:56 DEBUG "data": { 05:24:56 DEBUG ----- output ----- 05:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:24:56 DEBUG "data": { 05:24:56 DEBUG "data": { 05:24:56 DEBUG "resultType": "vector", 05:24:56 DEBUG "data": { 05:24:56 DEBUG "resultType": "vector", 05:24:56 DEBUG "data": { 05:24:56 DEBUG "data": { 05:24:56 DEBUG "data": { 05:24:56 DEBUG "resultType": "vector", 05:24:56 DEBUG "result": [ 05:24:56 DEBUG "status": "success", 05:24:56 DEBUG "data": { 05:24:56 DEBUG "resultType": "vector", 05:24:56 DEBUG { 05:24:56 DEBUG ----- output ----- 05:24:56 DEBUG "resultType": "vector", 05:24:56 DEBUG "resultType": "vector", 05:24:56 DEBUG "result": [ 05:24:56 DEBUG "resultType": "vector", 05:24:56 DEBUG "result": [ 05:24:56 DEBUG "resultType": "vector", 05:24:56 DEBUG "resultType": "vector", 05:24:56 DEBUG "resultType": "vector", 05:24:56 DEBUG "result": [ 05:24:56 DEBUG { 05:24:56 DEBUG "data": { 05:24:56 DEBUG "resultType": "vector", 05:24:56 DEBUG "result": [ 05:24:56 DEBUG "status": "success", 05:24:56 DEBUG { 05:24:56 DEBUG "result": [ 05:24:56 DEBUG "result": [ 05:24:56 DEBUG { 05:24:56 DEBUG "result": [ 05:24:56 DEBUG { 05:24:56 DEBUG "result": [ 05:24:56 DEBUG "result": [ 05:24:56 DEBUG "result": [ 05:24:56 DEBUG { 05:24:56 DEBUG "metric": { 05:24:56 DEBUG "resultType": "vector", 05:24:56 DEBUG "result": [ 05:24:56 DEBUG { 05:24:56 DEBUG "data": { 05:24:56 DEBUG "status": "success", 05:24:56 DEBUG { 05:24:56 DEBUG { 05:24:56 DEBUG "metric": { 05:24:56 DEBUG { 05:24:56 DEBUG "metric": { 05:24:56 DEBUG { 05:24:56 DEBUG { 05:24:56 DEBUG { 05:24:56 DEBUG "metric": { 05:24:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:24:56 INFO [http_cmd]: http status code OK 05:24:56 INFO [http_cmd]: http status code OK 05:24:56 DEBUG "result": [ 05:24:56 DEBUG { 05:24:56 DEBUG "metric": { 05:24:56 DEBUG "resultType": "vector", 05:24:56 DEBUG "data": { 05:24:56 DEBUG "metric": { 05:24:56 DEBUG "metric": { 05:24:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:24:56 DEBUG "metric": { 05:24:56 DEBUG "pod": "ds-idrepo-0" 05:24:56 DEBUG "metric": { 05:24:56 DEBUG "metric": { 05:24:56 DEBUG "metric": { 05:24:56 DEBUG "instance": "10.142.0.107:9100" 05:24:56 DEBUG }, 05:24:56 DEBUG --- status code --- 05:24:56 DEBUG --- status code --- 05:24:56 DEBUG { 05:24:56 DEBUG "metric": { 05:24:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:24:56 DEBUG "result": [ 05:24:56 DEBUG "resultType": "vector", 05:24:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:24:56 DEBUG "instance": "10.142.0.107:9100" 05:24:56 DEBUG }, 05:24:56 DEBUG "pod": "ds-cts-0" 05:24:56 DEBUG }, 05:24:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:24:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:24:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:24:56 DEBUG }, 05:24:56 DEBUG "value": [ 05:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:24:56 DEBUG "metric": { 05:24:56 DEBUG "instance": "10.142.0.107:9100" 05:24:56 DEBUG }, 05:24:56 DEBUG { 05:24:56 DEBUG "result": [ 05:24:56 DEBUG }, 05:24:56 DEBUG }, 05:24:56 DEBUG "value": [ 05:24:56 DEBUG }, 05:24:56 DEBUG "value": [ 05:24:56 DEBUG }, 05:24:56 DEBUG }, 05:24:56 DEBUG }, 05:24:56 DEBUG "value": [ 05:24:56 DEBUG 05:24:56 DEBUG --- http response --- 05:24:56 DEBUG --- http response --- 05:24:56 DEBUG "pod": "ds-idrepo-0" 05:24:56 DEBUG }, 05:24:56 DEBUG "value": [ 05:24:56 DEBUG "metric": { 05:24:56 DEBUG { 05:24:56 DEBUG "value": [ 05:24:56 DEBUG "value": [ 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG "value": [ 05:24:56 DEBUG "value": [ 05:24:56 DEBUG 05:24:56 INFO [http_cmd]: http status code OK 05:24:56 DEBUG "value": [ 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:24:56 DEBUG }, 05:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:24:56 DEBUG "value": [ 05:24:56 DEBUG 05:24:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:24:56 DEBUG "metric": { 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG "value": [ 05:24:56 DEBUG 05:24:56 DEBUG --- status code --- 05:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG ----- output ----- 05:24:56 DEBUG "value": [ 05:24:56 DEBUG ----- output ----- 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG }, 05:24:56 DEBUG "pod": "ds-cts-0" 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG [...] 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG --- http response --- 05:24:56 DEBUG 05:24:56 DEBUG [...] 05:24:56 DEBUG { 05:24:56 DEBUG 05:24:56 DEBUG { 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG "value": [ 05:24:56 DEBUG }, 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG [...] 05:24:56 DEBUG 05:24:56 DEBUG [...] 05:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:24:56 DEBUG 05:24:56 DEBUG "status": "success", 05:24:56 DEBUG 05:24:56 DEBUG "status": "success", 05:24:56 DEBUG 05:24:56 DEBUG [...] 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG [...] 05:24:56 DEBUG [...] 05:24:56 DEBUG [...] 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG ----- output ----- 05:24:56 DEBUG 05:24:56 DEBUG "data": { 05:24:56 DEBUG 05:24:56 DEBUG "data": { 05:24:56 DEBUG [...] 05:24:56 DEBUG 05:24:56 DEBUG "value": [ 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:24:56 DEBUG [...] 05:24:56 DEBUG [...] 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG [...] 05:24:56 DEBUG { 05:24:56 DEBUG 05:24:56 DEBUG "resultType": "vector", 05:24:56 DEBUG [...] 05:24:56 DEBUG "resultType": "vector", 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG [...] 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG }, 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG "status": "success", 05:24:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:24:56 DEBUG "result": [ 05:24:56 DEBUG 05:24:56 DEBUG "result": [ 05:24:56 DEBUG { 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:24:56 DEBUG 05:24:56 DEBUG "value": [ 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:24:56 DEBUG }, 05:24:56 DEBUG 05:24:56 DEBUG "data": { 05:24:56 DEBUG }, 05:24:56 DEBUG { 05:24:56 DEBUG "metric": { 05:24:56 DEBUG 05:24:56 DEBUG "metric": { 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG }, 05:24:56 DEBUG "pod": "ds-idrepo-2" 05:24:56 DEBUG 1713500696, 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG "instance": "10.142.0.43:9100" 05:24:56 DEBUG "value": [ 05:24:56 DEBUG 05:24:56 DEBUG "resultType": "vector", 05:24:56 DEBUG "value": [ 05:24:56 DEBUG 05:24:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:24:56 DEBUG }, 05:24:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:24:56 DEBUG "instance": "10.142.0.43:9100" 05:24:56 DEBUG [...] 05:24:56 DEBUG 05:24:56 DEBUG "instance": "10.142.0.43:9100" 05:24:56 DEBUG "value": [ 05:24:56 DEBUG 1713500696, 05:24:56 DEBUG "40419.86058285416" 05:24:56 DEBUG "pod": "ds-cts-2" 05:24:56 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:24:56 DEBUG }, 05:24:56 DEBUG "value": [ 05:24:56 DEBUG 1713500696, 05:24:56 DEBUG "result": [ 05:24:56 DEBUG { 05:24:56 DEBUG 05:24:56 DEBUG "value": [ 05:24:56 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:24:56 DEBUG }, 05:24:56 DEBUG }, 05:24:56 DEBUG 05:24:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:24:56 DEBUG }, 05:24:56 DEBUG }, 05:24:56 DEBUG "7.091101685077476" 05:24:56 DEBUG ] 05:24:56 DEBUG }, 05:24:56 DEBUG }, 05:24:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:24:56 DEBUG 1713500696, 05:24:56 DEBUG "2161.733333333333" 05:24:56 DEBUG 1713500696, 05:24:56 DEBUG "metric": { 05:24:56 DEBUG "pod": "ds-idrepo-2" 05:24:56 DEBUG 05:24:56 DEBUG }, 05:24:56 DEBUG "value": [ 05:24:56 DEBUG "value": [ 05:24:56 DEBUG 05:24:56 DEBUG }, 05:24:56 DEBUG "value": [ 05:24:56 DEBUG "value": [ 05:24:56 DEBUG ] 05:24:56 DEBUG } 05:24:56 DEBUG "value": [ 05:24:56 DEBUG "value": [ 05:24:56 DEBUG }, 05:24:56 DEBUG "63785364.56666666" 05:24:56 DEBUG ] 05:24:56 DEBUG "2218954752" 05:24:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:24:56 DEBUG }, 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG "value": [ 05:24:56 DEBUG 1713500696, 05:24:56 DEBUG 05:24:56 DEBUG "value": [ 05:24:56 DEBUG 1713500696, 05:24:56 DEBUG 1713500696, 05:24:56 DEBUG } 05:24:56 DEBUG ] 05:24:56 DEBUG 1713500696, 05:24:56 DEBUG 1713500696, 05:24:56 DEBUG "value": [ 05:24:56 DEBUG ] 05:24:56 DEBUG } 05:24:56 DEBUG ] 05:24:56 DEBUG }, 05:24:56 DEBUG "value": [ 05:24:56 DEBUG 05:24:56 DEBUG [...] 05:24:56 DEBUG 05:24:56 DEBUG "484914951168" 05:24:56 DEBUG "pod": "ds-cts-2" 05:24:56 DEBUG }, 05:24:56 DEBUG "63780875.13333333" 05:24:56 DEBUG "0" 05:24:56 DEBUG ] 05:24:56 DEBUG } 05:24:56 DEBUG "0.0008333333333333334" 05:24:56 DEBUG "0" 05:24:56 DEBUG 1713500696, 05:24:56 DEBUG } 05:24:56 DEBUG ] 05:24:56 DEBUG } 05:24:56 DEBUG "value": [ 05:24:56 DEBUG 1713500696, 05:24:56 DEBUG 05:24:56 DEBUG 1713500696, 05:24:56 DEBUG 05:24:56 DEBUG ] 05:24:56 DEBUG 1713500696, 05:24:56 DEBUG "value": [ 05:24:56 DEBUG ] 05:24:56 DEBUG ] 05:24:56 DEBUG } 05:24:56 DEBUG } 05:24:56 DEBUG ] 05:24:56 DEBUG ] 05:24:56 DEBUG "2162" 05:24:56 DEBUG ] 05:24:56 DEBUG } 05:24:56 DEBUG ] 05:24:56 DEBUG 05:24:56 DEBUG "0" 05:24:56 DEBUG 05:24:56 DEBUG "11.881794427589483" 05:24:56 DEBUG 05:24:56 DEBUG } 05:24:56 DEBUG "3574875" 05:24:56 DEBUG 1713500696, 05:24:56 DEBUG "25.3" 05:24:56 DEBUG } 05:24:56 DEBUG } 05:24:56 DEBUG } 05:24:56 DEBUG ] 05:24:56 DEBUG ] 05:24:56 DEBUG } 05:24:56 DEBUG } 05:24:56 DEBUG } 05:24:56 DEBUG 05:24:56 DEBUG ] 05:24:56 DEBUG [...] 05:24:56 DEBUG ] 05:24:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:24:56 DEBUG ] 05:24:56 DEBUG ] 05:24:56 DEBUG } 05:24:56 DEBUG ] 05:24:56 DEBUG ] 05:24:56 DEBUG } 05:24:56 DEBUG } 05:24:56 DEBUG } 05:24:56 DEBUG } 05:24:56 DEBUG } 05:24:56 DEBUG 05:24:56 DEBUG [...] 05:24:56 DEBUG 05:24:56 DEBUG } 05:24:56 DEBUG }, 05:24:56 DEBUG } 05:24:56 DEBUG } 05:24:56 DEBUG ] 05:24:56 DEBUG } 05:24:56 DEBUG ] 05:24:56 DEBUG ] 05:24:56 DEBUG } 05:24:56 DEBUG ] 05:24:56 DEBUG } 05:24:56 DEBUG } 05:24:56 DEBUG 05:24:56 DEBUG ] 05:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500696" 05:24:56 DEBUG "value": [ 05:24:56 DEBUG } 05:24:56 DEBUG ] 05:24:56 DEBUG } 05:24:56 DEBUG } 05:24:56 DEBUG } 05:24:56 DEBUG } 05:24:56 DEBUG 05:24:56 DEBUG } 05:24:56 DEBUG ] 05:24:56 DEBUG } 05:24:56 DEBUG } 05:24:56 DEBUG 1713500696, 05:24:56 DEBUG } 05:24:56 DEBUG } 05:24:56 DEBUG } 05:24:56 DEBUG } 05:24:56 DEBUG } 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG } 05:24:56 DEBUG } 05:24:56 DEBUG "2161.9333333333334" 05:24:56 DEBUG } 05:24:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:24:56 DEBUG 05:24:56 DEBUG ] 05:24:56 DEBUG }, 05:24:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:24:56 DEBUG } 05:24:56 DEBUG "value": [ 05:24:56 DEBUG }, 05:24:56 DEBUG ] 05:24:56 DEBUG 1713500696, 05:24:56 DEBUG "value": [ 05:24:56 DEBUG } 05:24:56 DEBUG "1994.8999999999999" 05:24:56 DEBUG 1713500696, 05:24:56 DEBUG } 05:24:56 DEBUG ] 05:24:56 DEBUG "55.172" 05:24:56 DEBUG } 05:24:56 DEBUG ] 05:24:56 DEBUG ] 05:24:56 DEBUG } 05:24:56 DEBUG } 05:24:56 DEBUG ] 05:24:56 DEBUG } 05:24:56 DEBUG } 05:24:56 DEBUG } 05:24:56 INFO [http_cmd]: http status code OK 05:24:56 DEBUG --- status code --- 05:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:24:56 DEBUG --- http response --- 05:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:24:56 DEBUG ----- output ----- 05:24:56 DEBUG { 05:24:56 DEBUG "status": "success", 05:24:56 DEBUG "data": { 05:24:56 DEBUG "resultType": "vector", 05:24:56 DEBUG "result": [ 05:24:56 DEBUG { 05:24:56 DEBUG "metric": { 05:24:56 DEBUG "instance": "10.142.0.107:9100" 05:24:56 DEBUG }, 05:24:56 DEBUG "value": [ 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG [...] 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG "instance": "10.142.0.43:9100" 05:24:56 DEBUG }, 05:24:56 DEBUG "value": [ 05:24:56 DEBUG 1713500696, 05:24:56 DEBUG "0.6812999999999988" 05:24:56 DEBUG ] 05:24:56 DEBUG } 05:24:56 DEBUG ] 05:24:56 DEBUG } 05:24:56 DEBUG } 05:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500696" 05:24:56 INFO [http_cmd]: http status code OK 05:24:56 DEBUG --- status code --- 05:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:24:56 DEBUG --- http response --- 05:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:24:56 DEBUG ----- output ----- 05:24:56 DEBUG { 05:24:56 DEBUG "status": "success", 05:24:56 DEBUG "data": { 05:24:56 DEBUG "resultType": "vector", 05:24:56 DEBUG "result": [ 05:24:56 DEBUG { 05:24:56 DEBUG "metric": { 05:24:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:24:56 DEBUG }, 05:24:56 DEBUG "value": [ 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG [...] 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:24:56 DEBUG }, 05:24:56 DEBUG "value": [ 05:24:56 DEBUG 1713500696, 05:24:56 DEBUG "0" 05:24:56 DEBUG ] 05:24:56 DEBUG } 05:24:56 DEBUG ] 05:24:56 DEBUG } 05:24:56 DEBUG } 05:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500696" 05:24:56 INFO [http_cmd]: http status code OK 05:24:56 DEBUG --- status code --- 05:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:24:56 DEBUG --- http response --- 05:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:24:56 DEBUG ----- output ----- 05:24:56 DEBUG { 05:24:56 DEBUG "status": "success", 05:24:56 DEBUG "data": { 05:24:56 DEBUG "resultType": "vector", 05:24:56 DEBUG "result": [ 05:24:56 DEBUG { 05:24:56 DEBUG "metric": { 05:24:56 DEBUG "pod": "ds-idrepo-0" 05:24:56 DEBUG }, 05:24:56 DEBUG "value": [ 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG [...] 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG "pod": "ds-idrepo-2" 05:24:56 DEBUG }, 05:24:56 DEBUG "value": [ 05:24:56 DEBUG 1713500696, 05:24:56 DEBUG "0" 05:24:56 DEBUG ] 05:24:56 DEBUG } 05:24:56 DEBUG ] 05:24:56 DEBUG } 05:24:56 DEBUG } 05:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500696" 05:24:56 INFO [http_cmd]: http status code OK 05:24:56 DEBUG --- status code --- 05:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:24:56 DEBUG --- http response --- 05:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:24:56 DEBUG ----- output ----- 05:24:56 DEBUG { 05:24:56 DEBUG "status": "success", 05:24:56 DEBUG "data": { 05:24:56 DEBUG "resultType": "vector", 05:24:56 DEBUG "result": [ 05:24:56 DEBUG { 05:24:56 DEBUG "metric": { 05:24:56 DEBUG "instance": "10.142.0.107:9100" 05:24:56 DEBUG }, 05:24:56 DEBUG "value": [ 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG [...] 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG 05:24:56 DEBUG "instance": "10.142.0.43:9100" 05:24:56 DEBUG }, 05:24:56 DEBUG "value": [ 05:24:56 DEBUG 1713500696, 05:24:56 DEBUG "0.4250000000000028" 05:24:56 DEBUG ] 05:24:56 DEBUG } 05:24:56 DEBUG ] 05:24:56 DEBUG } 05:24:56 DEBUG } 05:24:58 INFO 05:24:58 INFO [loop_until]: kubectl --namespace=xlou top pods 05:24:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:24:58 INFO [loop_until]: OK (rc = 0) 05:24:58 DEBUG --- stdout --- 05:24:58 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 13653m 9142Mi am-6fd554f8f7-hvvkw 13252m 8902Mi am-6fd554f8f7-pmj4b 13996m 8789Mi ds-cts-0 18497m 13051Mi ds-cts-1 17567m 12934Mi ds-cts-2 19143m 12876Mi ds-idrepo-0 5770m 3267Mi ds-idrepo-1 5733m 3199Mi ds-idrepo-2 5664m 3066Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 4m 1215Mi lodemon-6b89d774d5-rf7sk 1m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7056m 2022Mi 05:24:58 DEBUG --- stderr --- 05:24:58 DEBUG 05:24:58 INFO 05:24:58 INFO [loop_until]: kubectl --namespace=xlou top node 05:24:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:24:58 INFO [loop_until]: OK (rc = 0) 05:24:58 DEBUG --- stdout --- 05:24:58 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 69m 0% 6245Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5788m 19% 6715Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13147m 44% 11618Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 85m 0% 3249Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 18651m 62% 15490Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 152m 0% 5194Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 13728m 45% 11922Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 11159m 37% 2729Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 15152m 50% 10986Mi 9% gke-xlou-cdm-ds-0f601751-j488 6030m 20% 6159Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 17798m 59% 16965Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6150m 20% 6878Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 18543m 62% 14738Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 7203m 45% 5743Mi 9% 05:24:58 DEBUG --- stderr --- 05:24:58 DEBUG 05:25:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713500756" 05:25:56 INFO [http_cmd]: http status code OK 05:25:56 DEBUG --- status code --- 05:25:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:25:56 DEBUG --- http response --- 05:25:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:25:56 DEBUG ----- output ----- 05:25:56 DEBUG { 05:25:56 DEBUG "status": "success", 05:25:56 DEBUG "data": { 05:25:56 DEBUG "resultType": "vector", 05:25:56 DEBUG "result": [ 05:25:56 DEBUG { 05:25:56 DEBUG "metric": { 05:25:56 DEBUG "pod": "ds-idrepo-0" 05:25:56 DEBUG }, 05:25:56 DEBUG "value": [ 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG [...] 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG "pod": "ds-cts-2" 05:25:56 DEBUG }, 05:25:56 DEBUG "value": [ 05:25:56 DEBUG 1713500756, 05:25:56 DEBUG "0" 05:25:56 DEBUG ] 05:25:56 DEBUG } 05:25:56 DEBUG ] 05:25:56 DEBUG } 05:25:56 DEBUG } 05:25:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500756" 05:25:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500756" 05:25:56 INFO [http_cmd]: http status code OK 05:25:56 INFO [http_cmd]: http status code OK 05:25:56 DEBUG --- status code --- 05:25:56 DEBUG --- status code --- 05:25:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:25:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:25:56 DEBUG --- http response --- 05:25:56 DEBUG --- http response --- 05:25:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:25:56 DEBUG ----- output ----- 05:25:56 DEBUG { 05:25:56 DEBUG "status": "success", 05:25:56 DEBUG "data": { 05:25:56 DEBUG "resultType": "vector", 05:25:56 DEBUG "result": [ 05:25:56 DEBUG { 05:25:56 DEBUG "metric": { 05:25:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:25:56 DEBUG }, 05:25:56 DEBUG "value": [ 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG [...] 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:25:56 DEBUG }, 05:25:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500756" 05:25:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:25:56 DEBUG "value": [ 05:25:56 DEBUG ----- output ----- 05:25:56 DEBUG 1713500756, 05:25:56 DEBUG { 05:25:56 DEBUG "0" 05:25:56 DEBUG "status": "success", 05:25:56 DEBUG ] 05:25:56 DEBUG "data": { 05:25:56 DEBUG } 05:25:56 DEBUG "resultType": "vector", 05:25:56 DEBUG ] 05:25:56 DEBUG "result": [ 05:25:56 DEBUG } 05:25:56 DEBUG { 05:25:56 DEBUG } 05:25:56 DEBUG "metric": { 05:25:56 DEBUG "pod": "ds-cts-0" 05:25:56 DEBUG }, 05:25:56 DEBUG "value": [ 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG [...] 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG "pod": "ds-cts-2" 05:25:56 DEBUG }, 05:25:56 DEBUG "value": [ 05:25:56 DEBUG 1713500756, 05:25:56 DEBUG "428.0333333333333" 05:25:56 DEBUG ] 05:25:56 DEBUG } 05:25:56 DEBUG ] 05:25:56 DEBUG } 05:25:56 DEBUG } 05:25:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713500756" 05:25:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500756" 05:25:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500756" 05:25:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500756" 05:25:56 INFO [http_cmd]: http status code OK 05:25:56 DEBUG --- status code --- 05:25:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:25:56 DEBUG --- http response --- 05:25:56 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:25:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500756" 05:25:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500756" 05:25:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500756" 05:25:56 INFO [http_cmd]: http status code OK 05:25:56 DEBUG --- status code --- 05:25:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:25:56 DEBUG --- http response --- 05:25:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:25:56 DEBUG ----- output ----- 05:25:56 DEBUG { 05:25:56 DEBUG "status": "success", 05:25:56 DEBUG "data": { 05:25:56 DEBUG "resultType": "vector", 05:25:56 DEBUG "result": [ 05:25:56 DEBUG { 05:25:56 DEBUG "metric": { 05:25:56 DEBUG "instance": "10.142.0.107:9100" 05:25:56 DEBUG }, 05:25:56 DEBUG "value": [ 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG [...] 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG "instance": "10.142.0.43:9100" 05:25:56 DEBUG }, 05:25:56 DEBUG "value": [ 05:25:56 DEBUG 1713500756, 05:25:56 DEBUG "488051856384" 05:25:56 DEBUG ] 05:25:56 DEBUG } 05:25:56 DEBUG ] 05:25:56 DEBUG } 05:25:56 DEBUG } 05:25:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500756" 05:25:56 ERROR [http_cmd]: ERROR 05:25:56 ERROR --- status code --- 05:25:56 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:25:56 ERROR --- http response --- 05:25:56 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:25:56 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:25:56 INFO [http_cmd]: http status code OK 05:25:56 INFO [http_cmd]: http status code OK 05:25:56 INFO [http_cmd]: http status code OK 05:25:56 DEBUG --- status code --- 05:25:56 DEBUG --- status code --- 05:25:56 DEBUG --- status code --- 05:25:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:25:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:25:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:25:56 DEBUG --- http response --- 05:25:56 DEBUG --- http response --- 05:25:56 DEBUG --- http response --- 05:25:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:25:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:25:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:25:56 DEBUG ----- output ----- 05:25:56 DEBUG ----- output ----- 05:25:56 DEBUG ----- output ----- 05:25:56 DEBUG { 05:25:56 DEBUG { 05:25:56 DEBUG { 05:25:56 DEBUG "status": "success", 05:25:56 DEBUG "status": "success", 05:25:56 DEBUG "status": "success", 05:25:56 INFO [http_cmd]: http status code OK 05:25:56 INFO [http_cmd]: http status code OK 05:25:56 DEBUG "data": { 05:25:56 DEBUG "data": { 05:25:56 DEBUG "data": { 05:25:56 DEBUG --- status code --- 05:25:56 DEBUG --- status code --- 05:25:56 DEBUG "resultType": "vector", 05:25:56 DEBUG "resultType": "vector", 05:25:56 DEBUG "resultType": "vector", 05:25:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:25:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:25:56 DEBUG "result": [ 05:25:56 DEBUG "result": [ 05:25:56 DEBUG "result": [ 05:25:56 DEBUG --- http response --- 05:25:56 DEBUG --- http response --- 05:25:56 DEBUG { 05:25:56 DEBUG { 05:25:56 DEBUG { 05:25:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:25:56 DEBUG "metric": { 05:25:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:25:56 DEBUG "metric": { 05:25:56 DEBUG "metric": { 05:25:56 DEBUG ----- output ----- 05:25:56 DEBUG "pod": "ds-cts-0" 05:25:56 DEBUG ----- output ----- 05:25:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:25:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:25:56 DEBUG { 05:25:56 DEBUG }, 05:25:56 DEBUG { 05:25:56 DEBUG }, 05:25:56 DEBUG }, 05:25:56 DEBUG "status": "success", 05:25:56 DEBUG "value": [ 05:25:56 DEBUG "status": "success", 05:25:56 DEBUG "value": [ 05:25:56 DEBUG "value": [ 05:25:56 DEBUG "data": { 05:25:56 DEBUG 05:25:56 DEBUG "data": { 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG "resultType": "vector", 05:25:56 DEBUG 05:25:56 DEBUG "resultType": "vector", 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG "result": [ 05:25:56 DEBUG 05:25:56 DEBUG "result": [ 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG { 05:25:56 DEBUG [...] 05:25:56 DEBUG { 05:25:56 DEBUG [...] 05:25:56 DEBUG [...] 05:25:56 DEBUG "metric": { 05:25:56 DEBUG 05:25:56 DEBUG "metric": { 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:25:56 DEBUG 05:25:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG }, 05:25:56 DEBUG 05:25:56 DEBUG }, 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG "value": [ 05:25:56 DEBUG "pod": "ds-cts-2" 05:25:56 DEBUG "value": [ 05:25:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:25:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:25:56 DEBUG 05:25:56 DEBUG }, 05:25:56 DEBUG 05:25:56 DEBUG }, 05:25:56 DEBUG }, 05:25:56 DEBUG 05:25:56 DEBUG "value": [ 05:25:56 DEBUG 05:25:56 DEBUG "value": [ 05:25:56 DEBUG "value": [ 05:25:56 DEBUG 05:25:56 DEBUG 1713500756, 05:25:56 DEBUG 05:25:56 DEBUG 1713500756, 05:25:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713500756" 05:25:56 DEBUG 1713500756, 05:25:56 DEBUG [...] 05:25:56 DEBUG "30" 05:25:56 DEBUG [...] 05:25:56 DEBUG "2172.4333333333334" 05:25:56 DEBUG "3584919" 05:25:56 DEBUG 05:25:56 DEBUG ] 05:25:56 DEBUG 05:25:56 DEBUG ] 05:25:56 DEBUG ] 05:25:56 DEBUG 05:25:56 DEBUG } 05:25:56 DEBUG 05:25:56 DEBUG } 05:25:56 DEBUG } 05:25:56 DEBUG 05:25:56 DEBUG ] 05:25:56 DEBUG 05:25:56 DEBUG ] 05:25:56 DEBUG ] 05:25:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:25:56 DEBUG } 05:25:56 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 05:25:56 DEBUG } 05:25:56 DEBUG } 05:25:56 DEBUG }, 05:25:56 DEBUG } 05:25:56 DEBUG }, 05:25:56 DEBUG } 05:25:56 DEBUG } 05:25:56 DEBUG "value": [ 05:25:56 DEBUG "value": [ 05:25:56 INFO [http_cmd]: http status code OK 05:25:56 DEBUG 1713500756, 05:25:56 DEBUG 1713500756, 05:25:56 DEBUG --- status code --- 05:25:56 DEBUG "2171.6666666666665" 05:25:56 DEBUG "0.000023561680050235807" 05:25:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:25:56 DEBUG ] 05:25:56 DEBUG ] 05:25:56 DEBUG --- http response --- 05:25:56 DEBUG } 05:25:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500756" 05:25:56 DEBUG } 05:25:56 DEBUG ] 05:25:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:25:56 DEBUG ] 05:25:56 DEBUG } 05:25:56 DEBUG ----- output ----- 05:25:56 DEBUG } 05:25:56 DEBUG } 05:25:56 DEBUG { 05:25:56 DEBUG } 05:25:56 DEBUG "status": "success", 05:25:56 DEBUG "data": { 05:25:56 DEBUG "resultType": "vector", 05:25:56 DEBUG "result": [ 05:25:56 DEBUG { 05:25:56 DEBUG "metric": { 05:25:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:25:56 DEBUG }, 05:25:56 DEBUG "value": [ 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG [...] 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 05:25:56 DEBUG }, 05:25:56 DEBUG "value": [ 05:25:56 DEBUG 1713500756, 05:25:56 DEBUG "0" 05:25:56 DEBUG ] 05:25:56 DEBUG } 05:25:56 DEBUG ] 05:25:56 DEBUG } 05:25:56 DEBUG } 05:25:56 INFO [http_cmd]: http status code OK 05:25:56 DEBUG --- status code --- 05:25:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:25:56 DEBUG --- http response --- 05:25:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:25:56 DEBUG ----- output ----- 05:25:56 DEBUG { 05:25:56 DEBUG "status": "success", 05:25:56 DEBUG "data": { 05:25:56 DEBUG "resultType": "vector", 05:25:56 DEBUG "result": [ 05:25:56 DEBUG { 05:25:56 DEBUG "metric": { 05:25:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:25:56 DEBUG }, 05:25:56 DEBUG "value": [ 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG [...] 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:25:56 DEBUG }, 05:25:56 DEBUG "value": [ 05:25:56 DEBUG 1713500756, 05:25:56 DEBUG "55.172" 05:25:56 DEBUG ] 05:25:56 DEBUG } 05:25:56 DEBUG ] 05:25:56 DEBUG } 05:25:56 DEBUG } 05:25:56 INFO [http_cmd]: http status code OK 05:25:56 DEBUG --- status code --- 05:25:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:25:56 DEBUG --- http response --- 05:25:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:25:56 DEBUG ----- output ----- 05:25:56 DEBUG { 05:25:56 DEBUG "status": "success", 05:25:56 DEBUG "data": { 05:25:56 DEBUG "resultType": "vector", 05:25:56 DEBUG "result": [ 05:25:56 DEBUG { 05:25:56 DEBUG "metric": { 05:25:56 DEBUG "instance": "10.142.0.107:9100" 05:25:56 DEBUG }, 05:25:56 DEBUG "value": [ 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG [...] 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG "instance": "10.142.0.43:9100" 05:25:56 DEBUG }, 05:25:56 DEBUG "value": [ 05:25:56 DEBUG 1713500756, 05:25:56 DEBUG "0.7334666666666683" 05:25:56 DEBUG ] 05:25:56 DEBUG } 05:25:56 DEBUG ] 05:25:56 DEBUG } 05:25:56 DEBUG } 05:25:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713500756" 05:25:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500756" 05:25:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500756" 05:25:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713500756" 05:25:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500756" 05:25:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500756" 05:25:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500756" 05:25:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500756" 05:25:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500756" 05:25:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500756" 05:25:56 INFO [http_cmd]: http status code OK 05:25:56 INFO [http_cmd]: http status code OK 05:25:56 INFO [http_cmd]: http status code OK 05:25:56 DEBUG --- status code --- 05:25:56 DEBUG --- status code --- 05:25:56 DEBUG --- status code --- 05:25:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:25:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:25:56 INFO [http_cmd]: http status code OK 05:25:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:25:56 INFO [http_cmd]: http status code OK 05:25:56 DEBUG --- status code --- 05:25:56 DEBUG --- http response --- 05:25:56 DEBUG --- status code --- 05:25:56 DEBUG --- http response --- 05:25:56 DEBUG --- http response --- 05:25:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:25:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:25:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:25:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:25:56 DEBUG --- http response --- 05:25:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:25:56 INFO [http_cmd]: http status code OK 05:25:56 DEBUG --- http response --- 05:25:56 DEBUG ----- output ----- 05:25:56 DEBUG ----- output ----- 05:25:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:25:56 DEBUG ----- output ----- 05:25:56 INFO [http_cmd]: http status code OK 05:25:56 DEBUG --- status code --- 05:25:56 DEBUG { 05:25:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:25:56 INFO [http_cmd]: http status code OK 05:25:56 DEBUG { 05:25:56 DEBUG ----- output ----- 05:25:56 INFO [http_cmd]: http status code OK 05:25:56 DEBUG { 05:25:56 DEBUG --- status code --- 05:25:56 INFO [http_cmd]: http status code OK 05:25:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:25:56 DEBUG "status": "success", 05:25:56 DEBUG ----- output ----- 05:25:56 DEBUG --- status code --- 05:25:56 DEBUG "status": "success", 05:25:56 DEBUG { 05:25:56 DEBUG --- status code --- 05:25:56 DEBUG "status": "success", 05:25:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:25:56 DEBUG --- status code --- 05:25:56 DEBUG --- http response --- 05:25:56 DEBUG "data": { 05:25:56 DEBUG { 05:25:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:25:56 DEBUG "data": { 05:25:56 DEBUG "status": "success", 05:25:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:25:56 DEBUG "data": { 05:25:56 DEBUG --- http response --- 05:25:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:25:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:25:56 DEBUG "resultType": "vector", 05:25:56 DEBUG "status": "success", 05:25:56 DEBUG --- http response --- 05:25:56 DEBUG "resultType": "vector", 05:25:56 DEBUG "data": { 05:25:56 DEBUG --- http response --- 05:25:56 DEBUG "resultType": "vector", 05:25:56 DEBUG --- http response --- 05:25:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:25:56 DEBUG ----- output ----- 05:25:56 DEBUG "result": [ 05:25:56 DEBUG "data": { 05:25:56 DEBUG "result": [ 05:25:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:25:56 DEBUG "resultType": "vector", 05:25:56 DEBUG "result": [ 05:25:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:25:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:25:56 DEBUG ----- output ----- 05:25:56 DEBUG { 05:25:56 DEBUG { 05:25:56 DEBUG "resultType": "vector", 05:25:56 DEBUG { 05:25:56 DEBUG ----- output ----- 05:25:56 DEBUG "result": [ 05:25:56 DEBUG { 05:25:56 DEBUG ----- output ----- 05:25:56 DEBUG ----- output ----- 05:25:56 DEBUG { 05:25:56 DEBUG "status": "success", 05:25:56 DEBUG "metric": { 05:25:56 DEBUG "result": [ 05:25:56 DEBUG "metric": { 05:25:56 DEBUG { 05:25:56 DEBUG { 05:25:56 DEBUG "metric": { 05:25:56 DEBUG { 05:25:56 DEBUG { 05:25:56 DEBUG "status": "success", 05:25:56 DEBUG "data": { 05:25:56 DEBUG "instance": "10.142.0.107:9100" 05:25:56 DEBUG { 05:25:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:25:56 DEBUG "status": "success", 05:25:56 DEBUG "metric": { 05:25:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:25:56 DEBUG "status": "success", 05:25:56 DEBUG "status": "success", 05:25:56 DEBUG "data": { 05:25:56 DEBUG "resultType": "vector", 05:25:56 DEBUG }, 05:25:56 DEBUG "metric": { 05:25:56 DEBUG }, 05:25:56 DEBUG "data": { 05:25:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:25:56 DEBUG }, 05:25:56 DEBUG "data": { 05:25:56 DEBUG "data": { 05:25:56 DEBUG "resultType": "vector", 05:25:56 DEBUG "result": [ 05:25:56 DEBUG "value": [ 05:25:56 DEBUG "pod": "ds-cts-0" 05:25:56 DEBUG "value": [ 05:25:56 DEBUG "resultType": "vector", 05:25:56 DEBUG }, 05:25:56 DEBUG "value": [ 05:25:56 DEBUG "resultType": "vector", 05:25:56 DEBUG "resultType": "vector", 05:25:56 DEBUG "result": [ 05:25:56 DEBUG { 05:25:56 DEBUG 05:25:56 DEBUG }, 05:25:56 DEBUG 05:25:56 DEBUG "result": [ 05:25:56 DEBUG "value": [ 05:25:56 DEBUG 05:25:56 DEBUG "result": [ 05:25:56 DEBUG "result": [ 05:25:56 DEBUG { 05:25:56 DEBUG "metric": { 05:25:56 DEBUG 05:25:56 DEBUG "value": [ 05:25:56 DEBUG 05:25:56 DEBUG { 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG { 05:25:56 DEBUG { 05:25:56 DEBUG "metric": { 05:25:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG "metric": { 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG "metric": { 05:25:56 DEBUG "metric": { 05:25:56 DEBUG "pod": "ds-idrepo-0" 05:25:56 DEBUG }, 05:25:56 DEBUG [...] 05:25:56 DEBUG 05:25:56 DEBUG [...] 05:25:56 DEBUG "instance": "10.142.0.107:9100" 05:25:56 DEBUG 05:25:56 DEBUG [...] 05:25:56 DEBUG "pod": "ds-idrepo-0" 05:25:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:25:56 DEBUG }, 05:25:56 DEBUG "value": [ 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG }, 05:25:56 DEBUG [...] 05:25:56 DEBUG 05:25:56 DEBUG }, 05:25:56 DEBUG }, 05:25:56 DEBUG "value": [ 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG [...] 05:25:56 DEBUG 05:25:56 DEBUG "value": [ 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG "value": [ 05:25:56 DEBUG "value": [ 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG "instance": "10.142.0.43:9100" 05:25:56 DEBUG 05:25:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG [...] 05:25:56 DEBUG }, 05:25:56 DEBUG 05:25:56 DEBUG }, 05:25:56 DEBUG 05:25:56 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 05:25:56 DEBUG }, 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG [...] 05:25:56 DEBUG 05:25:56 DEBUG "value": [ 05:25:56 DEBUG "pod": "ds-cts-2" 05:25:56 DEBUG "value": [ 05:25:56 DEBUG [...] 05:25:56 DEBUG }, 05:25:56 DEBUG "value": [ 05:25:56 DEBUG [...] 05:25:56 DEBUG [...] 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG 1713500756, 05:25:56 DEBUG }, 05:25:56 DEBUG 1713500756, 05:25:56 DEBUG 05:25:56 DEBUG "value": [ 05:25:56 DEBUG 1713500756, 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG "64079790.2" 05:25:56 DEBUG "value": [ 05:25:56 DEBUG "2171.733333333333" 05:25:56 DEBUG 05:25:56 DEBUG 1713500756, 05:25:56 DEBUG "7.1637068316971755" 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:25:56 DEBUG ] 05:25:56 DEBUG 1713500756, 05:25:56 DEBUG ] 05:25:56 DEBUG 05:25:56 DEBUG "0" 05:25:56 DEBUG ] 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG "pod": "ds-idrepo-2" 05:25:56 DEBUG }, 05:25:56 DEBUG } 05:25:56 DEBUG "0.0006794288888888889" 05:25:56 DEBUG } 05:25:56 DEBUG "instance": "10.142.0.43:9100" 05:25:56 DEBUG ] 05:25:56 DEBUG } 05:25:56 DEBUG "pod": "ds-idrepo-2" 05:25:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:25:56 DEBUG }, 05:25:56 DEBUG "value": [ 05:25:56 DEBUG ] 05:25:56 DEBUG ] 05:25:56 DEBUG ] 05:25:56 DEBUG }, 05:25:56 DEBUG } 05:25:56 DEBUG ] 05:25:56 DEBUG }, 05:25:56 DEBUG }, 05:25:56 DEBUG "value": [ 05:25:56 DEBUG 1713500756, 05:25:56 DEBUG } 05:25:56 DEBUG } 05:25:56 DEBUG } 05:25:56 DEBUG "value": [ 05:25:56 DEBUG ] 05:25:56 DEBUG } 05:25:56 DEBUG "value": [ 05:25:56 DEBUG "value": [ 05:25:56 DEBUG 1713500756, 05:25:56 DEBUG "2217250816" 05:25:56 DEBUG } 05:25:56 DEBUG ] 05:25:56 DEBUG } 05:25:56 DEBUG 1713500756, 05:25:56 DEBUG } 05:25:56 DEBUG } 05:25:56 DEBUG 1713500756, 05:25:56 DEBUG 1713500756, 05:25:56 DEBUG "1997.1333333333332" 05:25:56 DEBUG ] 05:25:56 DEBUG } 05:25:56 DEBUG "64075417.93333333" 05:25:56 DEBUG } 05:25:56 DEBUG "0" 05:25:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500756" 05:25:56 DEBUG "0" 05:25:56 DEBUG ] 05:25:56 DEBUG } 05:25:56 DEBUG } 05:25:56 DEBUG ] 05:25:56 DEBUG ] 05:25:56 DEBUG ] 05:25:56 DEBUG } 05:25:56 DEBUG ] 05:25:56 DEBUG } 05:25:56 DEBUG } 05:25:56 DEBUG } 05:25:56 DEBUG ] 05:25:56 DEBUG } 05:25:56 DEBUG ] 05:25:56 DEBUG ] 05:25:56 DEBUG ] 05:25:56 DEBUG } 05:25:56 DEBUG } 05:25:56 DEBUG } 05:25:56 DEBUG } 05:25:56 DEBUG } 05:25:56 DEBUG } 05:25:56 DEBUG } 05:25:56 DEBUG } 05:25:56 DEBUG } 05:25:56 INFO [http_cmd]: http status code OK 05:25:56 DEBUG --- status code --- 05:25:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:25:56 DEBUG --- http response --- 05:25:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500756" 05:25:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:25:56 DEBUG ----- output ----- 05:25:56 DEBUG { 05:25:56 DEBUG "status": "success", 05:25:56 DEBUG "data": { 05:25:56 DEBUG "resultType": "vector", 05:25:56 DEBUG "result": [ 05:25:56 DEBUG { 05:25:56 DEBUG "metric": { 05:25:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:25:56 DEBUG }, 05:25:56 DEBUG "value": [ 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG [...] 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 05:25:56 DEBUG }, 05:25:56 DEBUG "value": [ 05:25:56 DEBUG 1713500756, 05:25:56 DEBUG "0" 05:25:56 DEBUG ] 05:25:56 DEBUG } 05:25:56 DEBUG ] 05:25:56 DEBUG } 05:25:56 DEBUG } 05:25:56 INFO [http_cmd]: http status code OK 05:25:56 DEBUG --- status code --- 05:25:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:25:56 DEBUG --- http response --- 05:25:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:25:56 DEBUG ----- output ----- 05:25:56 DEBUG { 05:25:56 DEBUG "status": "success", 05:25:56 DEBUG "data": { 05:25:56 DEBUG "resultType": "vector", 05:25:56 DEBUG "result": [ 05:25:56 DEBUG { 05:25:56 DEBUG "metric": { 05:25:56 DEBUG "pod": "ds-idrepo-0" 05:25:56 DEBUG }, 05:25:56 DEBUG "value": [ 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG [...] 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG "pod": "ds-idrepo-2" 05:25:56 DEBUG }, 05:25:56 DEBUG "value": [ 05:25:56 DEBUG 1713500756, 05:25:56 DEBUG "0" 05:25:56 DEBUG ] 05:25:56 DEBUG } 05:25:56 DEBUG ] 05:25:56 DEBUG } 05:25:56 DEBUG } 05:25:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500756" 05:25:56 INFO [http_cmd]: http status code OK 05:25:56 DEBUG --- status code --- 05:25:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:25:56 DEBUG --- http response --- 05:25:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:25:56 DEBUG ----- output ----- 05:25:56 DEBUG { 05:25:56 DEBUG "status": "success", 05:25:56 DEBUG "data": { 05:25:56 DEBUG "resultType": "vector", 05:25:56 DEBUG "result": [ 05:25:56 DEBUG { 05:25:56 DEBUG "metric": { 05:25:56 DEBUG "instance": "10.142.0.107:9100" 05:25:56 DEBUG }, 05:25:56 DEBUG "value": [ 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG [...] 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG 05:25:56 DEBUG "instance": "10.142.0.43:9100" 05:25:56 DEBUG }, 05:25:56 DEBUG "value": [ 05:25:56 DEBUG 1713500756, 05:25:56 DEBUG "0.44166666666666554" 05:25:56 DEBUG ] 05:25:56 DEBUG } 05:25:56 DEBUG ] 05:25:56 DEBUG } 05:25:56 DEBUG } 05:25:58 INFO 05:25:58 INFO [loop_until]: kubectl --namespace=xlou top pods 05:25:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:25:58 INFO [loop_until]: OK (rc = 0) 05:25:58 DEBUG --- stdout --- 05:25:58 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 13684m 9140Mi am-6fd554f8f7-hvvkw 13376m 8905Mi am-6fd554f8f7-pmj4b 14256m 8789Mi ds-cts-0 18134m 12496Mi ds-cts-1 18239m 12467Mi ds-cts-2 18676m 12958Mi ds-idrepo-0 5778m 3267Mi ds-idrepo-1 5834m 3198Mi ds-idrepo-2 5666m 3065Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 3m 1215Mi lodemon-6b89d774d5-rf7sk 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7122m 2024Mi 05:25:58 DEBUG --- stderr --- 05:25:58 DEBUG 05:25:58 INFO 05:25:58 INFO [loop_until]: kubectl --namespace=xlou top node 05:25:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:25:59 INFO [loop_until]: OK (rc = 0) 05:25:59 DEBUG --- stdout --- 05:25:59 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 69m 0% 6251Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5794m 19% 6716Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13204m 44% 11614Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 85m 0% 3250Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 18023m 60% 15073Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 134m 0% 5185Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 13659m 45% 11925Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 11209m 37% 2734Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 15115m 50% 11003Mi 9% gke-xlou-cdm-ds-0f601751-j488 6138m 20% 6163Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 17289m 57% 16510Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 6381m 21% 6873Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 18824m 63% 14376Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 7265m 45% 5736Mi 9% 05:25:59 DEBUG --- stderr --- 05:25:59 DEBUG 05:26:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713500816" 05:26:56 INFO [http_cmd]: http status code OK 05:26:56 DEBUG --- status code --- 05:26:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:26:56 DEBUG --- http response --- 05:26:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:26:56 DEBUG ----- output ----- 05:26:56 DEBUG { 05:26:56 DEBUG "status": "success", 05:26:56 DEBUG "data": { 05:26:56 DEBUG "resultType": "vector", 05:26:56 DEBUG "result": [ 05:26:56 DEBUG { 05:26:56 DEBUG "metric": { 05:26:56 DEBUG "pod": "ds-idrepo-0" 05:26:56 DEBUG }, 05:26:56 DEBUG "value": [ 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG [...] 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG "pod": "ds-cts-2" 05:26:56 DEBUG }, 05:26:56 DEBUG "value": [ 05:26:56 DEBUG 1713500816, 05:26:56 DEBUG "0" 05:26:56 DEBUG ] 05:26:56 DEBUG } 05:26:56 DEBUG ] 05:26:56 DEBUG } 05:26:56 DEBUG } 05:26:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500816" 05:26:56 INFO [http_cmd]: http status code OK 05:26:56 DEBUG --- status code --- 05:26:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:26:56 DEBUG --- http response --- 05:26:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:26:56 DEBUG ----- output ----- 05:26:56 DEBUG { 05:26:56 DEBUG "status": "success", 05:26:56 DEBUG "data": { 05:26:56 DEBUG "resultType": "vector", 05:26:56 DEBUG "result": [ 05:26:56 DEBUG { 05:26:56 DEBUG "metric": { 05:26:56 DEBUG "pod": "ds-cts-0" 05:26:56 DEBUG }, 05:26:56 DEBUG "value": [ 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG [...] 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG "pod": "ds-cts-2" 05:26:56 DEBUG }, 05:26:56 DEBUG "value": [ 05:26:56 DEBUG 1713500816, 05:26:56 DEBUG "360.23333333333335" 05:26:56 DEBUG ] 05:26:56 DEBUG } 05:26:56 DEBUG ] 05:26:56 DEBUG } 05:26:56 DEBUG } 05:26:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500816" 05:26:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500816" 05:26:56 INFO [http_cmd]: http status code OK 05:26:56 DEBUG --- status code --- 05:26:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:26:56 DEBUG --- http response --- 05:26:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:26:56 DEBUG ----- output ----- 05:26:56 DEBUG { 05:26:56 DEBUG "status": "success", 05:26:56 DEBUG "data": { 05:26:56 DEBUG "resultType": "vector", 05:26:56 DEBUG "result": [ 05:26:56 DEBUG { 05:26:56 DEBUG "metric": { 05:26:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:26:56 DEBUG }, 05:26:56 DEBUG "value": [ 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG [...] 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:26:56 DEBUG }, 05:26:56 DEBUG "value": [ 05:26:56 DEBUG 1713500816, 05:26:56 DEBUG "2173.2" 05:26:56 DEBUG ] 05:26:56 DEBUG } 05:26:56 DEBUG ] 05:26:56 DEBUG } 05:26:56 DEBUG } 05:26:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500816" 05:26:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500816" 05:26:56 INFO [http_cmd]: http status code OK 05:26:56 INFO [http_cmd]: http status code OK 05:26:56 DEBUG --- status code --- 05:26:56 DEBUG --- status code --- 05:26:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:26:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:26:56 DEBUG --- http response --- 05:26:56 DEBUG --- http response --- 05:26:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:26:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:26:56 DEBUG ----- output ----- 05:26:56 DEBUG ----- output ----- 05:26:56 DEBUG { 05:26:56 DEBUG { 05:26:56 DEBUG "status": "success", 05:26:56 DEBUG "status": "success", 05:26:56 DEBUG "data": { 05:26:56 DEBUG "data": { 05:26:56 DEBUG "resultType": "vector", 05:26:56 DEBUG "resultType": "vector", 05:26:56 DEBUG "result": [ 05:26:56 DEBUG "result": [ 05:26:56 DEBUG { 05:26:56 DEBUG { 05:26:56 DEBUG "metric": { 05:26:56 DEBUG "metric": { 05:26:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:26:56 DEBUG "instance": "10.142.0.107:9100" 05:26:56 DEBUG }, 05:26:56 DEBUG }, 05:26:56 DEBUG "value": [ 05:26:56 DEBUG "value": [ 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG [...] 05:26:56 DEBUG [...] 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:26:56 DEBUG "instance": "10.142.0.43:9100" 05:26:56 DEBUG }, 05:26:56 DEBUG }, 05:26:56 DEBUG "value": [ 05:26:56 DEBUG "value": [ 05:26:56 DEBUG 1713500816, 05:26:56 DEBUG 1713500816, 05:26:56 DEBUG "0" 05:26:56 DEBUG "491354174464" 05:26:56 DEBUG ] 05:26:56 DEBUG ] 05:26:56 DEBUG } 05:26:56 DEBUG } 05:26:56 DEBUG ] 05:26:56 DEBUG ] 05:26:56 DEBUG } 05:26:56 DEBUG } 05:26:56 DEBUG } 05:26:56 DEBUG } 05:26:56 INFO [http_cmd]: http status code OK 05:26:56 DEBUG --- status code --- 05:26:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:26:56 DEBUG --- http response --- 05:26:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:26:56 DEBUG ----- output ----- 05:26:56 DEBUG { 05:26:56 DEBUG "status": "success", 05:26:56 DEBUG "data": { 05:26:56 DEBUG "resultType": "vector", 05:26:56 DEBUG "result": [ 05:26:56 DEBUG { 05:26:56 DEBUG "metric": { 05:26:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:26:56 DEBUG }, 05:26:56 DEBUG "value": [ 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG [...] 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 05:26:56 DEBUG }, 05:26:56 DEBUG "value": [ 05:26:56 DEBUG 1713500816, 05:26:56 DEBUG "0" 05:26:56 DEBUG ] 05:26:56 DEBUG } 05:26:56 DEBUG ] 05:26:56 DEBUG } 05:26:56 DEBUG } 05:26:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500816" 05:26:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713500816" 05:26:56 INFO [http_cmd]: http status code OK 05:26:56 DEBUG --- status code --- 05:26:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:26:56 DEBUG --- http response --- 05:26:56 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:26:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500816" 05:26:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500816" 05:26:56 ERROR [http_cmd]: ERROR 05:26:56 ERROR --- status code --- 05:26:56 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:26:56 ERROR --- http response --- 05:26:56 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:26:56 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:26:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500816" 05:26:56 INFO [http_cmd]: http status code OK 05:26:56 DEBUG --- status code --- 05:26:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:26:56 DEBUG --- http response --- 05:26:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:26:56 DEBUG ----- output ----- 05:26:56 DEBUG { 05:26:56 DEBUG "status": "success", 05:26:56 DEBUG "data": { 05:26:56 DEBUG "resultType": "vector", 05:26:56 DEBUG "result": [ 05:26:56 DEBUG { 05:26:56 DEBUG "metric": { 05:26:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:26:56 DEBUG }, 05:26:56 DEBUG "value": [ 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG [...] 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:26:56 DEBUG }, 05:26:56 DEBUG "value": [ 05:26:56 DEBUG 1713500816, 05:26:56 DEBUG "3594851" 05:26:56 DEBUG ] 05:26:56 DEBUG } 05:26:56 DEBUG ] 05:26:56 DEBUG } 05:26:56 DEBUG } 05:26:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500816" 05:26:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500816" 05:26:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713500816" 05:26:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713500816" 05:26:56 INFO [http_cmd]: http status code OK 05:26:56 DEBUG --- status code --- 05:26:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:26:56 DEBUG --- http response --- 05:26:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:26:56 DEBUG ----- output ----- 05:26:56 DEBUG { 05:26:56 DEBUG "status": "success", 05:26:56 DEBUG "data": { 05:26:56 DEBUG "resultType": "vector", 05:26:56 DEBUG "result": [ 05:26:56 DEBUG { 05:26:56 DEBUG "metric": { 05:26:56 DEBUG "pod": "ds-cts-0" 05:26:56 DEBUG }, 05:26:56 DEBUG "value": [ 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG [...] 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG "pod": "ds-cts-2" 05:26:56 DEBUG }, 05:26:56 DEBUG "value": [ 05:26:56 DEBUG 1713500816, 05:26:56 DEBUG "30" 05:26:56 DEBUG ] 05:26:56 DEBUG } 05:26:56 DEBUG ] 05:26:56 DEBUG } 05:26:56 DEBUG } 05:26:56 INFO [http_cmd]: http status code OK 05:26:56 DEBUG --- status code --- 05:26:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:26:56 DEBUG --- http response --- 05:26:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:26:56 DEBUG ----- output ----- 05:26:56 DEBUG { 05:26:56 DEBUG "status": "success", 05:26:56 DEBUG "data": { 05:26:56 DEBUG "resultType": "vector", 05:26:56 DEBUG "result": [ 05:26:56 DEBUG { 05:26:56 DEBUG "metric": { 05:26:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:26:56 DEBUG }, 05:26:56 DEBUG "value": [ 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG [...] 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:26:56 DEBUG }, 05:26:56 DEBUG "value": [ 05:26:56 DEBUG 1713500816, 05:26:56 DEBUG "2172.8333333333335" 05:26:56 DEBUG ] 05:26:56 DEBUG } 05:26:56 DEBUG ] 05:26:56 DEBUG } 05:26:56 DEBUG } 05:26:56 INFO [http_cmd]: http status code OK 05:26:56 INFO [http_cmd]: http status code OK 05:26:56 DEBUG --- status code --- 05:26:56 DEBUG --- status code --- 05:26:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:26:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:26:56 DEBUG --- http response --- 05:26:56 DEBUG --- http response --- 05:26:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:26:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:26:56 DEBUG ----- output ----- 05:26:56 DEBUG ----- output ----- 05:26:56 DEBUG { 05:26:56 DEBUG { 05:26:56 DEBUG "status": "success", 05:26:56 DEBUG "status": "success", 05:26:56 DEBUG "data": { 05:26:56 DEBUG "data": { 05:26:56 DEBUG "resultType": "vector", 05:26:56 DEBUG "resultType": "vector", 05:26:56 DEBUG "result": [ 05:26:56 DEBUG "result": [ 05:26:56 DEBUG { 05:26:56 DEBUG { 05:26:56 DEBUG "metric": { 05:26:56 DEBUG "metric": { 05:26:56 DEBUG "instance": "10.142.0.107:9100" 05:26:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:26:56 INFO [http_cmd]: http status code OK 05:26:56 DEBUG }, 05:26:56 DEBUG }, 05:26:56 DEBUG --- status code --- 05:26:56 DEBUG "value": [ 05:26:56 DEBUG "value": [ 05:26:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG --- http response --- 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:26:56 DEBUG 05:26:56 DEBUG [...] 05:26:56 DEBUG ----- output ----- 05:26:56 DEBUG [...] 05:26:56 DEBUG 05:26:56 DEBUG { 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG "status": "success", 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 INFO [http_cmd]: http status code OK 05:26:56 DEBUG "data": { 05:26:56 DEBUG 05:26:56 DEBUG "instance": "10.142.0.43:9100" 05:26:56 DEBUG --- status code --- 05:26:56 DEBUG "resultType": "vector", 05:26:56 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 05:26:56 DEBUG }, 05:26:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:26:56 DEBUG "result": [ 05:26:56 DEBUG }, 05:26:56 DEBUG "value": [ 05:26:56 DEBUG --- http response --- 05:26:56 DEBUG { 05:26:56 DEBUG "value": [ 05:26:56 DEBUG 1713500816, 05:26:56 DEBUG "metric": { 05:26:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:26:56 DEBUG 1713500816, 05:26:56 DEBUG "0.6737999999999895" 05:26:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:26:56 DEBUG ----- output ----- 05:26:56 DEBUG "0.000011244287448945485" 05:26:56 DEBUG ] 05:26:56 DEBUG }, 05:26:56 DEBUG { 05:26:56 DEBUG ] 05:26:56 DEBUG } 05:26:56 DEBUG "value": [ 05:26:56 DEBUG "status": "success", 05:26:56 DEBUG } 05:26:56 DEBUG ] 05:26:56 DEBUG 05:26:56 DEBUG "data": { 05:26:56 DEBUG ] 05:26:56 DEBUG } 05:26:56 DEBUG 05:26:56 DEBUG "resultType": "vector", 05:26:56 DEBUG } 05:26:56 DEBUG } 05:26:56 DEBUG 05:26:56 DEBUG "result": [ 05:26:56 DEBUG } 05:26:56 DEBUG [...] 05:26:56 DEBUG { 05:26:56 DEBUG 05:26:56 DEBUG "metric": { 05:26:56 DEBUG 05:26:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:26:56 DEBUG 05:26:56 DEBUG }, 05:26:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:26:56 DEBUG "value": [ 05:26:56 DEBUG }, 05:26:56 DEBUG 05:26:56 DEBUG "value": [ 05:26:56 DEBUG 05:26:56 DEBUG 1713500816, 05:26:56 DEBUG 05:26:56 DEBUG "55.172" 05:26:56 DEBUG [...] 05:26:56 DEBUG ] 05:26:56 DEBUG 05:26:56 DEBUG } 05:26:56 DEBUG 05:26:56 DEBUG ] 05:26:56 DEBUG 05:26:56 DEBUG } 05:26:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:26:56 DEBUG } 05:26:56 DEBUG }, 05:26:56 DEBUG "value": [ 05:26:56 DEBUG 1713500816, 05:26:56 DEBUG "2217631744" 05:26:56 DEBUG ] 05:26:56 DEBUG } 05:26:56 DEBUG ] 05:26:56 DEBUG } 05:26:56 DEBUG } 05:26:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500816" 05:26:56 INFO [http_cmd]: http status code OK 05:26:56 DEBUG --- status code --- 05:26:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:26:56 DEBUG --- http response --- 05:26:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:26:56 DEBUG ----- output ----- 05:26:56 DEBUG { 05:26:56 DEBUG "status": "success", 05:26:56 DEBUG "data": { 05:26:56 DEBUG "resultType": "vector", 05:26:56 DEBUG "result": [ 05:26:56 DEBUG { 05:26:56 DEBUG "metric": { 05:26:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:26:56 DEBUG }, 05:26:56 DEBUG "value": [ 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG [...] 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:26:56 DEBUG }, 05:26:56 DEBUG "value": [ 05:26:56 DEBUG 1713500816, 05:26:56 DEBUG "2173.7" 05:26:56 DEBUG ] 05:26:56 DEBUG } 05:26:56 DEBUG ] 05:26:56 DEBUG } 05:26:56 DEBUG } 05:26:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500816" 05:26:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500816" 05:26:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713500816" 05:26:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500816" 05:26:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500816" 05:26:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500816" 05:26:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500816" 05:26:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500816" 05:26:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500816" 05:26:56 INFO [http_cmd]: http status code OK 05:26:56 INFO [http_cmd]: http status code OK 05:26:56 DEBUG --- status code --- 05:26:56 INFO [http_cmd]: http status code OK 05:26:56 DEBUG --- status code --- 05:26:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:26:56 DEBUG --- status code --- 05:26:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:26:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500816" 05:26:56 DEBUG --- http response --- 05:26:56 INFO [http_cmd]: http status code OK 05:26:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:26:56 DEBUG --- http response --- 05:26:56 INFO [http_cmd]: http status code OK 05:26:56 INFO [http_cmd]: http status code OK 05:26:56 INFO [http_cmd]: http status code OK 05:26:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:26:56 DEBUG --- status code --- 05:26:56 INFO [http_cmd]: http status code OK 05:26:56 INFO [http_cmd]: http status code OK 05:26:56 DEBUG --- http response --- 05:26:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:26:56 DEBUG --- status code --- 05:26:56 DEBUG --- status code --- 05:26:56 DEBUG --- status code --- 05:26:56 DEBUG ----- output ----- 05:26:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:26:56 DEBUG --- status code --- 05:26:56 DEBUG --- status code --- 05:26:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:26:56 DEBUG ----- output ----- 05:26:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:26:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:26:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:26:56 DEBUG { 05:26:56 DEBUG --- http response --- 05:26:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:26:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:26:56 DEBUG ----- output ----- 05:26:56 DEBUG { 05:26:56 DEBUG --- http response --- 05:26:56 DEBUG --- http response --- 05:26:56 DEBUG --- http response --- 05:26:56 DEBUG "status": "success", 05:26:56 DEBUG --- http response --- 05:26:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:26:56 DEBUG --- http response --- 05:26:56 DEBUG { 05:26:56 DEBUG "status": "success", 05:26:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:26:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:26:56 DEBUG "data": { 05:26:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:26:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:26:56 DEBUG ----- output ----- 05:26:56 DEBUG "status": "success", 05:26:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:26:56 DEBUG "data": { 05:26:56 DEBUG ----- output ----- 05:26:56 DEBUG ----- output ----- 05:26:56 DEBUG "resultType": "vector", 05:26:56 DEBUG ----- output ----- 05:26:56 DEBUG ----- output ----- 05:26:56 DEBUG { 05:26:56 DEBUG "data": { 05:26:56 DEBUG ----- output ----- 05:26:56 DEBUG "resultType": "vector", 05:26:56 DEBUG { 05:26:56 DEBUG { 05:26:56 DEBUG "result": [ 05:26:56 DEBUG { 05:26:56 DEBUG { 05:26:56 DEBUG "status": "success", 05:26:56 DEBUG "resultType": "vector", 05:26:56 DEBUG { 05:26:56 DEBUG "result": [ 05:26:56 DEBUG "status": "success", 05:26:56 DEBUG "status": "success", 05:26:56 DEBUG { 05:26:56 DEBUG "status": "success", 05:26:56 DEBUG "status": "success", 05:26:56 DEBUG "data": { 05:26:56 DEBUG "result": [ 05:26:56 DEBUG "status": "success", 05:26:56 DEBUG { 05:26:56 DEBUG "data": { 05:26:56 DEBUG "data": { 05:26:56 DEBUG "metric": { 05:26:56 DEBUG "data": { 05:26:56 DEBUG "data": { 05:26:56 DEBUG "resultType": "vector", 05:26:56 DEBUG { 05:26:56 DEBUG "data": { 05:26:56 DEBUG "metric": { 05:26:56 DEBUG "resultType": "vector", 05:26:56 DEBUG "resultType": "vector", 05:26:56 DEBUG "instance": "10.142.0.107:9100" 05:26:56 DEBUG "resultType": "vector", 05:26:56 DEBUG "resultType": "vector", 05:26:56 DEBUG "result": [ 05:26:56 DEBUG "metric": { 05:26:56 DEBUG "resultType": "vector", 05:26:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:26:56 DEBUG "result": [ 05:26:56 DEBUG "result": [ 05:26:56 DEBUG }, 05:26:56 DEBUG "result": [ 05:26:56 DEBUG "result": [ 05:26:56 DEBUG { 05:26:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:26:56 DEBUG "result": [ 05:26:56 DEBUG }, 05:26:56 DEBUG { 05:26:56 DEBUG { 05:26:56 DEBUG "value": [ 05:26:56 DEBUG { 05:26:56 DEBUG { 05:26:56 DEBUG "metric": { 05:26:56 DEBUG }, 05:26:56 DEBUG { 05:26:56 DEBUG "value": [ 05:26:56 DEBUG "metric": { 05:26:56 DEBUG "metric": { 05:26:56 DEBUG 05:26:56 DEBUG "metric": { 05:26:56 DEBUG "metric": { 05:26:56 DEBUG "pod": "ds-cts-0" 05:26:56 DEBUG "value": [ 05:26:56 DEBUG "metric": { 05:26:56 DEBUG 05:26:56 DEBUG "pod": "ds-idrepo-0" 05:26:56 DEBUG "pod": "ds-idrepo-0" 05:26:56 DEBUG 05:26:56 INFO [http_cmd]: http status code OK 05:26:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:26:56 DEBUG "instance": "10.142.0.107:9100" 05:26:56 DEBUG }, 05:26:56 DEBUG 05:26:56 DEBUG "pod": "ds-idrepo-0" 05:26:56 DEBUG 05:26:56 DEBUG }, 05:26:56 DEBUG }, 05:26:56 DEBUG 05:26:56 DEBUG --- status code --- 05:26:56 DEBUG }, 05:26:56 DEBUG }, 05:26:56 DEBUG "value": [ 05:26:56 DEBUG 05:26:56 DEBUG }, 05:26:56 DEBUG 05:26:56 DEBUG "value": [ 05:26:56 DEBUG "value": [ 05:26:56 DEBUG [...] 05:26:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:26:56 DEBUG "value": [ 05:26:56 DEBUG "value": [ 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG "value": [ 05:26:56 DEBUG [...] 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG --- http response --- 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG [...] 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG ----- output ----- 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG [...] 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG [...] 05:26:56 DEBUG [...] 05:26:56 DEBUG "instance": "10.142.0.43:9100" 05:26:56 DEBUG { 05:26:56 DEBUG [...] 05:26:56 DEBUG [...] 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG [...] 05:26:56 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG }, 05:26:56 DEBUG "status": "success", 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:26:56 DEBUG 05:26:56 DEBUG }, 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG "value": [ 05:26:56 DEBUG "data": { 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG }, 05:26:56 DEBUG 05:26:56 DEBUG "value": [ 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG 1713500816, 05:26:56 DEBUG "resultType": "vector", 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG "pod": "ds-cts-2" 05:26:56 DEBUG "value": [ 05:26:56 DEBUG 05:26:56 DEBUG 1713500816, 05:26:56 DEBUG "pod": "ds-idrepo-2" 05:26:56 DEBUG "pod": "ds-idrepo-2" 05:26:56 DEBUG "64337779.53333333" 05:26:56 DEBUG "result": [ 05:26:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:26:56 DEBUG "instance": "10.142.0.43:9100" 05:26:56 DEBUG }, 05:26:56 DEBUG 1713500816, 05:26:56 DEBUG "pod": "ds-idrepo-2" 05:26:56 DEBUG "0" 05:26:56 DEBUG }, 05:26:56 DEBUG }, 05:26:56 DEBUG ] 05:26:56 DEBUG { 05:26:56 DEBUG }, 05:26:56 DEBUG }, 05:26:56 DEBUG "value": [ 05:26:56 DEBUG "7.172791658033764" 05:26:56 DEBUG }, 05:26:56 DEBUG ] 05:26:56 DEBUG "value": [ 05:26:56 DEBUG "value": [ 05:26:56 DEBUG } 05:26:56 DEBUG "metric": { 05:26:56 DEBUG "value": [ 05:26:56 DEBUG "value": [ 05:26:56 DEBUG 1713500816, 05:26:56 DEBUG ] 05:26:56 DEBUG "value": [ 05:26:56 DEBUG } 05:26:56 DEBUG 1713500816, 05:26:56 DEBUG 1713500816, 05:26:56 DEBUG ] 05:26:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:26:56 DEBUG 1713500816, 05:26:56 DEBUG 1713500816, 05:26:56 DEBUG "0.0003666666666666667" 05:26:56 DEBUG } 05:26:56 DEBUG 1713500816, 05:26:56 DEBUG ] 05:26:56 DEBUG "0" 05:26:56 DEBUG "0" 05:26:56 DEBUG } 05:26:56 DEBUG }, 05:26:56 DEBUG "1997.0333333333333" 05:26:56 DEBUG "64332443.56666666" 05:26:56 DEBUG ] 05:26:56 DEBUG ] 05:26:56 DEBUG "0" 05:26:56 DEBUG } 05:26:56 DEBUG ] 05:26:56 DEBUG ] 05:26:56 DEBUG } 05:26:56 DEBUG "value": [ 05:26:56 DEBUG ] 05:26:56 DEBUG ] 05:26:56 DEBUG } 05:26:56 DEBUG } 05:26:56 DEBUG ] 05:26:56 DEBUG } 05:26:56 DEBUG } 05:26:56 DEBUG } 05:26:56 DEBUG 05:26:56 DEBUG } 05:26:56 DEBUG } 05:26:56 DEBUG ] 05:26:56 DEBUG } 05:26:56 DEBUG } 05:26:56 DEBUG ] 05:26:56 DEBUG ] 05:26:56 DEBUG 05:26:56 DEBUG ] 05:26:56 DEBUG ] 05:26:56 DEBUG } 05:26:56 DEBUG ] 05:26:56 DEBUG } 05:26:56 DEBUG } 05:26:56 DEBUG 05:26:56 DEBUG } 05:26:56 DEBUG } 05:26:56 DEBUG } 05:26:56 DEBUG } 05:26:56 DEBUG } 05:26:56 DEBUG } 05:26:56 DEBUG [...] 05:26:56 DEBUG } 05:26:56 DEBUG } 05:26:56 DEBUG } 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 05:26:56 DEBUG }, 05:26:56 DEBUG "value": [ 05:26:56 DEBUG 1713500816, 05:26:56 DEBUG "0" 05:26:56 DEBUG ] 05:26:56 DEBUG } 05:26:56 DEBUG ] 05:26:56 DEBUG } 05:26:56 DEBUG } 05:26:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500816" 05:26:56 INFO [http_cmd]: http status code OK 05:26:56 DEBUG --- status code --- 05:26:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:26:56 DEBUG --- http response --- 05:26:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:26:56 DEBUG ----- output ----- 05:26:56 DEBUG { 05:26:56 DEBUG "status": "success", 05:26:56 DEBUG "data": { 05:26:56 DEBUG "resultType": "vector", 05:26:56 DEBUG "result": [ 05:26:56 DEBUG { 05:26:56 DEBUG "metric": { 05:26:56 DEBUG "instance": "10.142.0.107:9100" 05:26:56 DEBUG }, 05:26:56 DEBUG "value": [ 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG [...] 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG 05:26:56 DEBUG "instance": "10.142.0.43:9100" 05:26:56 DEBUG }, 05:26:56 DEBUG "value": [ 05:26:56 DEBUG 1713500816, 05:26:56 DEBUG "0.3963333333333328" 05:26:56 DEBUG ] 05:26:56 DEBUG } 05:26:56 DEBUG ] 05:26:56 DEBUG } 05:26:56 DEBUG } 05:26:58 INFO 05:26:58 INFO [loop_until]: kubectl --namespace=xlou top pods 05:26:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:26:58 INFO [loop_until]: OK (rc = 0) 05:26:58 DEBUG --- stdout --- 05:26:58 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 13538m 9139Mi am-6fd554f8f7-hvvkw 13224m 8904Mi am-6fd554f8f7-pmj4b 14398m 8792Mi ds-cts-0 19259m 12888Mi ds-cts-1 18054m 12722Mi ds-cts-2 19720m 12786Mi ds-idrepo-0 5750m 3266Mi ds-idrepo-1 5901m 3199Mi ds-idrepo-2 5723m 3065Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 4m 1215Mi lodemon-6b89d774d5-rf7sk 1m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7032m 2027Mi 05:26:58 DEBUG --- stderr --- 05:26:58 DEBUG 05:26:59 INFO 05:26:59 INFO [loop_until]: kubectl --namespace=xlou top node 05:26:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:26:59 INFO [loop_until]: OK (rc = 0) 05:26:59 DEBUG --- stdout --- 05:26:59 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 69m 0% 6247Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5817m 19% 6720Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13260m 44% 11621Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 79m 0% 3258Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 19098m 63% 15367Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 161m 0% 5160Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 13724m 45% 11929Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 11274m 37% 2732Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 14815m 49% 10992Mi 9% gke-xlou-cdm-ds-0f601751-j488 6139m 20% 6174Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 18222m 61% 16773Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6410m 21% 6880Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 19552m 65% 14847Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 7278m 45% 5738Mi 9% 05:26:59 DEBUG --- stderr --- 05:26:59 DEBUG 05:27:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713500876" 05:27:56 INFO [http_cmd]: http status code OK 05:27:56 DEBUG --- status code --- 05:27:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:27:56 DEBUG --- http response --- 05:27:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:27:56 DEBUG ----- output ----- 05:27:56 DEBUG { 05:27:56 DEBUG "status": "success", 05:27:56 DEBUG "data": { 05:27:56 DEBUG "resultType": "vector", 05:27:56 DEBUG "result": [ 05:27:56 DEBUG { 05:27:56 DEBUG "metric": { 05:27:56 DEBUG "pod": "ds-idrepo-0" 05:27:56 DEBUG }, 05:27:56 DEBUG "value": [ 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG [...] 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG "pod": "ds-cts-2" 05:27:56 DEBUG }, 05:27:56 DEBUG "value": [ 05:27:56 DEBUG 1713500876, 05:27:56 DEBUG "0" 05:27:56 DEBUG ] 05:27:56 DEBUG } 05:27:56 DEBUG ] 05:27:56 DEBUG } 05:27:56 DEBUG } 05:27:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500876" 05:27:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500876" 05:27:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500876" 05:27:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500876" 05:27:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713500876" 05:27:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500876" 05:27:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500876" 05:27:56 INFO [http_cmd]: http status code OK 05:27:56 DEBUG --- status code --- 05:27:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:27:56 DEBUG --- http response --- 05:27:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:27:56 DEBUG ----- output ----- 05:27:56 DEBUG { 05:27:56 DEBUG "status": "success", 05:27:56 DEBUG "data": { 05:27:56 DEBUG "resultType": "vector", 05:27:56 DEBUG "result": [ 05:27:56 DEBUG { 05:27:56 DEBUG "metric": { 05:27:56 DEBUG "instance": "10.142.0.107:9100" 05:27:56 DEBUG }, 05:27:56 DEBUG "value": [ 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG [...] 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG "instance": "10.142.0.43:9100" 05:27:56 DEBUG }, 05:27:56 DEBUG "value": [ 05:27:56 DEBUG 1713500876, 05:27:56 DEBUG "494819826688" 05:27:56 DEBUG ] 05:27:56 DEBUG } 05:27:56 DEBUG ] 05:27:56 DEBUG } 05:27:56 DEBUG } 05:27:56 INFO [http_cmd]: http status code OK 05:27:56 INFO [http_cmd]: http status code OK 05:27:56 DEBUG --- status code --- 05:27:56 DEBUG --- status code --- 05:27:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:27:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:27:56 DEBUG --- http response --- 05:27:56 DEBUG --- http response --- 05:27:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:27:56 DEBUG ----- output ----- 05:27:56 DEBUG { 05:27:56 DEBUG "status": "success", 05:27:56 DEBUG "data": { 05:27:56 DEBUG "resultType": "vector", 05:27:56 DEBUG "result": [ 05:27:56 DEBUG { 05:27:56 DEBUG "metric": { 05:27:56 DEBUG "pod": "ds-cts-0" 05:27:56 DEBUG }, 05:27:56 DEBUG "value": [ 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG [...] 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG "pod": "ds-cts-2" 05:27:56 DEBUG }, 05:27:56 DEBUG "value": [ 05:27:56 DEBUG 1713500876, 05:27:56 DEBUG "314.2" 05:27:56 DEBUG ] 05:27:56 DEBUG } 05:27:56 DEBUG ] 05:27:56 DEBUG } 05:27:56 DEBUG } 05:27:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:27:56 DEBUG ----- output ----- 05:27:56 DEBUG { 05:27:56 DEBUG "status": "success", 05:27:56 DEBUG "data": { 05:27:56 DEBUG "resultType": "vector", 05:27:56 DEBUG "result": [ 05:27:56 DEBUG { 05:27:56 DEBUG "metric": { 05:27:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:27:56 DEBUG }, 05:27:56 DEBUG "value": [ 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG [...] 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:27:56 DEBUG }, 05:27:56 DEBUG "value": [ 05:27:56 DEBUG 1713500876, 05:27:56 DEBUG "2172" 05:27:56 DEBUG ] 05:27:56 DEBUG } 05:27:56 DEBUG ] 05:27:56 DEBUG } 05:27:56 DEBUG } 05:27:56 INFO [http_cmd]: http status code OK 05:27:56 DEBUG --- status code --- 05:27:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:27:56 DEBUG --- http response --- 05:27:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:27:56 DEBUG ----- output ----- 05:27:56 DEBUG { 05:27:56 DEBUG "status": "success", 05:27:56 DEBUG "data": { 05:27:56 DEBUG "resultType": "vector", 05:27:56 DEBUG "result": [ 05:27:56 DEBUG { 05:27:56 DEBUG "metric": { 05:27:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:27:56 DEBUG }, 05:27:56 DEBUG "value": [ 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG [...] 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:27:56 DEBUG }, 05:27:56 DEBUG "value": [ 05:27:56 DEBUG 1713500876, 05:27:56 DEBUG "32394.638069705095" 05:27:56 DEBUG ] 05:27:56 DEBUG } 05:27:56 DEBUG ] 05:27:56 DEBUG } 05:27:56 DEBUG } 05:27:56 ERROR [http_cmd]: ERROR 05:27:56 INFO [http_cmd]: http status code OK 05:27:56 ERROR --- status code --- 05:27:56 DEBUG --- status code --- 05:27:56 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:27:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:27:56 ERROR --- http response --- 05:27:56 DEBUG --- http response --- 05:27:56 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:27:56 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:27:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:27:56 DEBUG ----- output ----- 05:27:56 DEBUG { 05:27:56 DEBUG "status": "success", 05:27:56 DEBUG "data": { 05:27:56 DEBUG "resultType": "vector", 05:27:56 DEBUG "result": [ 05:27:56 DEBUG { 05:27:56 DEBUG "metric": { 05:27:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:27:56 DEBUG }, 05:27:56 DEBUG "value": [ 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG [...] 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:27:56 DEBUG }, 05:27:56 DEBUG "value": [ 05:27:56 DEBUG 1713500876, 05:27:56 DEBUG "0" 05:27:56 DEBUG ] 05:27:56 DEBUG } 05:27:56 DEBUG ] 05:27:56 DEBUG } 05:27:56 DEBUG } 05:27:56 INFO [http_cmd]: http status code OK 05:27:56 DEBUG --- status code --- 05:27:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:27:56 DEBUG --- http response --- 05:27:56 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:27:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500876" 05:27:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500876" 05:27:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500876" 05:27:56 INFO [http_cmd]: http status code OK 05:27:56 DEBUG --- status code --- 05:27:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:27:56 DEBUG --- http response --- 05:27:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:27:56 DEBUG ----- output ----- 05:27:56 DEBUG { 05:27:56 DEBUG "status": "success", 05:27:56 DEBUG "data": { 05:27:56 DEBUG "resultType": "vector", 05:27:56 DEBUG "result": [ 05:27:56 DEBUG { 05:27:56 DEBUG "metric": { 05:27:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:27:56 DEBUG }, 05:27:56 DEBUG "value": [ 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG [...] 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:27:56 DEBUG }, 05:27:56 DEBUG "value": [ 05:27:56 DEBUG 1713500876, 05:27:56 DEBUG "3604991" 05:27:56 DEBUG ] 05:27:56 DEBUG } 05:27:56 DEBUG ] 05:27:56 DEBUG } 05:27:56 DEBUG } 05:27:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500876" 05:27:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500876" 05:27:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713500876" 05:27:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713500876" 05:27:56 INFO [http_cmd]: http status code OK 05:27:56 INFO [http_cmd]: http status code OK 05:27:56 DEBUG --- status code --- 05:27:56 DEBUG --- status code --- 05:27:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:27:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:27:56 DEBUG --- http response --- 05:27:56 DEBUG --- http response --- 05:27:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:27:56 DEBUG ----- output ----- 05:27:56 DEBUG { 05:27:56 DEBUG "status": "success", 05:27:56 DEBUG "data": { 05:27:56 DEBUG "resultType": "vector", 05:27:56 DEBUG "result": [ 05:27:56 DEBUG { 05:27:56 DEBUG "metric": { 05:27:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:27:56 DEBUG }, 05:27:56 DEBUG "value": [ 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG [...] 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:27:56 DEBUG }, 05:27:56 DEBUG "value": [ 05:27:56 DEBUG 1713500876, 05:27:56 DEBUG "2173.8333333333335" 05:27:56 DEBUG ] 05:27:56 DEBUG } 05:27:56 DEBUG ] 05:27:56 DEBUG } 05:27:56 DEBUG } 05:27:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:27:56 DEBUG ----- output ----- 05:27:56 DEBUG { 05:27:56 DEBUG "status": "success", 05:27:56 DEBUG "data": { 05:27:56 DEBUG "resultType": "vector", 05:27:56 DEBUG "result": [ 05:27:56 DEBUG { 05:27:56 DEBUG "metric": { 05:27:56 DEBUG "pod": "ds-cts-0" 05:27:56 DEBUG }, 05:27:56 DEBUG "value": [ 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG [...] 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG "pod": "ds-cts-2" 05:27:56 DEBUG }, 05:27:56 DEBUG "value": [ 05:27:56 DEBUG 1713500876, 05:27:56 DEBUG "32.3" 05:27:56 DEBUG ] 05:27:56 DEBUG } 05:27:56 DEBUG ] 05:27:56 DEBUG } 05:27:56 DEBUG } 05:27:56 INFO [http_cmd]: http status code OK 05:27:56 INFO [http_cmd]: http status code OK 05:27:56 DEBUG --- status code --- 05:27:56 DEBUG --- status code --- 05:27:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:27:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:27:56 DEBUG --- http response --- 05:27:56 DEBUG --- http response --- 05:27:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:27:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:27:56 DEBUG ----- output ----- 05:27:56 DEBUG ----- output ----- 05:27:56 DEBUG { 05:27:56 DEBUG { 05:27:56 DEBUG "status": "success", 05:27:56 DEBUG "status": "success", 05:27:56 DEBUG "data": { 05:27:56 DEBUG "data": { 05:27:56 DEBUG "resultType": "vector", 05:27:56 DEBUG "resultType": "vector", 05:27:56 DEBUG "result": [ 05:27:56 DEBUG "result": [ 05:27:56 DEBUG { 05:27:56 DEBUG { 05:27:56 DEBUG "metric": { 05:27:56 DEBUG "metric": { 05:27:56 DEBUG "instance": "10.142.0.107:9100" 05:27:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:27:56 DEBUG }, 05:27:56 DEBUG }, 05:27:56 DEBUG "value": [ 05:27:56 DEBUG "value": [ 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 INFO [http_cmd]: http status code OK 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG --- status code --- 05:27:56 DEBUG 05:27:56 DEBUG [...] 05:27:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:27:56 DEBUG [...] 05:27:56 DEBUG 05:27:56 DEBUG --- http response --- 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG ----- output ----- 05:27:56 DEBUG "instance": "10.142.0.43:9100" 05:27:56 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:27:56 DEBUG { 05:27:56 INFO [http_cmd]: http status code OK 05:27:56 DEBUG }, 05:27:56 DEBUG }, 05:27:56 DEBUG "status": "success", 05:27:56 DEBUG --- status code --- 05:27:56 DEBUG "value": [ 05:27:56 DEBUG "value": [ 05:27:56 DEBUG "data": { 05:27:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:27:56 DEBUG 1713500876, 05:27:56 DEBUG 1713500876, 05:27:56 DEBUG "resultType": "vector", 05:27:56 DEBUG --- http response --- 05:27:56 DEBUG "0.494833333333339" 05:27:56 DEBUG "7.100131632540036" 05:27:56 DEBUG "result": [ 05:27:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:27:56 DEBUG ] 05:27:56 DEBUG ] 05:27:56 DEBUG { 05:27:56 DEBUG ----- output ----- 05:27:56 DEBUG } 05:27:56 DEBUG } 05:27:56 DEBUG "metric": { 05:27:56 DEBUG { 05:27:56 DEBUG ] 05:27:56 DEBUG ] 05:27:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:27:56 DEBUG "status": "success", 05:27:56 DEBUG } 05:27:56 DEBUG } 05:27:56 DEBUG }, 05:27:56 DEBUG "data": { 05:27:56 DEBUG } 05:27:56 DEBUG } 05:27:56 DEBUG "value": [ 05:27:56 DEBUG "resultType": "vector", 05:27:56 DEBUG 05:27:56 DEBUG "result": [ 05:27:56 DEBUG 05:27:56 DEBUG { 05:27:56 DEBUG 05:27:56 DEBUG "metric": { 05:27:56 DEBUG [...] 05:27:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:27:56 DEBUG 05:27:56 DEBUG }, 05:27:56 DEBUG 05:27:56 DEBUG "value": [ 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:27:56 DEBUG 05:27:56 DEBUG }, 05:27:56 DEBUG 05:27:56 DEBUG "value": [ 05:27:56 DEBUG [...] 05:27:56 DEBUG 1713500876, 05:27:56 DEBUG 05:27:56 DEBUG "55.172" 05:27:56 DEBUG 05:27:56 DEBUG ] 05:27:56 DEBUG 05:27:56 DEBUG } 05:27:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:27:56 DEBUG ] 05:27:56 DEBUG }, 05:27:56 DEBUG } 05:27:56 DEBUG "value": [ 05:27:56 DEBUG } 05:27:56 DEBUG 1713500876, 05:27:56 DEBUG "2220978176" 05:27:56 DEBUG ] 05:27:56 DEBUG } 05:27:56 DEBUG ] 05:27:56 DEBUG } 05:27:56 DEBUG } 05:27:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500876" 05:27:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500876" 05:27:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500876" 05:27:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500876" 05:27:56 INFO [http_cmd]: http status code OK 05:27:56 DEBUG --- status code --- 05:27:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:27:56 DEBUG --- http response --- 05:27:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:27:56 DEBUG ----- output ----- 05:27:56 DEBUG { 05:27:56 DEBUG "status": "success", 05:27:56 DEBUG "data": { 05:27:56 DEBUG "resultType": "vector", 05:27:56 DEBUG "result": [ 05:27:56 DEBUG { 05:27:56 DEBUG "metric": { 05:27:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:27:56 DEBUG }, 05:27:56 DEBUG "value": [ 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG [...] 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:27:56 DEBUG }, 05:27:56 DEBUG "value": [ 05:27:56 DEBUG 1713500876, 05:27:56 DEBUG "2006.8" 05:27:56 DEBUG ] 05:27:56 DEBUG } 05:27:56 DEBUG ] 05:27:56 DEBUG } 05:27:56 DEBUG } 05:27:56 INFO [http_cmd]: http status code OK 05:27:56 DEBUG --- status code --- 05:27:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:27:56 DEBUG --- http response --- 05:27:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:27:56 DEBUG ----- output ----- 05:27:56 DEBUG { 05:27:56 DEBUG "status": "success", 05:27:56 DEBUG "data": { 05:27:56 DEBUG "resultType": "vector", 05:27:56 DEBUG "result": [ 05:27:56 DEBUG { 05:27:56 DEBUG "metric": { 05:27:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:27:56 DEBUG }, 05:27:56 DEBUG "value": [ 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG [...] 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:27:56 DEBUG }, 05:27:56 DEBUG "value": [ 05:27:56 DEBUG 1713500876, 05:27:56 DEBUG "0" 05:27:56 DEBUG ] 05:27:56 DEBUG } 05:27:56 DEBUG ] 05:27:56 DEBUG } 05:27:56 DEBUG } 05:27:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500876" 05:27:56 INFO [http_cmd]: http status code OK 05:27:56 DEBUG --- status code --- 05:27:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:27:56 DEBUG --- http response --- 05:27:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:27:56 DEBUG ----- output ----- 05:27:56 DEBUG { 05:27:56 DEBUG "status": "success", 05:27:56 DEBUG "data": { 05:27:56 DEBUG "resultType": "vector", 05:27:56 DEBUG "result": [ 05:27:56 DEBUG { 05:27:56 DEBUG "metric": { 05:27:56 DEBUG "instance": "10.142.0.107:9100" 05:27:56 DEBUG }, 05:27:56 DEBUG "value": [ 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG [...] 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG "instance": "10.142.0.43:9100" 05:27:56 DEBUG }, 05:27:56 DEBUG "value": [ 05:27:56 DEBUG 1713500876, 05:27:56 DEBUG "64192468.36666666" 05:27:56 DEBUG ] 05:27:56 DEBUG } 05:27:56 DEBUG ] 05:27:56 DEBUG } 05:27:56 DEBUG } 05:27:56 INFO [http_cmd]: http status code OK 05:27:56 DEBUG --- status code --- 05:27:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:27:56 DEBUG --- http response --- 05:27:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:27:56 DEBUG ----- output ----- 05:27:56 DEBUG { 05:27:56 DEBUG "status": "success", 05:27:56 DEBUG "data": { 05:27:56 DEBUG "resultType": "vector", 05:27:56 DEBUG "result": [ 05:27:56 DEBUG { 05:27:56 DEBUG "metric": { 05:27:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:27:56 DEBUG }, 05:27:56 DEBUG "value": [ 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG [...] 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:27:56 DEBUG }, 05:27:56 DEBUG "value": [ 05:27:56 DEBUG 1713500876, 05:27:56 DEBUG "2172.2" 05:27:56 DEBUG ] 05:27:56 DEBUG } 05:27:56 DEBUG ] 05:27:56 DEBUG } 05:27:56 DEBUG } 05:27:56 INFO [http_cmd]: http status code OK 05:27:56 DEBUG --- status code --- 05:27:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:27:56 DEBUG --- http response --- 05:27:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:27:56 DEBUG ----- output ----- 05:27:56 DEBUG { 05:27:56 DEBUG "status": "success", 05:27:56 DEBUG "data": { 05:27:56 DEBUG "resultType": "vector", 05:27:56 DEBUG "result": [ 05:27:56 DEBUG { 05:27:56 DEBUG "metric": { 05:27:56 DEBUG "instance": "10.142.0.107:9100" 05:27:56 DEBUG }, 05:27:56 DEBUG "value": [ 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG [...] 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG "instance": "10.142.0.43:9100" 05:27:56 DEBUG }, 05:27:56 DEBUG "value": [ 05:27:56 DEBUG 1713500876, 05:27:56 DEBUG "64185804.06666666" 05:27:56 DEBUG ] 05:27:56 DEBUG } 05:27:56 DEBUG ] 05:27:56 DEBUG } 05:27:56 DEBUG } 05:27:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500876" 05:27:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500876" 05:27:56 INFO [http_cmd]: http status code OK 05:27:56 DEBUG --- status code --- 05:27:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:27:56 DEBUG --- http response --- 05:27:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:27:56 DEBUG ----- output ----- 05:27:56 DEBUG { 05:27:56 DEBUG "status": "success", 05:27:56 DEBUG "data": { 05:27:56 DEBUG "resultType": "vector", 05:27:56 DEBUG "result": [ 05:27:56 DEBUG { 05:27:56 DEBUG "metric": { 05:27:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:27:56 DEBUG }, 05:27:56 DEBUG "value": [ 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG [...] 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:27:56 DEBUG }, 05:27:56 DEBUG "value": [ 05:27:56 DEBUG 1713500876, 05:27:56 DEBUG "0" 05:27:56 DEBUG ] 05:27:56 DEBUG } 05:27:56 DEBUG ] 05:27:56 DEBUG } 05:27:56 DEBUG } 05:27:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500876" 05:27:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500876" 05:27:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713500876" 05:27:56 INFO [http_cmd]: http status code OK 05:27:56 DEBUG --- status code --- 05:27:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:27:56 DEBUG --- http response --- 05:27:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:27:56 DEBUG ----- output ----- 05:27:56 DEBUG { 05:27:56 DEBUG "status": "success", 05:27:56 DEBUG "data": { 05:27:56 DEBUG "resultType": "vector", 05:27:56 DEBUG "result": [ 05:27:56 DEBUG { 05:27:56 DEBUG "metric": { 05:27:56 DEBUG "pod": "ds-idrepo-0" 05:27:56 DEBUG }, 05:27:56 DEBUG "value": [ 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG [...] 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG "pod": "ds-idrepo-2" 05:27:56 DEBUG }, 05:27:56 DEBUG "value": [ 05:27:56 DEBUG 1713500876, 05:27:56 DEBUG "0" 05:27:56 DEBUG ] 05:27:56 DEBUG } 05:27:56 DEBUG ] 05:27:56 DEBUG } 05:27:56 DEBUG } 05:27:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500876" 05:27:56 INFO [http_cmd]: http status code OK 05:27:56 DEBUG --- status code --- 05:27:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:27:56 DEBUG --- http response --- 05:27:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:27:56 DEBUG ----- output ----- 05:27:56 DEBUG { 05:27:56 DEBUG "status": "success", 05:27:56 DEBUG "data": { 05:27:56 DEBUG "resultType": "vector", 05:27:56 DEBUG "result": [ 05:27:56 DEBUG { 05:27:56 DEBUG "metric": { 05:27:56 DEBUG "pod": "ds-cts-0" 05:27:56 DEBUG }, 05:27:56 DEBUG "value": [ 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG [...] 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG "pod": "ds-cts-2" 05:27:56 DEBUG }, 05:27:56 DEBUG "value": [ 05:27:56 DEBUG 1713500876, 05:27:56 DEBUG "0.0009410133333333331" 05:27:56 DEBUG ] 05:27:56 DEBUG } 05:27:56 DEBUG ] 05:27:56 DEBUG } 05:27:56 DEBUG } 05:27:56 INFO [http_cmd]: http status code OK 05:27:56 DEBUG --- status code --- 05:27:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:27:56 DEBUG --- http response --- 05:27:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:27:56 DEBUG ----- output ----- 05:27:56 DEBUG { 05:27:56 DEBUG "status": "success", 05:27:56 DEBUG "data": { 05:27:56 DEBUG "resultType": "vector", 05:27:56 DEBUG "result": [ 05:27:56 DEBUG { 05:27:56 DEBUG "metric": { 05:27:56 DEBUG "pod": "ds-idrepo-0" 05:27:56 DEBUG }, 05:27:56 DEBUG "value": [ 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG [...] 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG "pod": "ds-idrepo-2" 05:27:56 DEBUG }, 05:27:56 DEBUG "value": [ 05:27:56 DEBUG 1713500876, 05:27:56 DEBUG "0" 05:27:56 DEBUG ] 05:27:56 DEBUG } 05:27:56 DEBUG ] 05:27:56 DEBUG } 05:27:56 DEBUG } 05:27:56 INFO [http_cmd]: http status code OK 05:27:56 DEBUG --- status code --- 05:27:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:27:56 DEBUG --- http response --- 05:27:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:27:56 DEBUG ----- output ----- 05:27:56 DEBUG { 05:27:56 DEBUG "status": "success", 05:27:56 DEBUG "data": { 05:27:56 DEBUG "resultType": "vector", 05:27:56 DEBUG "result": [ 05:27:56 DEBUG { 05:27:56 DEBUG "metric": { 05:27:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:27:56 DEBUG }, 05:27:56 DEBUG "value": [ 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG [...] 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:27:56 DEBUG }, 05:27:56 DEBUG "value": [ 05:27:56 DEBUG 1713500876, 05:27:56 DEBUG "7.122008449405028" 05:27:56 DEBUG ] 05:27:56 DEBUG } 05:27:56 DEBUG ] 05:27:56 DEBUG } 05:27:56 DEBUG } 05:27:56 INFO [http_cmd]: http status code OK 05:27:56 DEBUG --- status code --- 05:27:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:27:56 DEBUG --- http response --- 05:27:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:27:56 DEBUG ----- output ----- 05:27:56 DEBUG { 05:27:56 DEBUG "status": "success", 05:27:56 DEBUG "data": { 05:27:56 DEBUG "resultType": "vector", 05:27:56 DEBUG "result": [ 05:27:56 DEBUG { 05:27:56 DEBUG "metric": { 05:27:56 DEBUG "pod": "ds-idrepo-0" 05:27:56 DEBUG }, 05:27:56 DEBUG "value": [ 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG [...] 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG "pod": "ds-idrepo-2" 05:27:56 DEBUG }, 05:27:56 DEBUG "value": [ 05:27:56 DEBUG 1713500876, 05:27:56 DEBUG "0" 05:27:56 DEBUG ] 05:27:56 DEBUG } 05:27:56 DEBUG ] 05:27:56 DEBUG } 05:27:56 DEBUG } 05:27:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500876" 05:27:56 INFO [http_cmd]: http status code OK 05:27:56 DEBUG --- status code --- 05:27:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:27:56 DEBUG --- http response --- 05:27:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:27:56 DEBUG ----- output ----- 05:27:56 DEBUG { 05:27:56 DEBUG "status": "success", 05:27:56 DEBUG "data": { 05:27:56 DEBUG "resultType": "vector", 05:27:56 DEBUG "result": [ 05:27:56 DEBUG { 05:27:56 DEBUG "metric": { 05:27:56 DEBUG "instance": "10.142.0.107:9100" 05:27:56 DEBUG }, 05:27:56 DEBUG "value": [ 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG [...] 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG 05:27:56 DEBUG "instance": "10.142.0.43:9100" 05:27:56 DEBUG }, 05:27:56 DEBUG "value": [ 05:27:56 DEBUG 1713500876, 05:27:56 DEBUG "0.2933333333333342" 05:27:56 DEBUG ] 05:27:56 DEBUG } 05:27:56 DEBUG ] 05:27:56 DEBUG } 05:27:56 DEBUG } 05:27:58 INFO 05:27:58 INFO [loop_until]: kubectl --namespace=xlou top pods 05:27:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:27:58 INFO [loop_until]: OK (rc = 0) 05:27:58 DEBUG --- stdout --- 05:27:58 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 13606m 9140Mi am-6fd554f8f7-hvvkw 13712m 8905Mi am-6fd554f8f7-pmj4b 14261m 8792Mi ds-cts-0 18824m 12657Mi ds-cts-1 18210m 12619Mi ds-cts-2 19507m 13319Mi ds-idrepo-0 5801m 3267Mi ds-idrepo-1 5695m 3198Mi ds-idrepo-2 5625m 3066Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 5m 1172Mi idm-b85f9d978-wnhxb 3m 1215Mi lodemon-6b89d774d5-rf7sk 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7120m 2028Mi 05:27:58 DEBUG --- stderr --- 05:27:58 DEBUG 05:27:59 INFO 05:27:59 INFO [loop_until]: kubectl --namespace=xlou top node 05:27:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:27:59 INFO [loop_until]: OK (rc = 0) 05:27:59 DEBUG --- stdout --- 05:27:59 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 68m 0% 6245Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5815m 19% 6721Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13227m 44% 11618Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 89m 0% 3256Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 18713m 62% 15282Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 154m 0% 5161Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 14016m 46% 11929Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 11290m 37% 2739Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 15205m 50% 10990Mi 9% gke-xlou-cdm-ds-0f601751-j488 6127m 20% 6166Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 18026m 60% 16636Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 6421m 21% 6868Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 19323m 64% 15198Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 7260m 45% 5742Mi 9% 05:27:59 DEBUG --- stderr --- 05:27:59 DEBUG 05:28:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713500936" 05:28:56 INFO [http_cmd]: http status code OK 05:28:56 DEBUG --- status code --- 05:28:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:28:56 DEBUG --- http response --- 05:28:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:28:56 DEBUG ----- output ----- 05:28:56 DEBUG { 05:28:56 DEBUG "status": "success", 05:28:56 DEBUG "data": { 05:28:56 DEBUG "resultType": "vector", 05:28:56 DEBUG "result": [ 05:28:56 DEBUG { 05:28:56 DEBUG "metric": { 05:28:56 DEBUG "pod": "ds-idrepo-0" 05:28:56 DEBUG }, 05:28:56 DEBUG "value": [ 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG [...] 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG "pod": "ds-cts-2" 05:28:56 DEBUG }, 05:28:56 DEBUG "value": [ 05:28:56 DEBUG 1713500936, 05:28:56 DEBUG "0" 05:28:56 DEBUG ] 05:28:56 DEBUG } 05:28:56 DEBUG ] 05:28:56 DEBUG } 05:28:56 DEBUG } 05:28:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500936" 05:28:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500936" 05:28:56 INFO [http_cmd]: http status code OK 05:28:56 DEBUG --- status code --- 05:28:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:28:56 DEBUG --- http response --- 05:28:56 INFO [http_cmd]: http status code OK 05:28:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:28:56 DEBUG --- status code --- 05:28:56 DEBUG ----- output ----- 05:28:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:28:56 DEBUG { 05:28:56 DEBUG --- http response --- 05:28:56 DEBUG "status": "success", 05:28:56 DEBUG "data": { 05:28:56 DEBUG "resultType": "vector", 05:28:56 DEBUG "result": [ 05:28:56 DEBUG { 05:28:56 DEBUG "metric": { 05:28:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:28:56 DEBUG }, 05:28:56 DEBUG "value": [ 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG [...] 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:28:56 DEBUG }, 05:28:56 DEBUG "value": [ 05:28:56 DEBUG 1713500936, 05:28:56 DEBUG "2168.3" 05:28:56 DEBUG ] 05:28:56 DEBUG } 05:28:56 DEBUG ] 05:28:56 DEBUG } 05:28:56 DEBUG } 05:28:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 05:28:56 DEBUG ----- output ----- 05:28:56 DEBUG { 05:28:56 DEBUG "status": "success", 05:28:56 DEBUG "data": { 05:28:56 DEBUG "resultType": "vector", 05:28:56 DEBUG "result": [ 05:28:56 DEBUG { 05:28:56 DEBUG "metric": { 05:28:56 DEBUG "pod": "ds-idrepo-1" 05:28:56 DEBUG }, 05:28:56 DEBUG "value": [ 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG [...] 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:28:56 DEBUG }, 05:28:56 DEBUG "value": [ 05:28:56 DEBUG 1713500936, 05:28:56 DEBUG "0" 05:28:56 DEBUG ] 05:28:56 DEBUG } 05:28:56 DEBUG ] 05:28:56 DEBUG } 05:28:56 DEBUG } 05:28:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500936" 05:28:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500936" 05:28:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500936" 05:28:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713500936" 05:28:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500936" 05:28:56 ERROR [http_cmd]: ERROR 05:28:56 ERROR --- status code --- 05:28:56 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:28:56 ERROR --- http response --- 05:28:56 INFO [http_cmd]: http status code OK 05:28:56 INFO [http_cmd]: http status code OK 05:28:56 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:28:56 DEBUG --- status code --- 05:28:56 DEBUG --- status code --- 05:28:56 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:28:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:28:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:28:56 DEBUG --- http response --- 05:28:56 DEBUG --- http response --- 05:28:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:28:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:28:56 DEBUG ----- output ----- 05:28:56 DEBUG ----- output ----- 05:28:56 DEBUG { 05:28:56 DEBUG { 05:28:56 DEBUG "status": "success", 05:28:56 DEBUG "status": "success", 05:28:56 DEBUG "data": { 05:28:56 INFO [http_cmd]: http status code OK 05:28:56 DEBUG "data": { 05:28:56 DEBUG "resultType": "vector", 05:28:56 DEBUG --- status code --- 05:28:56 INFO [http_cmd]: http status code OK 05:28:56 DEBUG "resultType": "vector", 05:28:56 DEBUG "result": [ 05:28:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:28:56 DEBUG --- status code --- 05:28:56 DEBUG "result": [ 05:28:56 DEBUG { 05:28:56 DEBUG --- http response --- 05:28:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:28:56 DEBUG { 05:28:56 DEBUG "metric": { 05:28:56 DEBUG --- http response --- 05:28:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:28:56 DEBUG "metric": { 05:28:56 DEBUG "instance": "10.142.0.107:9100" 05:28:56 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:28:56 DEBUG ----- output ----- 05:28:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:28:56 DEBUG }, 05:28:56 DEBUG { 05:28:56 DEBUG }, 05:28:56 DEBUG "value": [ 05:28:56 DEBUG "status": "success", 05:28:56 DEBUG "value": [ 05:28:56 DEBUG 05:28:56 DEBUG "data": { 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG "resultType": "vector", 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG "result": [ 05:28:56 DEBUG 05:28:56 DEBUG [...] 05:28:56 DEBUG { 05:28:56 DEBUG [...] 05:28:56 DEBUG 05:28:56 DEBUG "metric": { 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG "pod": "ds-cts-0" 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG }, 05:28:56 DEBUG 05:28:56 DEBUG "instance": "10.142.0.43:9100" 05:28:56 DEBUG "value": [ 05:28:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:28:56 DEBUG }, 05:28:56 DEBUG 05:28:56 DEBUG }, 05:28:56 DEBUG "value": [ 05:28:56 DEBUG 05:28:56 DEBUG "value": [ 05:28:56 DEBUG 1713500936, 05:28:56 DEBUG 05:28:56 DEBUG 1713500936, 05:28:56 DEBUG "498708270080" 05:28:56 DEBUG [...] 05:28:56 DEBUG "0" 05:28:56 DEBUG ] 05:28:56 DEBUG 05:28:56 DEBUG ] 05:28:56 DEBUG } 05:28:56 DEBUG 05:28:56 DEBUG } 05:28:56 DEBUG ] 05:28:56 DEBUG 05:28:56 DEBUG ] 05:28:56 DEBUG } 05:28:56 DEBUG "pod": "ds-cts-2" 05:28:56 DEBUG } 05:28:56 DEBUG } 05:28:56 DEBUG }, 05:28:56 DEBUG } 05:28:56 DEBUG "value": [ 05:28:56 DEBUG 1713500936, 05:28:56 DEBUG "373" 05:28:56 DEBUG ] 05:28:56 DEBUG } 05:28:56 DEBUG ] 05:28:56 DEBUG } 05:28:56 DEBUG } 05:28:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713500936" 05:28:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500936" 05:28:56 INFO [http_cmd]: http status code OK 05:28:56 DEBUG --- status code --- 05:28:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:28:56 DEBUG --- http response --- 05:28:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:28:56 DEBUG ----- output ----- 05:28:56 DEBUG { 05:28:56 DEBUG "status": "success", 05:28:56 DEBUG "data": { 05:28:56 DEBUG "resultType": "vector", 05:28:56 DEBUG "result": [ 05:28:56 DEBUG { 05:28:56 DEBUG "metric": { 05:28:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:28:56 DEBUG }, 05:28:56 DEBUG "value": [ 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG [...] 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:28:56 DEBUG }, 05:28:56 DEBUG "value": [ 05:28:56 DEBUG 1713500936, 05:28:56 DEBUG "55.172" 05:28:56 DEBUG ] 05:28:56 DEBUG } 05:28:56 DEBUG ] 05:28:56 DEBUG } 05:28:56 DEBUG } 05:28:56 INFO [http_cmd]: http status code OK 05:28:56 DEBUG --- status code --- 05:28:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:28:56 DEBUG --- http response --- 05:28:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:28:56 DEBUG ----- output ----- 05:28:56 DEBUG { 05:28:56 DEBUG "status": "success", 05:28:56 DEBUG "data": { 05:28:56 DEBUG "resultType": "vector", 05:28:56 DEBUG "result": [ 05:28:56 DEBUG { 05:28:56 DEBUG "metric": { 05:28:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:28:56 DEBUG }, 05:28:56 DEBUG "value": [ 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG [...] 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:28:56 DEBUG }, 05:28:56 DEBUG "value": [ 05:28:56 DEBUG 1713500936, 05:28:56 DEBUG "3614998.5" 05:28:56 DEBUG ] 05:28:56 DEBUG } 05:28:56 DEBUG ] 05:28:56 DEBUG } 05:28:56 DEBUG } 05:28:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500936" 05:28:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500936" 05:28:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500936" 05:28:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500936" 05:28:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713500936" 05:28:56 INFO [http_cmd]: http status code OK 05:28:56 INFO [http_cmd]: http status code OK 05:28:56 DEBUG --- status code --- 05:28:56 DEBUG --- status code --- 05:28:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:28:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:28:56 DEBUG --- http response --- 05:28:56 DEBUG --- http response --- 05:28:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:28:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:28:56 DEBUG ----- output ----- 05:28:56 DEBUG ----- output ----- 05:28:56 DEBUG { 05:28:56 DEBUG { 05:28:56 DEBUG "status": "success", 05:28:56 DEBUG "status": "success", 05:28:56 DEBUG "data": { 05:28:56 DEBUG "data": { 05:28:56 DEBUG "resultType": "vector", 05:28:56 DEBUG "resultType": "vector", 05:28:56 DEBUG "result": [ 05:28:56 DEBUG "result": [ 05:28:56 DEBUG { 05:28:56 DEBUG { 05:28:56 DEBUG "metric": { 05:28:56 DEBUG "metric": { 05:28:56 DEBUG "pod": "ds-cts-0" 05:28:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:28:56 DEBUG }, 05:28:56 DEBUG }, 05:28:56 DEBUG "value": [ 05:28:56 DEBUG "value": [ 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG [...] 05:28:56 DEBUG [...] 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG "pod": "ds-cts-2" 05:28:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:28:56 DEBUG }, 05:28:56 DEBUG }, 05:28:56 DEBUG "value": [ 05:28:56 DEBUG "value": [ 05:28:56 DEBUG 1713500936, 05:28:56 DEBUG 1713500936, 05:28:56 DEBUG "27.466666666666665" 05:28:56 DEBUG "2168.366666666667" 05:28:56 DEBUG ] 05:28:56 DEBUG ] 05:28:56 DEBUG } 05:28:56 DEBUG } 05:28:56 DEBUG ] 05:28:56 DEBUG ] 05:28:56 DEBUG } 05:28:56 DEBUG } 05:28:56 DEBUG } 05:28:56 DEBUG } 05:28:56 INFO [http_cmd]: http status code OK 05:28:56 DEBUG --- status code --- 05:28:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:28:56 DEBUG --- http response --- 05:28:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 05:28:56 DEBUG ----- output ----- 05:28:56 DEBUG { 05:28:56 DEBUG "status": "success", 05:28:56 DEBUG "data": { 05:28:56 DEBUG "resultType": "vector", 05:28:56 DEBUG "result": [ 05:28:56 DEBUG { 05:28:56 DEBUG "metric": { 05:28:56 DEBUG "pod": "ds-idrepo-1" 05:28:56 DEBUG }, 05:28:56 DEBUG "value": [ 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG [...] 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:28:56 DEBUG }, 05:28:56 DEBUG "value": [ 05:28:56 DEBUG 1713500936, 05:28:56 DEBUG "0.000028104378119453124" 05:28:56 DEBUG ] 05:28:56 DEBUG } 05:28:56 DEBUG ] 05:28:56 DEBUG } 05:28:56 DEBUG } 05:28:56 INFO [http_cmd]: http status code OK 05:28:56 DEBUG --- status code --- 05:28:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:28:56 DEBUG --- http response --- 05:28:56 INFO [http_cmd]: http status code OK 05:28:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:28:56 DEBUG --- status code --- 05:28:56 DEBUG ----- output ----- 05:28:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:28:56 DEBUG { 05:28:56 DEBUG --- http response --- 05:28:56 DEBUG "status": "success", 05:28:56 DEBUG "data": { 05:28:56 DEBUG "resultType": "vector", 05:28:56 DEBUG "result": [ 05:28:56 DEBUG { 05:28:56 DEBUG "metric": { 05:28:56 DEBUG "instance": "10.142.0.107:9100" 05:28:56 DEBUG }, 05:28:56 DEBUG "value": [ 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG [...] 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG "instance": "10.142.0.43:9100" 05:28:56 DEBUG }, 05:28:56 DEBUG "value": [ 05:28:56 DEBUG 1713500936, 05:28:56 DEBUG "0.4986333333333287" 05:28:56 DEBUG ] 05:28:56 DEBUG } 05:28:56 DEBUG ] 05:28:56 DEBUG } 05:28:56 DEBUG } 05:28:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:28:56 DEBUG ----- output ----- 05:28:56 DEBUG { 05:28:56 DEBUG "status": "success", 05:28:56 DEBUG "data": { 05:28:56 DEBUG "resultType": "vector", 05:28:56 DEBUG "result": [ 05:28:56 DEBUG { 05:28:56 DEBUG "metric": { 05:28:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:28:56 DEBUG }, 05:28:56 DEBUG "value": [ 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG [...] 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:28:56 DEBUG }, 05:28:56 DEBUG "value": [ 05:28:56 DEBUG 1713500936, 05:28:56 DEBUG "2224758784" 05:28:56 DEBUG ] 05:28:56 DEBUG } 05:28:56 DEBUG ] 05:28:56 DEBUG } 05:28:56 DEBUG } 05:28:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500936" 05:28:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500936" 05:28:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500936" 05:28:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500936" 05:28:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500936" 05:28:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500936" 05:28:56 INFO [http_cmd]: http status code OK 05:28:56 INFO [http_cmd]: http status code OK 05:28:56 DEBUG --- status code --- 05:28:56 INFO [http_cmd]: http status code OK 05:28:56 DEBUG --- status code --- 05:28:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:28:56 DEBUG --- status code --- 05:28:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:28:56 DEBUG --- http response --- 05:28:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:28:56 DEBUG --- http response --- 05:28:56 DEBUG --- http response --- 05:28:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:28:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 05:28:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 05:28:56 INFO [http_cmd]: http status code OK 05:28:56 DEBUG ----- output ----- 05:28:56 DEBUG ----- output ----- 05:28:56 DEBUG ----- output ----- 05:28:56 DEBUG --- status code --- 05:28:56 DEBUG { 05:28:56 DEBUG { 05:28:56 DEBUG { 05:28:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:28:56 DEBUG "status": "success", 05:28:56 DEBUG "status": "success", 05:28:56 DEBUG "status": "success", 05:28:56 DEBUG --- http response --- 05:28:56 DEBUG "data": { 05:28:56 DEBUG "data": { 05:28:56 DEBUG "data": { 05:28:56 DEBUG "resultType": "vector", 05:28:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:28:56 DEBUG "resultType": "vector", 05:28:56 DEBUG "resultType": "vector", 05:28:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500936" 05:28:56 DEBUG "result": [ 05:28:56 DEBUG ----- output ----- 05:28:56 DEBUG "result": [ 05:28:56 DEBUG "result": [ 05:28:56 DEBUG { 05:28:56 DEBUG { 05:28:56 INFO [http_cmd]: http status code OK 05:28:56 DEBUG { 05:28:56 DEBUG { 05:28:56 DEBUG "metric": { 05:28:56 DEBUG "status": "success", 05:28:56 DEBUG --- status code --- 05:28:56 DEBUG "metric": { 05:28:56 DEBUG "metric": { 05:28:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:28:56 DEBUG "data": { 05:28:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:28:56 DEBUG "pod": "ds-idrepo-1" 05:28:56 DEBUG "pod": "ds-idrepo-1" 05:28:56 DEBUG }, 05:28:56 DEBUG "resultType": "vector", 05:28:56 DEBUG --- http response --- 05:28:56 DEBUG }, 05:28:56 DEBUG }, 05:28:56 DEBUG "value": [ 05:28:56 DEBUG "result": [ 05:28:56 DEBUG "value": [ 05:28:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:28:56 DEBUG "value": [ 05:28:56 INFO [http_cmd]: http status code OK 05:28:56 DEBUG 05:28:56 DEBUG { 05:28:56 DEBUG 05:28:56 DEBUG ----- output ----- 05:28:56 DEBUG 05:28:56 DEBUG --- status code --- 05:28:56 DEBUG 05:28:56 DEBUG "metric": { 05:28:56 DEBUG 05:28:56 DEBUG { 05:28:56 DEBUG 05:28:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:28:56 DEBUG 05:28:56 DEBUG "pod": "ds-idrepo-0" 05:28:56 DEBUG 05:28:56 DEBUG "status": "success", 05:28:56 DEBUG 05:28:56 DEBUG --- http response --- 05:28:56 DEBUG [...] 05:28:56 DEBUG }, 05:28:56 DEBUG [...] 05:28:56 DEBUG "data": { 05:28:56 DEBUG [...] 05:28:56 DEBUG 05:28:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:28:56 DEBUG "value": [ 05:28:56 DEBUG 05:28:56 DEBUG "resultType": "vector", 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG ----- output ----- 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG "result": [ 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG { 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG { 05:28:56 DEBUG 05:28:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:28:56 DEBUG "status": "success", 05:28:56 DEBUG 05:28:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:28:56 DEBUG "metric": { 05:28:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:28:56 DEBUG }, 05:28:56 DEBUG "data": { 05:28:56 DEBUG [...] 05:28:56 DEBUG }, 05:28:56 DEBUG "instance": "10.142.0.107:9100" 05:28:56 DEBUG }, 05:28:56 DEBUG "value": [ 05:28:56 DEBUG "resultType": "vector", 05:28:56 DEBUG 05:28:56 DEBUG "value": [ 05:28:56 DEBUG }, 05:28:56 DEBUG "value": [ 05:28:56 DEBUG 1713500936, 05:28:56 DEBUG "result": [ 05:28:56 DEBUG 05:28:56 DEBUG 1713500936, 05:28:56 DEBUG "value": [ 05:28:56 DEBUG 1713500936, 05:28:56 DEBUG "2006.9666666666667" 05:28:56 DEBUG { 05:28:56 DEBUG 05:28:56 DEBUG "0" 05:28:56 DEBUG 05:28:56 DEBUG "0" 05:28:56 DEBUG ] 05:28:56 DEBUG "metric": { 05:28:56 DEBUG "pod": "ds-idrepo-2" 05:28:56 DEBUG ] 05:28:56 DEBUG 05:28:56 DEBUG ] 05:28:56 DEBUG } 05:28:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:28:56 DEBUG }, 05:28:56 DEBUG } 05:28:56 DEBUG 05:28:56 DEBUG } 05:28:56 DEBUG ] 05:28:56 DEBUG }, 05:28:56 DEBUG "value": [ 05:28:56 DEBUG ] 05:28:56 DEBUG [...] 05:28:56 DEBUG ] 05:28:56 DEBUG } 05:28:56 DEBUG "value": [ 05:28:56 DEBUG 1713500936, 05:28:56 DEBUG } 05:28:56 DEBUG 05:28:56 DEBUG } 05:28:56 DEBUG } 05:28:56 DEBUG 05:28:56 DEBUG "0" 05:28:56 DEBUG } 05:28:56 DEBUG 05:28:56 DEBUG } 05:28:56 DEBUG 05:28:56 DEBUG ] 05:28:56 INFO [http_cmd]: http status code OK 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG } 05:28:56 DEBUG --- status code --- 05:28:56 DEBUG "instance": "10.142.0.43:9100" 05:28:56 DEBUG [...] 05:28:56 DEBUG ] 05:28:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:28:56 DEBUG }, 05:28:56 DEBUG 05:28:56 DEBUG } 05:28:56 DEBUG --- http response --- 05:28:56 DEBUG "value": [ 05:28:56 DEBUG 05:28:56 DEBUG } 05:28:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:28:56 DEBUG 1713500936, 05:28:56 DEBUG 05:28:56 DEBUG ----- output ----- 05:28:56 DEBUG "64105746.166666664" 05:28:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:28:56 DEBUG { 05:28:56 DEBUG ] 05:28:56 DEBUG }, 05:28:56 DEBUG "status": "success", 05:28:56 DEBUG } 05:28:56 DEBUG "value": [ 05:28:56 DEBUG "data": { 05:28:56 DEBUG ] 05:28:56 DEBUG 1713500936, 05:28:56 DEBUG "resultType": "vector", 05:28:56 DEBUG } 05:28:56 DEBUG "2168.2" 05:28:56 DEBUG "result": [ 05:28:56 DEBUG } 05:28:56 DEBUG ] 05:28:56 DEBUG { 05:28:56 DEBUG } 05:28:56 DEBUG "metric": { 05:28:56 DEBUG ] 05:28:56 DEBUG "instance": "10.142.0.107:9100" 05:28:56 DEBUG } 05:28:56 DEBUG }, 05:28:56 DEBUG } 05:28:56 DEBUG "value": [ 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG [...] 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG 05:28:56 DEBUG "instance": "10.142.0.43:9100" 05:28:56 DEBUG }, 05:28:56 DEBUG "value": [ 05:28:56 DEBUG 1713500936, 05:28:56 DEBUG "64098011.666666664" 05:28:56 DEBUG ] 05:28:56 DEBUG } 05:28:56 DEBUG ] 05:28:56 DEBUG } 05:28:56 DEBUG } 05:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500937" 05:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500937" 05:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713500937" 05:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500937" 05:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500937" 05:28:57 INFO [http_cmd]: http status code OK 05:28:57 DEBUG --- status code --- 05:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:28:57 DEBUG --- http response --- 05:28:57 INFO [http_cmd]: http status code OK 05:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:28:57 DEBUG --- status code --- 05:28:57 DEBUG ----- output ----- 05:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:28:57 DEBUG { 05:28:57 DEBUG --- http response --- 05:28:57 DEBUG "status": "success", 05:28:57 DEBUG "data": { 05:28:57 DEBUG "resultType": "vector", 05:28:57 DEBUG "result": [ 05:28:57 DEBUG { 05:28:57 DEBUG "metric": { 05:28:57 DEBUG "pod": "ds-cts-0" 05:28:57 DEBUG }, 05:28:57 DEBUG "value": [ 05:28:57 DEBUG 05:28:57 DEBUG 05:28:57 DEBUG 05:28:57 DEBUG [...] 05:28:57 DEBUG 05:28:57 DEBUG 05:28:57 DEBUG 05:28:57 DEBUG "pod": "ds-cts-2" 05:28:57 DEBUG }, 05:28:57 DEBUG "value": [ 05:28:57 DEBUG 1713500937, 05:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:28:57 DEBUG "0.0006333333333333333" 05:28:57 DEBUG ----- output ----- 05:28:57 DEBUG ] 05:28:57 DEBUG { 05:28:57 DEBUG } 05:28:57 DEBUG "status": "success", 05:28:57 DEBUG ] 05:28:57 DEBUG "data": { 05:28:57 DEBUG } 05:28:57 DEBUG "resultType": "vector", 05:28:57 DEBUG } 05:28:57 DEBUG "result": [ 05:28:57 DEBUG { 05:28:57 DEBUG "metric": { 05:28:57 DEBUG "pod": "ds-idrepo-0" 05:28:57 DEBUG }, 05:28:57 DEBUG "value": [ 05:28:57 DEBUG 05:28:57 DEBUG 05:28:57 DEBUG 05:28:57 DEBUG [...] 05:28:57 DEBUG 05:28:57 DEBUG 05:28:57 DEBUG 05:28:57 DEBUG "pod": "ds-idrepo-2" 05:28:57 DEBUG }, 05:28:57 DEBUG "value": [ 05:28:57 DEBUG 1713500937, 05:28:57 DEBUG "0" 05:28:57 DEBUG ] 05:28:57 DEBUG } 05:28:57 DEBUG ] 05:28:57 DEBUG } 05:28:57 DEBUG } 05:28:57 INFO [http_cmd]: http status code OK 05:28:57 DEBUG --- status code --- 05:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:28:57 DEBUG --- http response --- 05:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:28:57 DEBUG ----- output ----- 05:28:57 DEBUG { 05:28:57 DEBUG "status": "success", 05:28:57 DEBUG "data": { 05:28:57 DEBUG "resultType": "vector", 05:28:57 DEBUG "result": [ 05:28:57 DEBUG { 05:28:57 DEBUG "metric": { 05:28:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:28:57 DEBUG }, 05:28:57 DEBUG "value": [ 05:28:57 DEBUG 05:28:57 DEBUG 05:28:57 DEBUG 05:28:57 DEBUG [...] 05:28:57 DEBUG 05:28:57 DEBUG 05:28:57 DEBUG 05:28:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:28:57 DEBUG }, 05:28:57 DEBUG "value": [ 05:28:57 DEBUG 1713500937, 05:28:57 DEBUG "7.096460584726277" 05:28:57 DEBUG ] 05:28:57 DEBUG } 05:28:57 DEBUG ] 05:28:57 DEBUG } 05:28:57 DEBUG } 05:28:57 INFO [http_cmd]: http status code OK 05:28:57 DEBUG --- status code --- 05:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:28:57 DEBUG --- http response --- 05:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:28:57 DEBUG ----- output ----- 05:28:57 DEBUG { 05:28:57 DEBUG "status": "success", 05:28:57 DEBUG "data": { 05:28:57 DEBUG "resultType": "vector", 05:28:57 DEBUG "result": [ 05:28:57 DEBUG { 05:28:57 DEBUG "metric": { 05:28:57 DEBUG "instance": "10.142.0.107:9100" 05:28:57 DEBUG }, 05:28:57 DEBUG "value": [ 05:28:57 DEBUG 05:28:57 DEBUG 05:28:57 DEBUG 05:28:57 DEBUG [...] 05:28:57 DEBUG 05:28:57 DEBUG 05:28:57 DEBUG 05:28:57 DEBUG "instance": "10.142.0.43:9100" 05:28:57 DEBUG }, 05:28:57 DEBUG "value": [ 05:28:57 DEBUG 1713500937, 05:28:57 DEBUG "0.2399999999999996" 05:28:57 DEBUG ] 05:28:57 DEBUG } 05:28:57 DEBUG ] 05:28:57 DEBUG } 05:28:57 DEBUG } 05:28:57 INFO [http_cmd]: http status code OK 05:28:57 DEBUG --- status code --- 05:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:28:57 DEBUG --- http response --- 05:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:28:57 DEBUG ----- output ----- 05:28:57 DEBUG { 05:28:57 DEBUG "status": "success", 05:28:57 DEBUG "data": { 05:28:57 DEBUG "resultType": "vector", 05:28:57 DEBUG "result": [ 05:28:57 DEBUG { 05:28:57 DEBUG "metric": { 05:28:57 DEBUG "pod": "ds-idrepo-0" 05:28:57 DEBUG }, 05:28:57 DEBUG "value": [ 05:28:57 DEBUG 05:28:57 DEBUG 05:28:57 DEBUG 05:28:57 DEBUG [...] 05:28:57 DEBUG 05:28:57 DEBUG 05:28:57 DEBUG 05:28:57 DEBUG "pod": "ds-idrepo-2" 05:28:57 DEBUG }, 05:28:57 DEBUG "value": [ 05:28:57 DEBUG 1713500937, 05:28:57 DEBUG "0" 05:28:57 DEBUG ] 05:28:57 DEBUG } 05:28:57 DEBUG ] 05:28:57 DEBUG } 05:28:57 DEBUG } 05:28:58 INFO 05:28:58 INFO [loop_until]: kubectl --namespace=xlou top pods 05:28:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:28:58 INFO [loop_until]: OK (rc = 0) 05:28:58 DEBUG --- stdout --- 05:28:58 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 13770m 9138Mi am-6fd554f8f7-hvvkw 13550m 8904Mi am-6fd554f8f7-pmj4b 14013m 8792Mi ds-cts-0 19241m 13294Mi ds-cts-1 19232m 13056Mi ds-cts-2 19323m 13019Mi ds-idrepo-0 5868m 3267Mi ds-idrepo-1 5819m 3198Mi ds-idrepo-2 5690m 3066Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 4m 1215Mi lodemon-6b89d774d5-rf7sk 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7143m 2030Mi 05:28:58 DEBUG --- stderr --- 05:28:58 DEBUG 05:28:59 INFO 05:28:59 INFO [loop_until]: kubectl --namespace=xlou top node 05:28:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:28:59 INFO [loop_until]: OK (rc = 0) 05:28:59 DEBUG --- stdout --- 05:28:59 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 80m 0% 6245Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5836m 19% 6714Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13275m 44% 11612Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 82m 0% 3242Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 19547m 65% 15744Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 150m 0% 5180Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 13295m 44% 11925Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 11296m 37% 2737Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 15327m 51% 10989Mi 9% gke-xlou-cdm-ds-0f601751-j488 6182m 20% 6169Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 18934m 63% 17070Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6413m 21% 6885Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 18835m 63% 14801Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 7272m 45% 5741Mi 9% 05:28:59 DEBUG --- stderr --- 05:28:59 DEBUG 05:29:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713500996" 05:29:56 INFO [http_cmd]: http status code OK 05:29:56 DEBUG --- status code --- 05:29:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:29:56 DEBUG --- http response --- 05:29:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:29:56 DEBUG ----- output ----- 05:29:56 DEBUG { 05:29:56 DEBUG "status": "success", 05:29:56 DEBUG "data": { 05:29:56 DEBUG "resultType": "vector", 05:29:56 DEBUG "result": [ 05:29:56 DEBUG { 05:29:56 DEBUG "metric": { 05:29:56 DEBUG "pod": "ds-idrepo-0" 05:29:56 DEBUG }, 05:29:56 DEBUG "value": [ 05:29:56 DEBUG 05:29:56 DEBUG 05:29:56 DEBUG 05:29:56 DEBUG [...] 05:29:56 DEBUG 05:29:56 DEBUG 05:29:56 DEBUG 05:29:56 DEBUG "pod": "ds-cts-2" 05:29:56 DEBUG }, 05:29:56 DEBUG "value": [ 05:29:56 DEBUG 1713500996, 05:29:56 DEBUG "0" 05:29:56 DEBUG ] 05:29:56 DEBUG } 05:29:56 DEBUG ] 05:29:56 DEBUG } 05:29:56 DEBUG } 05:29:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500996" 05:29:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500996" 05:29:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500996" 05:29:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500996" 05:29:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713500996" 05:29:56 INFO [http_cmd]: http status code OK 05:29:56 DEBUG --- status code --- 05:29:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:29:56 DEBUG --- http response --- 05:29:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:29:56 DEBUG ----- output ----- 05:29:56 DEBUG { 05:29:56 DEBUG "status": "success", 05:29:56 DEBUG "data": { 05:29:56 DEBUG "resultType": "vector", 05:29:56 DEBUG "result": [ 05:29:56 DEBUG { 05:29:56 DEBUG "metric": { 05:29:56 DEBUG "instance": "10.142.0.107:9100" 05:29:56 DEBUG }, 05:29:56 DEBUG "value": [ 05:29:56 DEBUG 05:29:56 DEBUG 05:29:56 DEBUG 05:29:56 DEBUG [...] 05:29:56 DEBUG 05:29:56 DEBUG 05:29:56 DEBUG 05:29:56 DEBUG "instance": "10.142.0.43:9100" 05:29:56 DEBUG }, 05:29:56 DEBUG "value": [ 05:29:56 DEBUG 1713500996, 05:29:56 DEBUG "502799916032" 05:29:56 DEBUG ] 05:29:56 DEBUG } 05:29:56 DEBUG ] 05:29:56 DEBUG } 05:29:56 DEBUG } 05:29:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500996" 05:29:56 INFO [http_cmd]: http status code OK 05:29:56 INFO [http_cmd]: http status code OK 05:29:56 INFO [http_cmd]: http status code OK 05:29:56 DEBUG --- status code --- 05:29:56 DEBUG --- status code --- 05:29:56 DEBUG --- status code --- 05:29:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:29:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:29:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:29:56 DEBUG --- http response --- 05:29:56 DEBUG --- http response --- 05:29:56 DEBUG --- http response --- 05:29:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:29:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:29:56 DEBUG ----- output ----- 05:29:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:29:56 DEBUG ----- output ----- 05:29:56 DEBUG { 05:29:56 DEBUG ----- output ----- 05:29:56 DEBUG { 05:29:56 DEBUG "status": "success", 05:29:56 DEBUG { 05:29:56 DEBUG "status": "success", 05:29:56 DEBUG "data": { 05:29:56 DEBUG "status": "success", 05:29:56 DEBUG "data": { 05:29:56 DEBUG "resultType": "vector", 05:29:56 DEBUG "data": { 05:29:56 DEBUG "resultType": "vector", 05:29:56 DEBUG "result": [ 05:29:56 DEBUG "resultType": "vector", 05:29:56 DEBUG "result": [ 05:29:56 DEBUG { 05:29:56 DEBUG "result": [ 05:29:56 DEBUG { 05:29:56 DEBUG "metric": { 05:29:56 DEBUG { 05:29:56 DEBUG "metric": { 05:29:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:29:56 DEBUG "metric": { 05:29:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:29:56 DEBUG }, 05:29:56 DEBUG "pod": "ds-cts-0" 05:29:56 DEBUG }, 05:29:56 DEBUG "value": [ 05:29:56 DEBUG }, 05:29:56 DEBUG "value": [ 05:29:56 DEBUG 05:29:56 DEBUG "value": [ 05:29:56 DEBUG 05:29:56 DEBUG 05:29:56 DEBUG 05:29:56 DEBUG 05:29:56 DEBUG 05:29:56 DEBUG 05:29:56 DEBUG 05:29:56 DEBUG [...] 05:29:56 DEBUG 05:29:56 DEBUG [...] 05:29:56 DEBUG 05:29:56 DEBUG [...] 05:29:56 DEBUG 05:29:56 DEBUG 05:29:56 DEBUG 05:29:56 DEBUG 05:29:56 DEBUG 05:29:56 DEBUG 05:29:56 DEBUG 05:29:56 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:29:56 DEBUG 05:29:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:29:56 DEBUG }, 05:29:56 DEBUG "pod": "ds-cts-2" 05:29:56 ERROR [http_cmd]: ERROR 05:29:56 DEBUG }, 05:29:56 DEBUG "value": [ 05:29:56 DEBUG }, 05:29:56 ERROR --- status code --- 05:29:56 DEBUG "value": [ 05:29:56 DEBUG 1713500996, 05:29:56 DEBUG "value": [ 05:29:56 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:29:56 DEBUG 1713500996, 05:29:56 DEBUG "18424.016259620832" 05:29:56 DEBUG 1713500996, 05:29:56 ERROR --- http response --- 05:29:56 DEBUG "2177.0333333333333" 05:29:56 DEBUG ] 05:29:56 DEBUG "181.5" 05:29:56 DEBUG ] 05:29:56 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:29:56 DEBUG } 05:29:56 DEBUG ] 05:29:56 DEBUG } 05:29:56 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:29:56 DEBUG ] 05:29:56 DEBUG } 05:29:56 DEBUG ] 05:29:56 DEBUG } 05:29:56 DEBUG ] 05:29:56 DEBUG } 05:29:56 DEBUG } 05:29:56 DEBUG } 05:29:56 DEBUG } 05:29:56 DEBUG } 05:29:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500996" 05:29:56 INFO [http_cmd]: http status code OK 05:29:56 DEBUG --- status code --- 05:29:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:29:56 DEBUG --- http response --- 05:29:56 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:29:56 INFO [http_cmd]: http status code OK 05:29:56 DEBUG --- status code --- 05:29:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:29:56 DEBUG --- http response --- 05:29:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:29:56 DEBUG ----- output ----- 05:29:56 DEBUG { 05:29:56 DEBUG "status": "success", 05:29:56 DEBUG "data": { 05:29:56 DEBUG "resultType": "vector", 05:29:56 DEBUG "result": [ 05:29:56 DEBUG { 05:29:56 DEBUG "metric": { 05:29:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:29:56 DEBUG }, 05:29:56 DEBUG "value": [ 05:29:56 DEBUG 05:29:56 DEBUG 05:29:56 DEBUG 05:29:56 DEBUG [...] 05:29:56 DEBUG 05:29:56 DEBUG 05:29:56 DEBUG 05:29:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:29:56 DEBUG }, 05:29:56 DEBUG "value": [ 05:29:56 DEBUG 1713500996, 05:29:56 DEBUG "0" 05:29:56 DEBUG ] 05:29:56 DEBUG } 05:29:56 DEBUG ] 05:29:56 DEBUG } 05:29:56 DEBUG } 05:29:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500996" 05:29:56 INFO [http_cmd]: http status code OK 05:29:56 DEBUG --- status code --- 05:29:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:29:56 DEBUG --- http response --- 05:29:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:29:56 DEBUG ----- output ----- 05:29:56 DEBUG { 05:29:56 DEBUG "status": "success", 05:29:56 DEBUG "data": { 05:29:56 DEBUG "resultType": "vector", 05:29:56 DEBUG "result": [ 05:29:56 DEBUG { 05:29:56 DEBUG "metric": { 05:29:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:29:56 DEBUG }, 05:29:56 DEBUG "value": [ 05:29:56 DEBUG 05:29:56 DEBUG 05:29:56 DEBUG 05:29:56 DEBUG [...] 05:29:56 DEBUG 05:29:56 DEBUG 05:29:56 DEBUG 05:29:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:29:56 DEBUG }, 05:29:56 DEBUG "value": [ 05:29:56 DEBUG 1713500996, 05:29:56 DEBUG "3624697" 05:29:56 DEBUG ] 05:29:56 DEBUG } 05:29:56 DEBUG ] 05:29:56 DEBUG } 05:29:56 DEBUG } 05:29:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713500996" 05:29:56 INFO [http_cmd]: http status code OK 05:29:56 DEBUG --- status code --- 05:29:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:29:56 DEBUG --- http response --- 05:29:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:29:56 DEBUG ----- output ----- 05:29:56 DEBUG { 05:29:56 DEBUG "status": "success", 05:29:56 DEBUG "data": { 05:29:56 DEBUG "resultType": "vector", 05:29:56 DEBUG "result": [ 05:29:56 DEBUG { 05:29:56 DEBUG "metric": { 05:29:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:29:56 DEBUG }, 05:29:56 DEBUG "value": [ 05:29:56 DEBUG 05:29:56 DEBUG 05:29:56 DEBUG 05:29:56 DEBUG [...] 05:29:56 DEBUG 05:29:56 DEBUG 05:29:56 DEBUG 05:29:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:29:56 DEBUG }, 05:29:56 DEBUG "value": [ 05:29:56 DEBUG 1713500996, 05:29:56 DEBUG "55.172" 05:29:56 DEBUG ] 05:29:56 DEBUG } 05:29:56 DEBUG ] 05:29:56 DEBUG } 05:29:56 DEBUG } 05:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500997" 05:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500997" 05:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500997" 05:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500997" 05:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713500997" 05:29:57 INFO [http_cmd]: http status code OK 05:29:57 DEBUG --- status code --- 05:29:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:29:57 DEBUG --- http response --- 05:29:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:29:57 DEBUG ----- output ----- 05:29:57 DEBUG { 05:29:57 DEBUG "status": "success", 05:29:57 DEBUG "data": { 05:29:57 DEBUG "resultType": "vector", 05:29:57 DEBUG "result": [ 05:29:57 DEBUG { 05:29:57 DEBUG "metric": { 05:29:57 DEBUG "pod": "ds-cts-0" 05:29:57 DEBUG }, 05:29:57 DEBUG "value": [ 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG [...] 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG "pod": "ds-cts-2" 05:29:57 DEBUG }, 05:29:57 DEBUG "value": [ 05:29:57 DEBUG 1713500997, 05:29:57 DEBUG "30.333333333333332" 05:29:57 DEBUG ] 05:29:57 DEBUG } 05:29:57 DEBUG ] 05:29:57 DEBUG } 05:29:57 DEBUG } 05:29:57 INFO [http_cmd]: http status code OK 05:29:57 DEBUG --- status code --- 05:29:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:29:57 DEBUG --- http response --- 05:29:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:29:57 DEBUG ----- output ----- 05:29:57 DEBUG { 05:29:57 DEBUG "status": "success", 05:29:57 DEBUG "data": { 05:29:57 DEBUG "resultType": "vector", 05:29:57 DEBUG "result": [ 05:29:57 DEBUG { 05:29:57 DEBUG "metric": { 05:29:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:29:57 DEBUG }, 05:29:57 DEBUG "value": [ 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG [...] 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:29:57 DEBUG }, 05:29:57 DEBUG "value": [ 05:29:57 DEBUG 1713500997, 05:29:57 DEBUG "7.072401464565165" 05:29:57 DEBUG ] 05:29:57 DEBUG } 05:29:57 DEBUG ] 05:29:57 DEBUG } 05:29:57 DEBUG } 05:29:57 INFO [http_cmd]: http status code OK 05:29:57 INFO [http_cmd]: http status code OK 05:29:57 DEBUG --- status code --- 05:29:57 DEBUG --- status code --- 05:29:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:29:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:29:57 DEBUG --- http response --- 05:29:57 DEBUG --- http response --- 05:29:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:29:57 DEBUG ----- output ----- 05:29:57 DEBUG { 05:29:57 DEBUG "status": "success", 05:29:57 DEBUG "data": { 05:29:57 DEBUG "resultType": "vector", 05:29:57 DEBUG "result": [ 05:29:57 DEBUG { 05:29:57 DEBUG "metric": { 05:29:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:29:57 DEBUG }, 05:29:57 DEBUG "value": [ 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG [...] 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:29:57 DEBUG }, 05:29:57 DEBUG "value": [ 05:29:57 DEBUG 1713500997, 05:29:57 DEBUG "2177.7" 05:29:57 DEBUG ] 05:29:57 DEBUG } 05:29:57 DEBUG ] 05:29:57 DEBUG } 05:29:57 DEBUG } 05:29:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:29:57 DEBUG ----- output ----- 05:29:57 DEBUG { 05:29:57 DEBUG "status": "success", 05:29:57 DEBUG "data": { 05:29:57 DEBUG "resultType": "vector", 05:29:57 DEBUG "result": [ 05:29:57 DEBUG { 05:29:57 DEBUG "metric": { 05:29:57 DEBUG "instance": "10.142.0.107:9100" 05:29:57 DEBUG }, 05:29:57 DEBUG "value": [ 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG [...] 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG "instance": "10.142.0.43:9100" 05:29:57 DEBUG }, 05:29:57 DEBUG "value": [ 05:29:57 DEBUG 1713500997, 05:29:57 DEBUG "0.6891000000000009" 05:29:57 DEBUG ] 05:29:57 DEBUG } 05:29:57 DEBUG ] 05:29:57 DEBUG } 05:29:57 DEBUG } 05:29:57 INFO [http_cmd]: http status code OK 05:29:57 DEBUG --- status code --- 05:29:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:29:57 DEBUG --- http response --- 05:29:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:29:57 DEBUG ----- output ----- 05:29:57 DEBUG { 05:29:57 DEBUG "status": "success", 05:29:57 DEBUG "data": { 05:29:57 DEBUG "resultType": "vector", 05:29:57 DEBUG "result": [ 05:29:57 DEBUG { 05:29:57 DEBUG "metric": { 05:29:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:29:57 DEBUG }, 05:29:57 DEBUG "value": [ 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG [...] 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:29:57 DEBUG }, 05:29:57 DEBUG "value": [ 05:29:57 DEBUG 1713500997, 05:29:57 DEBUG "2228420608" 05:29:57 DEBUG ] 05:29:57 DEBUG } 05:29:57 DEBUG ] 05:29:57 DEBUG } 05:29:57 DEBUG } 05:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500997" 05:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500997" 05:29:57 INFO [http_cmd]: http status code OK 05:29:57 DEBUG --- status code --- 05:29:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:29:57 DEBUG --- http response --- 05:29:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:29:57 DEBUG ----- output ----- 05:29:57 DEBUG { 05:29:57 DEBUG "status": "success", 05:29:57 DEBUG "data": { 05:29:57 DEBUG "resultType": "vector", 05:29:57 DEBUG "result": [ 05:29:57 DEBUG { 05:29:57 DEBUG "metric": { 05:29:57 DEBUG "pod": "ds-idrepo-0" 05:29:57 DEBUG }, 05:29:57 DEBUG "value": [ 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG [...] 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG "pod": "ds-idrepo-2" 05:29:57 DEBUG }, 05:29:57 DEBUG "value": [ 05:29:57 DEBUG 1713500997, 05:29:57 DEBUG "0" 05:29:57 DEBUG ] 05:29:57 DEBUG } 05:29:57 DEBUG ] 05:29:57 DEBUG } 05:29:57 DEBUG } 05:29:57 INFO [http_cmd]: http status code OK 05:29:57 DEBUG --- status code --- 05:29:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:29:57 DEBUG --- http response --- 05:29:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:29:57 DEBUG ----- output ----- 05:29:57 DEBUG { 05:29:57 DEBUG "status": "success", 05:29:57 DEBUG "data": { 05:29:57 DEBUG "resultType": "vector", 05:29:57 DEBUG "result": [ 05:29:57 DEBUG { 05:29:57 DEBUG "metric": { 05:29:57 DEBUG "pod": "ds-cts-0" 05:29:57 DEBUG }, 05:29:57 DEBUG "value": [ 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG [...] 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG "pod": "ds-cts-2" 05:29:57 DEBUG }, 05:29:57 DEBUG "value": [ 05:29:57 DEBUG 1713500997, 05:29:57 DEBUG "0.0012333333333333332" 05:29:57 DEBUG ] 05:29:57 DEBUG } 05:29:57 DEBUG ] 05:29:57 DEBUG } 05:29:57 DEBUG } 05:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500997" 05:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500997" 05:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500997" 05:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500997" 05:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500997" 05:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500997" 05:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500997" 05:29:57 INFO [http_cmd]: http status code OK 05:29:57 DEBUG --- status code --- 05:29:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:29:57 DEBUG --- http response --- 05:29:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:29:57 DEBUG ----- output ----- 05:29:57 DEBUG { 05:29:57 DEBUG "status": "success", 05:29:57 DEBUG "data": { 05:29:57 DEBUG "resultType": "vector", 05:29:57 DEBUG "result": [ 05:29:57 DEBUG { 05:29:57 DEBUG "metric": { 05:29:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:29:57 DEBUG }, 05:29:57 DEBUG "value": [ 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG [...] 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:29:57 DEBUG }, 05:29:57 DEBUG "value": [ 05:29:57 DEBUG 1713500997, 05:29:57 DEBUG "2007.4333333333334" 05:29:57 DEBUG ] 05:29:57 DEBUG } 05:29:57 DEBUG ] 05:29:57 DEBUG } 05:29:57 DEBUG } 05:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713500997" 05:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713500997" 05:29:57 INFO [http_cmd]: http status code OK 05:29:57 INFO [http_cmd]: http status code OK 05:29:57 DEBUG --- status code --- 05:29:57 DEBUG --- status code --- 05:29:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:29:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:29:57 DEBUG --- http response --- 05:29:57 DEBUG --- http response --- 05:29:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:29:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:29:57 INFO [http_cmd]: http status code OK 05:29:57 DEBUG ----- output ----- 05:29:57 INFO [http_cmd]: http status code OK 05:29:57 INFO [http_cmd]: http status code OK 05:29:57 DEBUG ----- output ----- 05:29:57 INFO [http_cmd]: http status code OK 05:29:57 DEBUG --- status code --- 05:29:57 DEBUG { 05:29:57 DEBUG --- status code --- 05:29:57 DEBUG --- status code --- 05:29:57 DEBUG { 05:29:57 DEBUG --- status code --- 05:29:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:29:57 DEBUG "status": "success", 05:29:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:29:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:29:57 DEBUG "status": "success", 05:29:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:29:57 DEBUG --- http response --- 05:29:57 DEBUG "data": { 05:29:57 DEBUG --- http response --- 05:29:57 DEBUG --- http response --- 05:29:57 DEBUG "data": { 05:29:57 DEBUG --- http response --- 05:29:57 DEBUG "resultType": "vector", 05:29:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:29:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:29:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:29:57 DEBUG "resultType": "vector", 05:29:57 DEBUG "result": [ 05:29:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:29:57 DEBUG ----- output ----- 05:29:57 DEBUG ----- output ----- 05:29:57 DEBUG ----- output ----- 05:29:57 DEBUG "result": [ 05:29:57 DEBUG { 05:29:57 DEBUG ----- output ----- 05:29:57 DEBUG { 05:29:57 DEBUG { 05:29:57 DEBUG { 05:29:57 DEBUG { 05:29:57 DEBUG "metric": { 05:29:57 DEBUG { 05:29:57 DEBUG "status": "success", 05:29:57 DEBUG "status": "success", 05:29:57 DEBUG "status": "success", 05:29:57 DEBUG "metric": { 05:29:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:29:57 DEBUG "status": "success", 05:29:57 DEBUG "data": { 05:29:57 DEBUG "data": { 05:29:57 DEBUG "data": { 05:29:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:29:57 DEBUG }, 05:29:57 DEBUG "data": { 05:29:57 DEBUG "resultType": "vector", 05:29:57 DEBUG "resultType": "vector", 05:29:57 DEBUG "resultType": "vector", 05:29:57 DEBUG }, 05:29:57 DEBUG "value": [ 05:29:57 DEBUG "resultType": "vector", 05:29:57 DEBUG "result": [ 05:29:57 DEBUG "result": [ 05:29:57 INFO [http_cmd]: http status code OK 05:29:57 DEBUG "result": [ 05:29:57 DEBUG "value": [ 05:29:57 DEBUG 05:29:57 DEBUG "result": [ 05:29:57 DEBUG { 05:29:57 DEBUG { 05:29:57 DEBUG --- status code --- 05:29:57 INFO [http_cmd]: http status code OK 05:29:57 DEBUG { 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG { 05:29:57 DEBUG "metric": { 05:29:57 DEBUG "metric": { 05:29:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:29:57 DEBUG --- status code --- 05:29:57 DEBUG "metric": { 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG "metric": { 05:29:57 DEBUG "instance": "10.142.0.107:9100" 05:29:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:29:57 DEBUG --- http response --- 05:29:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713500997" 05:29:57 DEBUG "instance": "10.142.0.107:9100" 05:29:57 DEBUG 05:29:57 DEBUG [...] 05:29:57 DEBUG "pod": "ds-idrepo-0" 05:29:57 DEBUG }, 05:29:57 DEBUG }, 05:29:57 DEBUG --- http response --- 05:29:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:29:57 DEBUG }, 05:29:57 DEBUG [...] 05:29:57 DEBUG 05:29:57 DEBUG }, 05:29:57 DEBUG "value": [ 05:29:57 DEBUG "value": [ 05:29:57 DEBUG ----- output ----- 05:29:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:29:57 DEBUG "value": [ 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG "value": [ 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG { 05:29:57 DEBUG ----- output ----- 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG "status": "success", 05:29:57 DEBUG { 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG "data": { 05:29:57 DEBUG "status": "success", 05:29:57 DEBUG 05:29:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:29:57 DEBUG }, 05:29:57 DEBUG 05:29:57 DEBUG [...] 05:29:57 DEBUG [...] 05:29:57 DEBUG "resultType": "vector", 05:29:57 DEBUG "data": { 05:29:57 DEBUG [...] 05:29:57 DEBUG }, 05:29:57 DEBUG "value": [ 05:29:57 DEBUG [...] 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG "result": [ 05:29:57 DEBUG "resultType": "vector", 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG 1713500997, 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG { 05:29:57 DEBUG "result": [ 05:29:57 DEBUG "value": [ 05:29:57 DEBUG 05:29:57 DEBUG "0" 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG "metric": { 05:29:57 DEBUG { 05:29:57 DEBUG 1713500997, 05:29:57 DEBUG "instance": "10.142.0.43:9100" 05:29:57 DEBUG ] 05:29:57 DEBUG 05:29:57 DEBUG "instance": "10.142.0.43:9100" 05:29:57 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:29:57 DEBUG "pod": "ds-idrepo-0" 05:29:57 DEBUG "metric": { 05:29:57 DEBUG "2177.0666666666666" 05:29:57 DEBUG }, 05:29:57 DEBUG } 05:29:57 DEBUG "pod": "ds-idrepo-2" 05:29:57 DEBUG }, 05:29:57 DEBUG }, 05:29:57 DEBUG }, 05:29:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:29:57 DEBUG ] 05:29:57 DEBUG "value": [ 05:29:57 DEBUG ] 05:29:57 DEBUG }, 05:29:57 DEBUG "value": [ 05:29:57 DEBUG "value": [ 05:29:57 DEBUG "value": [ 05:29:57 DEBUG }, 05:29:57 DEBUG } 05:29:57 DEBUG 1713500997, 05:29:57 DEBUG } 05:29:57 DEBUG "value": [ 05:29:57 DEBUG 1713500997, 05:29:57 DEBUG 1713500997, 05:29:57 DEBUG 05:29:57 DEBUG "value": [ 05:29:57 DEBUG ] 05:29:57 DEBUG "64329904.63333333" 05:29:57 DEBUG } 05:29:57 DEBUG 1713500997, 05:29:57 DEBUG "64336180.06666666" 05:29:57 DEBUG "0" 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG } 05:29:57 DEBUG ] 05:29:57 DEBUG "0" 05:29:57 DEBUG ] 05:29:57 DEBUG ] 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG } 05:29:57 DEBUG } 05:29:57 DEBUG ] 05:29:57 DEBUG } 05:29:57 DEBUG } 05:29:57 DEBUG [...] 05:29:57 DEBUG 05:29:57 DEBUG ] 05:29:57 DEBUG } 05:29:57 DEBUG ] 05:29:57 DEBUG ] 05:29:57 DEBUG 05:29:57 DEBUG [...] 05:29:57 DEBUG } 05:29:57 DEBUG ] 05:29:57 DEBUG } 05:29:57 DEBUG } 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG } 05:29:57 DEBUG } 05:29:57 DEBUG } 05:29:57 DEBUG } 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG } 05:29:57 DEBUG "pod": "ds-idrepo-2" 05:29:57 DEBUG 05:29:57 DEBUG }, 05:29:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:29:57 DEBUG "value": [ 05:29:57 DEBUG }, 05:29:57 DEBUG 1713500997, 05:29:57 DEBUG "value": [ 05:29:57 DEBUG "0" 05:29:57 DEBUG 1713500997, 05:29:57 DEBUG ] 05:29:57 DEBUG "7.092319117986943" 05:29:57 DEBUG } 05:29:57 DEBUG ] 05:29:57 DEBUG ] 05:29:57 DEBUG } 05:29:57 DEBUG } 05:29:57 DEBUG ] 05:29:57 DEBUG } 05:29:57 DEBUG } 05:29:57 DEBUG } 05:29:57 INFO [http_cmd]: http status code OK 05:29:57 DEBUG --- status code --- 05:29:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:29:57 DEBUG --- http response --- 05:29:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:29:57 DEBUG ----- output ----- 05:29:57 DEBUG { 05:29:57 DEBUG "status": "success", 05:29:57 DEBUG "data": { 05:29:57 DEBUG "resultType": "vector", 05:29:57 DEBUG "result": [ 05:29:57 DEBUG { 05:29:57 DEBUG "metric": { 05:29:57 DEBUG "instance": "10.142.0.107:9100" 05:29:57 DEBUG }, 05:29:57 DEBUG "value": [ 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG [...] 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG 05:29:57 DEBUG "instance": "10.142.0.43:9100" 05:29:57 DEBUG }, 05:29:57 DEBUG "value": [ 05:29:57 DEBUG 1713500997, 05:29:57 DEBUG "0.40399999999999714" 05:29:57 DEBUG ] 05:29:57 DEBUG } 05:29:57 DEBUG ] 05:29:57 DEBUG } 05:29:57 DEBUG } 05:29:58 INFO 05:29:58 INFO [loop_until]: kubectl --namespace=xlou top pods 05:29:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:29:58 INFO [loop_until]: OK (rc = 0) 05:29:58 DEBUG --- stdout --- 05:29:58 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 13676m 9139Mi am-6fd554f8f7-hvvkw 13051m 8906Mi am-6fd554f8f7-pmj4b 14424m 8792Mi ds-cts-0 19207m 13055Mi ds-cts-1 18278m 12925Mi ds-cts-2 19054m 13306Mi ds-idrepo-0 5752m 3267Mi ds-idrepo-1 5728m 3199Mi ds-idrepo-2 5673m 3066Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1172Mi idm-b85f9d978-wnhxb 3m 1215Mi lodemon-6b89d774d5-rf7sk 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7160m 2032Mi 05:29:58 DEBUG --- stderr --- 05:29:58 DEBUG 05:29:59 INFO 05:29:59 INFO [loop_until]: kubectl --namespace=xlou top node 05:29:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:29:59 INFO [loop_until]: OK (rc = 0) 05:29:59 DEBUG --- stdout --- 05:29:59 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 70m 0% 6248Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5846m 19% 6708Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13270m 44% 11612Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 82m 0% 3250Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 19277m 64% 15495Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 162m 0% 5169Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 13718m 45% 11922Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 11297m 37% 2742Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 15253m 51% 11002Mi 9% gke-xlou-cdm-ds-0f601751-j488 6176m 20% 6180Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 18139m 60% 17286Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6372m 21% 6886Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 19532m 65% 15242Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 7280m 45% 5743Mi 9% 05:29:59 DEBUG --- stderr --- 05:29:59 DEBUG 05:30:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713501056" 05:30:56 INFO [http_cmd]: http status code OK 05:30:56 DEBUG --- status code --- 05:30:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:30:56 DEBUG --- http response --- 05:30:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:30:56 DEBUG ----- output ----- 05:30:56 DEBUG { 05:30:56 DEBUG "status": "success", 05:30:56 DEBUG "data": { 05:30:56 DEBUG "resultType": "vector", 05:30:56 DEBUG "result": [ 05:30:56 DEBUG { 05:30:56 DEBUG "metric": { 05:30:56 DEBUG "pod": "ds-idrepo-0" 05:30:56 DEBUG }, 05:30:56 DEBUG "value": [ 05:30:56 DEBUG 05:30:56 DEBUG 05:30:56 DEBUG 05:30:56 DEBUG [...] 05:30:56 DEBUG 05:30:56 DEBUG 05:30:56 DEBUG 05:30:56 DEBUG "pod": "ds-cts-2" 05:30:56 DEBUG }, 05:30:56 DEBUG "value": [ 05:30:56 DEBUG 1713501056, 05:30:56 DEBUG "0" 05:30:56 DEBUG ] 05:30:56 DEBUG } 05:30:56 DEBUG ] 05:30:56 DEBUG } 05:30:56 DEBUG } 05:30:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501056" 05:30:56 INFO [http_cmd]: http status code OK 05:30:56 DEBUG --- status code --- 05:30:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:30:56 DEBUG --- http response --- 05:30:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 05:30:56 DEBUG ----- output ----- 05:30:56 DEBUG { 05:30:56 DEBUG "status": "success", 05:30:56 DEBUG "data": { 05:30:56 DEBUG "resultType": "vector", 05:30:56 DEBUG "result": [ 05:30:56 DEBUG { 05:30:56 DEBUG "metric": { 05:30:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:30:56 DEBUG }, 05:30:56 DEBUG "value": [ 05:30:56 DEBUG 05:30:56 DEBUG 05:30:56 DEBUG 05:30:56 DEBUG [...] 05:30:56 DEBUG 05:30:56 DEBUG 05:30:56 DEBUG 05:30:56 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:30:56 DEBUG }, 05:30:56 DEBUG "value": [ 05:30:56 DEBUG 1713501056, 05:30:56 DEBUG "10282.279108175064" 05:30:56 DEBUG ] 05:30:56 DEBUG } 05:30:56 DEBUG ] 05:30:56 DEBUG } 05:30:56 DEBUG } 05:30:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501056" 05:30:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713501056" 05:30:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501056" 05:30:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501056" 05:30:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501056" 05:30:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501056" 05:30:56 INFO [http_cmd]: http status code OK 05:30:56 ERROR [http_cmd]: ERROR 05:30:56 INFO [http_cmd]: http status code OK 05:30:56 DEBUG --- status code --- 05:30:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501056" 05:30:56 ERROR --- status code --- 05:30:56 DEBUG --- status code --- 05:30:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:30:56 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:30:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:30:56 DEBUG --- http response --- 05:30:56 ERROR --- http response --- 05:30:56 DEBUG --- http response --- 05:30:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:30:56 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:30:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:30:56 DEBUG ----- output ----- 05:30:56 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:30:56 DEBUG ----- output ----- 05:30:56 INFO [http_cmd]: http status code OK 05:30:56 DEBUG { 05:30:56 DEBUG { 05:30:56 DEBUG --- status code --- 05:30:56 DEBUG "status": "success", 05:30:56 DEBUG "status": "success", 05:30:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:30:56 DEBUG "data": { 05:30:56 DEBUG "data": { 05:30:56 DEBUG --- http response --- 05:30:56 DEBUG "resultType": "vector", 05:30:56 DEBUG "resultType": "vector", 05:30:56 DEBUG "result": [ 05:30:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:30:56 DEBUG "result": [ 05:30:56 DEBUG { 05:30:56 DEBUG ----- output ----- 05:30:56 DEBUG { 05:30:56 DEBUG "metric": { 05:30:56 DEBUG { 05:30:56 INFO [http_cmd]: http status code OK 05:30:56 DEBUG "metric": { 05:30:56 DEBUG "instance": "10.142.0.107:9100" 05:30:56 DEBUG "status": "success", 05:30:56 DEBUG --- status code --- 05:30:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:30:56 DEBUG }, 05:30:56 DEBUG "data": { 05:30:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:30:56 DEBUG }, 05:30:56 DEBUG "value": [ 05:30:56 DEBUG "resultType": "vector", 05:30:56 DEBUG --- http response --- 05:30:56 DEBUG "value": [ 05:30:56 DEBUG 05:30:56 DEBUG "result": [ 05:30:56 DEBUG 05:30:56 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:30:56 DEBUG 05:30:56 DEBUG { 05:30:56 DEBUG 05:30:56 DEBUG 05:30:56 DEBUG "metric": { 05:30:56 DEBUG 05:30:56 DEBUG [...] 05:30:56 DEBUG "pod": "ds-cts-0" 05:30:56 DEBUG [...] 05:30:56 DEBUG 05:30:56 DEBUG }, 05:30:56 DEBUG 05:30:56 DEBUG 05:30:56 DEBUG "value": [ 05:30:56 DEBUG 05:30:56 DEBUG 05:30:56 DEBUG 05:30:56 DEBUG 05:30:56 DEBUG "instance": "10.142.0.43:9100" 05:30:56 DEBUG 05:30:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:30:56 DEBUG }, 05:30:56 DEBUG 05:30:56 INFO [http_cmd]: http status code OK 05:30:56 DEBUG }, 05:30:56 DEBUG "value": [ 05:30:56 DEBUG [...] 05:30:56 DEBUG --- status code --- 05:30:56 DEBUG "value": [ 05:30:56 DEBUG 1713501056, 05:30:56 DEBUG 05:30:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:30:56 DEBUG 1713501056, 05:30:56 DEBUG "506993028096" 05:30:56 DEBUG 05:30:56 DEBUG --- http response --- 05:30:56 DEBUG "2162.5333333333333" 05:30:56 DEBUG ] 05:30:56 DEBUG 05:30:56 DEBUG ] 05:30:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:30:56 DEBUG } 05:30:56 DEBUG "pod": "ds-cts-2" 05:30:56 DEBUG } 05:30:56 DEBUG ----- output ----- 05:30:56 DEBUG ] 05:30:56 DEBUG }, 05:30:56 DEBUG ] 05:30:56 DEBUG { 05:30:56 DEBUG } 05:30:56 DEBUG "value": [ 05:30:56 DEBUG } 05:30:56 DEBUG "status": "success", 05:30:56 DEBUG } 05:30:56 DEBUG 1713501056, 05:30:56 DEBUG } 05:30:56 DEBUG "data": { 05:30:56 DEBUG "223.53333333333333" 05:30:56 DEBUG "resultType": "vector", 05:30:56 DEBUG ] 05:30:56 DEBUG "result": [ 05:30:56 DEBUG } 05:30:56 DEBUG { 05:30:56 DEBUG ] 05:30:56 DEBUG "metric": { 05:30:56 DEBUG } 05:30:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:30:56 DEBUG } 05:30:56 DEBUG }, 05:30:56 DEBUG "value": [ 05:30:56 DEBUG 05:30:56 DEBUG 05:30:56 DEBUG 05:30:56 DEBUG [...] 05:30:56 DEBUG 05:30:56 DEBUG 05:30:56 DEBUG 05:30:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:30:56 DEBUG }, 05:30:56 DEBUG "value": [ 05:30:56 DEBUG 1713501056, 05:30:56 DEBUG "3634199" 05:30:56 DEBUG ] 05:30:56 DEBUG } 05:30:56 DEBUG ] 05:30:56 DEBUG } 05:30:56 DEBUG } 05:30:56 INFO [http_cmd]: http status code OK 05:30:56 DEBUG --- status code --- 05:30:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:30:56 DEBUG --- http response --- 05:30:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:30:56 DEBUG ----- output ----- 05:30:56 DEBUG { 05:30:56 DEBUG "status": "success", 05:30:56 DEBUG "data": { 05:30:56 DEBUG "resultType": "vector", 05:30:56 DEBUG "result": [ 05:30:56 DEBUG { 05:30:56 DEBUG "metric": { 05:30:56 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:30:56 DEBUG }, 05:30:56 DEBUG "value": [ 05:30:56 DEBUG 05:30:56 DEBUG 05:30:56 DEBUG 05:30:56 DEBUG [...] 05:30:56 DEBUG 05:30:56 DEBUG 05:30:56 DEBUG 05:30:56 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:30:56 DEBUG }, 05:30:56 DEBUG "value": [ 05:30:56 DEBUG 1713501056, 05:30:56 DEBUG "0" 05:30:56 DEBUG ] 05:30:56 DEBUG } 05:30:56 DEBUG ] 05:30:56 DEBUG } 05:30:56 DEBUG } 05:30:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713501057" 05:30:57 INFO [http_cmd]: http status code OK 05:30:57 DEBUG --- status code --- 05:30:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:30:57 DEBUG --- http response --- 05:30:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:30:57 DEBUG ----- output ----- 05:30:57 DEBUG { 05:30:57 DEBUG "status": "success", 05:30:57 DEBUG "data": { 05:30:57 DEBUG "resultType": "vector", 05:30:57 DEBUG "result": [ 05:30:57 DEBUG { 05:30:57 DEBUG "metric": { 05:30:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:30:57 DEBUG }, 05:30:57 DEBUG "value": [ 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG [...] 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:30:57 DEBUG }, 05:30:57 DEBUG "value": [ 05:30:57 DEBUG 1713501057, 05:30:57 DEBUG "55.172" 05:30:57 DEBUG ] 05:30:57 DEBUG } 05:30:57 DEBUG ] 05:30:57 DEBUG } 05:30:57 DEBUG } 05:30:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501057" 05:30:57 INFO [http_cmd]: http status code OK 05:30:57 DEBUG --- status code --- 05:30:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:30:57 DEBUG --- http response --- 05:30:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:30:57 DEBUG ----- output ----- 05:30:57 DEBUG { 05:30:57 DEBUG "status": "success", 05:30:57 DEBUG "data": { 05:30:57 DEBUG "resultType": "vector", 05:30:57 DEBUG "result": [ 05:30:57 DEBUG { 05:30:57 DEBUG "metric": { 05:30:57 DEBUG "pod": "ds-cts-0" 05:30:57 DEBUG }, 05:30:57 DEBUG "value": [ 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG [...] 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG "pod": "ds-cts-2" 05:30:57 DEBUG }, 05:30:57 DEBUG "value": [ 05:30:57 DEBUG 1713501057, 05:30:57 DEBUG "26.5" 05:30:57 DEBUG ] 05:30:57 DEBUG } 05:30:57 DEBUG ] 05:30:57 DEBUG } 05:30:57 DEBUG } 05:30:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501057" 05:30:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501057" 05:30:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501057" 05:30:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501057" 05:30:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713501057" 05:30:57 INFO [http_cmd]: http status code OK 05:30:57 DEBUG --- status code --- 05:30:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:30:57 DEBUG --- http response --- 05:30:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:30:57 DEBUG ----- output ----- 05:30:57 DEBUG { 05:30:57 DEBUG "status": "success", 05:30:57 DEBUG "data": { 05:30:57 DEBUG "resultType": "vector", 05:30:57 DEBUG "result": [ 05:30:57 DEBUG { 05:30:57 DEBUG "metric": { 05:30:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:30:57 DEBUG }, 05:30:57 DEBUG "value": [ 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG [...] 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:30:57 DEBUG }, 05:30:57 DEBUG "value": [ 05:30:57 DEBUG 1713501057, 05:30:57 DEBUG "2163.7666666666664" 05:30:57 DEBUG ] 05:30:57 DEBUG } 05:30:57 DEBUG ] 05:30:57 DEBUG } 05:30:57 DEBUG } 05:30:57 INFO [http_cmd]: http status code OK 05:30:57 INFO [http_cmd]: http status code OK 05:30:57 INFO [http_cmd]: http status code OK 05:30:57 DEBUG --- status code --- 05:30:57 DEBUG --- status code --- 05:30:57 DEBUG --- status code --- 05:30:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:30:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:30:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:30:57 DEBUG --- http response --- 05:30:57 DEBUG --- http response --- 05:30:57 DEBUG --- http response --- 05:30:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:30:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:30:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 05:30:57 DEBUG ----- output ----- 05:30:57 DEBUG ----- output ----- 05:30:57 DEBUG ----- output ----- 05:30:57 DEBUG { 05:30:57 DEBUG { 05:30:57 DEBUG { 05:30:57 DEBUG "status": "success", 05:30:57 DEBUG "status": "success", 05:30:57 DEBUG "status": "success", 05:30:57 DEBUG "data": { 05:30:57 DEBUG "data": { 05:30:57 DEBUG "data": { 05:30:57 DEBUG "resultType": "vector", 05:30:57 DEBUG "resultType": "vector", 05:30:57 DEBUG "resultType": "vector", 05:30:57 DEBUG "result": [ 05:30:57 DEBUG "result": [ 05:30:57 DEBUG "result": [ 05:30:57 DEBUG { 05:30:57 DEBUG { 05:30:57 DEBUG { 05:30:57 DEBUG "metric": { 05:30:57 DEBUG "metric": { 05:30:57 DEBUG "metric": { 05:30:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:30:57 DEBUG "instance": "10.142.0.107:9100" 05:30:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:30:57 DEBUG }, 05:30:57 DEBUG }, 05:30:57 DEBUG }, 05:30:57 DEBUG "value": [ 05:30:57 DEBUG "value": [ 05:30:57 DEBUG "value": [ 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG [...] 05:30:57 DEBUG [...] 05:30:57 DEBUG [...] 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:30:57 DEBUG "instance": "10.142.0.43:9100" 05:30:57 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:30:57 DEBUG }, 05:30:57 DEBUG }, 05:30:57 DEBUG }, 05:30:57 DEBUG "value": [ 05:30:57 DEBUG "value": [ 05:30:57 DEBUG "value": [ 05:30:57 DEBUG 1713501057, 05:30:57 DEBUG 1713501057, 05:30:57 DEBUG 1713501057, 05:30:57 DEBUG "2005.7666666666667" 05:30:57 DEBUG "0.42063333333332764" 05:30:57 DEBUG "7.056363079009362" 05:30:57 DEBUG ] 05:30:57 DEBUG ] 05:30:57 DEBUG ] 05:30:57 INFO [http_cmd]: http status code OK 05:30:57 DEBUG } 05:30:57 DEBUG } 05:30:57 DEBUG } 05:30:57 DEBUG --- status code --- 05:30:57 DEBUG ] 05:30:57 DEBUG ] 05:30:57 DEBUG ] 05:30:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:30:57 DEBUG } 05:30:57 DEBUG } 05:30:57 DEBUG } 05:30:57 DEBUG --- http response --- 05:30:57 DEBUG } 05:30:57 DEBUG } 05:30:57 DEBUG } 05:30:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501057" 05:30:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:30:57 DEBUG ----- output ----- 05:30:57 DEBUG { 05:30:57 DEBUG "status": "success", 05:30:57 DEBUG "data": { 05:30:57 DEBUG "resultType": "vector", 05:30:57 DEBUG "result": [ 05:30:57 DEBUG { 05:30:57 DEBUG "metric": { 05:30:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:30:57 DEBUG }, 05:30:57 DEBUG "value": [ 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG [...] 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:30:57 DEBUG }, 05:30:57 DEBUG "value": [ 05:30:57 DEBUG 1713501057, 05:30:57 DEBUG "2229641216" 05:30:57 DEBUG ] 05:30:57 DEBUG } 05:30:57 DEBUG ] 05:30:57 DEBUG } 05:30:57 DEBUG } 05:30:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501057" 05:30:57 INFO [http_cmd]: http status code OK 05:30:57 DEBUG --- status code --- 05:30:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:30:57 DEBUG --- http response --- 05:30:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:30:57 DEBUG ----- output ----- 05:30:57 DEBUG { 05:30:57 DEBUG "status": "success", 05:30:57 DEBUG "data": { 05:30:57 DEBUG "resultType": "vector", 05:30:57 DEBUG "result": [ 05:30:57 DEBUG { 05:30:57 DEBUG "metric": { 05:30:57 DEBUG "pod": "ds-idrepo-0" 05:30:57 DEBUG }, 05:30:57 DEBUG "value": [ 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG [...] 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG "pod": "ds-idrepo-2" 05:30:57 DEBUG }, 05:30:57 DEBUG "value": [ 05:30:57 DEBUG 1713501057, 05:30:57 DEBUG "0" 05:30:57 DEBUG ] 05:30:57 DEBUG } 05:30:57 DEBUG ] 05:30:57 DEBUG } 05:30:57 DEBUG } 05:30:57 INFO [http_cmd]: http status code OK 05:30:57 DEBUG --- status code --- 05:30:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:30:57 DEBUG --- http response --- 05:30:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:30:57 DEBUG ----- output ----- 05:30:57 DEBUG { 05:30:57 DEBUG "status": "success", 05:30:57 DEBUG "data": { 05:30:57 DEBUG "resultType": "vector", 05:30:57 DEBUG "result": [ 05:30:57 DEBUG { 05:30:57 DEBUG "metric": { 05:30:57 DEBUG "pod": "ds-cts-0" 05:30:57 DEBUG }, 05:30:57 DEBUG "value": [ 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG [...] 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG "pod": "ds-cts-2" 05:30:57 DEBUG }, 05:30:57 DEBUG "value": [ 05:30:57 DEBUG 1713501057, 05:30:57 DEBUG "0.0011510133333333334" 05:30:57 DEBUG ] 05:30:57 DEBUG } 05:30:57 DEBUG ] 05:30:57 DEBUG } 05:30:57 DEBUG } 05:30:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501057" 05:30:57 INFO [http_cmd]: http status code OK 05:30:57 DEBUG --- status code --- 05:30:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:30:57 DEBUG --- http response --- 05:30:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:30:57 DEBUG ----- output ----- 05:30:57 DEBUG { 05:30:57 DEBUG "status": "success", 05:30:57 DEBUG "data": { 05:30:57 DEBUG "resultType": "vector", 05:30:57 DEBUG "result": [ 05:30:57 DEBUG { 05:30:57 DEBUG "metric": { 05:30:57 DEBUG "pod": "ds-idrepo-0" 05:30:57 DEBUG }, 05:30:57 DEBUG "value": [ 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG [...] 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG "pod": "ds-idrepo-2" 05:30:57 DEBUG }, 05:30:57 DEBUG "value": [ 05:30:57 DEBUG 1713501057, 05:30:57 DEBUG "0" 05:30:57 DEBUG ] 05:30:57 DEBUG } 05:30:57 DEBUG ] 05:30:57 DEBUG } 05:30:57 DEBUG } 05:30:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501057" 05:30:57 INFO [http_cmd]: http status code OK 05:30:57 DEBUG --- status code --- 05:30:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:30:57 DEBUG --- http response --- 05:30:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:30:57 DEBUG ----- output ----- 05:30:57 DEBUG { 05:30:57 DEBUG "status": "success", 05:30:57 DEBUG "data": { 05:30:57 DEBUG "resultType": "vector", 05:30:57 DEBUG "result": [ 05:30:57 DEBUG { 05:30:57 DEBUG "metric": { 05:30:57 DEBUG "instance": "10.142.0.107:9100" 05:30:57 DEBUG }, 05:30:57 DEBUG "value": [ 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG [...] 05:30:57 DEBUG 05:30:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501057" 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG "instance": "10.142.0.43:9100" 05:30:57 DEBUG }, 05:30:57 DEBUG "value": [ 05:30:57 DEBUG 1713501057, 05:30:57 DEBUG "64073545.96666667" 05:30:57 DEBUG ] 05:30:57 DEBUG } 05:30:57 DEBUG ] 05:30:57 DEBUG } 05:30:57 DEBUG } 05:30:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501057" 05:30:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501057" 05:30:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501057" 05:30:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501057" 05:30:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713501057" 05:30:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501057" 05:30:57 INFO [http_cmd]: http status code OK 05:30:57 INFO [http_cmd]: http status code OK 05:30:57 DEBUG --- status code --- 05:30:57 DEBUG --- status code --- 05:30:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:30:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:30:57 DEBUG --- http response --- 05:30:57 DEBUG --- http response --- 05:30:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:30:57 DEBUG ----- output ----- 05:30:57 DEBUG { 05:30:57 DEBUG "status": "success", 05:30:57 DEBUG "data": { 05:30:57 DEBUG "resultType": "vector", 05:30:57 DEBUG "result": [ 05:30:57 DEBUG { 05:30:57 DEBUG "metric": { 05:30:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:30:57 DEBUG }, 05:30:57 DEBUG "value": [ 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG [...] 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:30:57 DEBUG }, 05:30:57 DEBUG "value": [ 05:30:57 DEBUG 1713501057, 05:30:57 DEBUG "2167.733333333333" 05:30:57 DEBUG ] 05:30:57 DEBUG } 05:30:57 DEBUG ] 05:30:57 DEBUG } 05:30:57 DEBUG } 05:30:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 05:30:57 DEBUG ----- output ----- 05:30:57 DEBUG { 05:30:57 DEBUG "status": "success", 05:30:57 DEBUG "data": { 05:30:57 DEBUG "resultType": "vector", 05:30:57 DEBUG "result": [ 05:30:57 DEBUG { 05:30:57 DEBUG "metric": { 05:30:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:30:57 DEBUG }, 05:30:57 DEBUG "value": [ 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG [...] 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:30:57 DEBUG }, 05:30:57 DEBUG "value": [ 05:30:57 DEBUG 1713501057, 05:30:57 DEBUG "0" 05:30:57 DEBUG ] 05:30:57 DEBUG } 05:30:57 DEBUG ] 05:30:57 DEBUG } 05:30:57 DEBUG } 05:30:57 INFO [http_cmd]: http status code OK 05:30:57 DEBUG --- status code --- 05:30:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:30:57 DEBUG --- http response --- 05:30:57 INFO [http_cmd]: http status code OK 05:30:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:30:57 INFO [http_cmd]: http status code OK 05:30:57 DEBUG --- status code --- 05:30:57 DEBUG ----- output ----- 05:30:57 DEBUG --- status code --- 05:30:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:30:57 DEBUG { 05:30:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:30:57 DEBUG --- http response --- 05:30:57 DEBUG "status": "success", 05:30:57 DEBUG --- http response --- 05:30:57 DEBUG "data": { 05:30:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:30:57 DEBUG "resultType": "vector", 05:30:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 05:30:57 DEBUG ----- output ----- 05:30:57 DEBUG "result": [ 05:30:57 DEBUG ----- output ----- 05:30:57 DEBUG { 05:30:57 DEBUG { 05:30:57 DEBUG { 05:30:57 DEBUG "status": "success", 05:30:57 DEBUG "metric": { 05:30:57 DEBUG "status": "success", 05:30:57 DEBUG "data": { 05:30:57 DEBUG "pod": "ds-idrepo-0" 05:30:57 DEBUG "data": { 05:30:57 DEBUG "resultType": "vector", 05:30:57 DEBUG }, 05:30:57 DEBUG "resultType": "vector", 05:30:57 DEBUG "result": [ 05:30:57 DEBUG "value": [ 05:30:57 DEBUG "result": [ 05:30:57 DEBUG { 05:30:57 DEBUG 05:30:57 DEBUG { 05:30:57 DEBUG "metric": { 05:30:57 DEBUG 05:30:57 DEBUG "metric": { 05:30:57 DEBUG "instance": "10.142.0.107:9100" 05:30:57 DEBUG 05:30:57 INFO [http_cmd]: http status code OK 05:30:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:30:57 DEBUG }, 05:30:57 DEBUG [...] 05:30:57 DEBUG --- status code --- 05:30:57 DEBUG }, 05:30:57 DEBUG "value": [ 05:30:57 DEBUG 05:30:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:30:57 DEBUG "value": [ 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG --- http response --- 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:30:57 DEBUG 05:30:57 DEBUG "pod": "ds-idrepo-2" 05:30:57 INFO [http_cmd]: http status code OK 05:30:57 DEBUG 05:30:57 DEBUG ----- output ----- 05:30:57 DEBUG [...] 05:30:57 DEBUG }, 05:30:57 DEBUG --- status code --- 05:30:57 DEBUG [...] 05:30:57 DEBUG { 05:30:57 DEBUG 05:30:57 DEBUG "value": [ 05:30:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:30:57 DEBUG 05:30:57 DEBUG "status": "success", 05:30:57 DEBUG 05:30:57 DEBUG 1713501057, 05:30:57 DEBUG --- http response --- 05:30:57 DEBUG 05:30:57 DEBUG "data": { 05:30:57 DEBUG 05:30:57 DEBUG "0" 05:30:57 DEBUG 05:30:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:30:57 DEBUG "resultType": "vector", 05:30:57 DEBUG "instance": "10.142.0.43:9100" 05:30:57 DEBUG ] 05:30:57 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:30:57 DEBUG ----- output ----- 05:30:57 DEBUG "result": [ 05:30:57 DEBUG }, 05:30:57 DEBUG } 05:30:57 DEBUG }, 05:30:57 DEBUG { 05:30:57 DEBUG { 05:30:57 DEBUG "value": [ 05:30:57 DEBUG ] 05:30:57 DEBUG "value": [ 05:30:57 DEBUG "status": "success", 05:30:57 DEBUG "metric": { 05:30:57 DEBUG 1713501057, 05:30:57 DEBUG } 05:30:57 DEBUG 1713501057, 05:30:57 DEBUG "data": { 05:30:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:30:57 DEBUG "64079884.6" 05:30:57 DEBUG } 05:30:57 DEBUG "0" 05:30:57 DEBUG "resultType": "vector", 05:30:57 DEBUG }, 05:30:57 DEBUG ] 05:30:57 DEBUG ] 05:30:57 DEBUG "result": [ 05:30:57 DEBUG "value": [ 05:30:57 DEBUG } 05:30:57 DEBUG } 05:30:57 DEBUG { 05:30:57 DEBUG 05:30:57 DEBUG ] 05:30:57 DEBUG ] 05:30:57 DEBUG "metric": { 05:30:57 DEBUG 05:30:57 DEBUG } 05:30:57 DEBUG } 05:30:57 DEBUG "instance": "10.142.0.107:9100" 05:30:57 DEBUG 05:30:57 DEBUG } 05:30:57 DEBUG } 05:30:57 DEBUG }, 05:30:57 DEBUG [...] 05:30:57 DEBUG "value": [ 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG 05:30:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:30:57 DEBUG [...] 05:30:57 DEBUG }, 05:30:57 DEBUG 05:30:57 DEBUG "value": [ 05:30:57 DEBUG 05:30:57 DEBUG 1713501057, 05:30:57 DEBUG 05:30:57 DEBUG "7.114500520495624" 05:30:57 DEBUG "instance": "10.142.0.43:9100" 05:30:57 DEBUG ] 05:30:57 DEBUG }, 05:30:57 DEBUG } 05:30:57 DEBUG "value": [ 05:30:57 DEBUG ] 05:30:57 DEBUG 1713501057, 05:30:57 DEBUG } 05:30:57 DEBUG "0.2040000000000015" 05:30:57 DEBUG } 05:30:57 DEBUG ] 05:30:57 DEBUG } 05:30:57 DEBUG ] 05:30:57 DEBUG } 05:30:57 DEBUG } 05:30:58 INFO 05:30:58 INFO [loop_until]: kubectl --namespace=xlou top pods 05:30:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:30:59 INFO [loop_until]: OK (rc = 0) 05:30:59 DEBUG --- stdout --- 05:30:59 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 13703m 9139Mi am-6fd554f8f7-hvvkw 13381m 8905Mi am-6fd554f8f7-pmj4b 14342m 8793Mi ds-cts-0 20159m 13105Mi ds-cts-1 18943m 13272Mi ds-cts-2 19178m 13251Mi ds-idrepo-0 5958m 3267Mi ds-idrepo-1 5799m 3199Mi ds-idrepo-2 5762m 3065Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 4m 1215Mi lodemon-6b89d774d5-rf7sk 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 6927m 2033Mi 05:30:59 DEBUG --- stderr --- 05:30:59 DEBUG 05:30:59 INFO 05:30:59 INFO [loop_until]: kubectl --namespace=xlou top node 05:30:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:30:59 INFO [loop_until]: OK (rc = 0) 05:30:59 DEBUG --- stdout --- 05:30:59 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 69m 0% 6247Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5809m 19% 6709Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13231m 44% 11622Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 82m 0% 3254Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 19713m 66% 15601Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 155m 0% 5172Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 13884m 46% 11923Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 11248m 37% 2734Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 15177m 50% 11002Mi 9% gke-xlou-cdm-ds-0f601751-j488 6163m 20% 6167Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 18514m 62% 17367Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6377m 21% 6884Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 19366m 64% 15274Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 7253m 45% 5739Mi 9% 05:30:59 DEBUG --- stderr --- 05:30:59 DEBUG 05:31:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713501116" 05:31:56 INFO [http_cmd]: http status code OK 05:31:56 DEBUG --- status code --- 05:31:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:31:56 DEBUG --- http response --- 05:31:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:31:56 DEBUG ----- output ----- 05:31:56 DEBUG { 05:31:56 DEBUG "status": "success", 05:31:56 DEBUG "data": { 05:31:56 DEBUG "resultType": "vector", 05:31:56 DEBUG "result": [ 05:31:56 DEBUG { 05:31:56 DEBUG "metric": { 05:31:56 DEBUG "pod": "ds-idrepo-0" 05:31:56 DEBUG }, 05:31:56 DEBUG "value": [ 05:31:56 DEBUG 05:31:56 DEBUG 05:31:56 DEBUG 05:31:56 DEBUG [...] 05:31:56 DEBUG 05:31:56 DEBUG 05:31:56 DEBUG 05:31:56 DEBUG "pod": "ds-cts-2" 05:31:56 DEBUG }, 05:31:56 DEBUG "value": [ 05:31:56 DEBUG 1713501116, 05:31:56 DEBUG "0" 05:31:56 DEBUG ] 05:31:56 DEBUG } 05:31:56 DEBUG ] 05:31:56 DEBUG } 05:31:56 DEBUG } 05:31:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501117" 05:31:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501117" 05:31:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501117" 05:31:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501117" 05:31:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713501117" 05:31:57 INFO [http_cmd]: http status code OK 05:31:57 INFO [http_cmd]: http status code OK 05:31:57 INFO [http_cmd]: http status code OK 05:31:57 DEBUG --- status code --- 05:31:57 DEBUG --- status code --- 05:31:57 DEBUG --- status code --- 05:31:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:31:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:31:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:31:57 DEBUG --- http response --- 05:31:57 DEBUG --- http response --- 05:31:57 DEBUG --- http response --- 05:31:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:31:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:31:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:31:57 DEBUG ----- output ----- 05:31:57 DEBUG ----- output ----- 05:31:57 DEBUG { 05:31:57 DEBUG { 05:31:57 DEBUG "status": "success", 05:31:57 DEBUG "status": "success", 05:31:57 DEBUG "data": { 05:31:57 DEBUG "data": { 05:31:57 DEBUG "resultType": "vector", 05:31:57 DEBUG "resultType": "vector", 05:31:57 DEBUG "result": [ 05:31:57 DEBUG "result": [ 05:31:57 DEBUG { 05:31:57 DEBUG { 05:31:57 DEBUG "metric": { 05:31:57 DEBUG "metric": { 05:31:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:31:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:31:57 DEBUG }, 05:31:57 DEBUG }, 05:31:57 DEBUG "value": [ 05:31:57 DEBUG "value": [ 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG [...] 05:31:57 DEBUG [...] 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:31:57 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 05:31:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501117" 05:31:57 DEBUG }, 05:31:57 DEBUG }, 05:31:57 DEBUG "value": [ 05:31:57 DEBUG "value": [ 05:31:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501117" 05:31:57 DEBUG 1713501117, 05:31:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501117" 05:31:57 DEBUG 1713501117, 05:31:57 DEBUG "0" 05:31:57 DEBUG "0" 05:31:57 DEBUG ] 05:31:57 DEBUG ] 05:31:57 DEBUG } 05:31:57 DEBUG } 05:31:57 DEBUG ] 05:31:57 DEBUG ] 05:31:57 DEBUG } 05:31:57 DEBUG } 05:31:57 DEBUG } 05:31:57 DEBUG } 05:31:57 ERROR [http_cmd]: ERROR 05:31:57 ERROR --- status code --- 05:31:57 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:31:57 INFO [http_cmd]: http status code OK 05:31:57 ERROR --- http response --- 05:31:57 DEBUG --- status code --- 05:31:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:31:57 DEBUG --- http response --- 05:31:57 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:31:57 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:31:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:31:57 DEBUG ----- output ----- 05:31:57 DEBUG { 05:31:57 DEBUG "status": "success", 05:31:57 DEBUG "data": { 05:31:57 DEBUG "resultType": "vector", 05:31:57 DEBUG "result": [ 05:31:57 DEBUG { 05:31:57 DEBUG "metric": { 05:31:57 DEBUG "instance": "10.142.0.107:9100" 05:31:57 DEBUG }, 05:31:57 DEBUG "value": [ 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG [...] 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG "instance": "10.142.0.43:9100" 05:31:57 DEBUG }, 05:31:57 DEBUG "value": [ 05:31:57 DEBUG 1713501117, 05:31:57 DEBUG "511283692544" 05:31:57 DEBUG ] 05:31:57 DEBUG } 05:31:57 DEBUG ] 05:31:57 DEBUG } 05:31:57 DEBUG } 05:31:57 INFO [http_cmd]: http status code OK 05:31:57 INFO [http_cmd]: http status code OK 05:31:57 INFO [http_cmd]: http status code OK 05:31:57 DEBUG --- status code --- 05:31:57 DEBUG --- status code --- 05:31:57 DEBUG --- status code --- 05:31:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:31:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:31:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:31:57 DEBUG --- http response --- 05:31:57 DEBUG --- http response --- 05:31:57 DEBUG --- http response --- 05:31:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:31:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:31:57 DEBUG ----- output ----- 05:31:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:31:57 DEBUG ----- output ----- 05:31:57 DEBUG { 05:31:57 DEBUG ----- output ----- 05:31:57 DEBUG { 05:31:57 DEBUG "status": "success", 05:31:57 DEBUG { 05:31:57 DEBUG "status": "success", 05:31:57 DEBUG "data": { 05:31:57 DEBUG "status": "success", 05:31:57 DEBUG "data": { 05:31:57 DEBUG "resultType": "vector", 05:31:57 DEBUG "data": { 05:31:57 DEBUG "resultType": "vector", 05:31:57 DEBUG "result": [ 05:31:57 DEBUG "resultType": "vector", 05:31:57 DEBUG "result": [ 05:31:57 DEBUG { 05:31:57 DEBUG "result": [ 05:31:57 DEBUG { 05:31:57 DEBUG "metric": { 05:31:57 DEBUG { 05:31:57 DEBUG "metric": { 05:31:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:31:57 DEBUG "metric": { 05:31:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:31:57 DEBUG }, 05:31:57 DEBUG "pod": "ds-cts-0" 05:31:57 DEBUG }, 05:31:57 DEBUG "value": [ 05:31:57 DEBUG }, 05:31:57 DEBUG "value": [ 05:31:57 DEBUG 05:31:57 DEBUG "value": [ 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG [...] 05:31:57 DEBUG 05:31:57 DEBUG [...] 05:31:57 DEBUG 05:31:57 DEBUG [...] 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:31:57 DEBUG 05:31:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:31:57 DEBUG }, 05:31:57 DEBUG "pod": "ds-cts-2" 05:31:57 DEBUG }, 05:31:57 DEBUG "value": [ 05:31:57 DEBUG }, 05:31:57 DEBUG "value": [ 05:31:57 DEBUG 1713501117, 05:31:57 DEBUG "value": [ 05:31:57 DEBUG 1713501117, 05:31:57 DEBUG "3643851.5" 05:31:57 DEBUG 1713501117, 05:31:57 DEBUG "2153" 05:31:57 DEBUG ] 05:31:57 DEBUG "178.9" 05:31:57 DEBUG ] 05:31:57 DEBUG } 05:31:57 DEBUG ] 05:31:57 DEBUG } 05:31:57 DEBUG ] 05:31:57 DEBUG } 05:31:57 DEBUG ] 05:31:57 DEBUG } 05:31:57 DEBUG ] 05:31:57 DEBUG } 05:31:57 DEBUG } 05:31:57 DEBUG } 05:31:57 DEBUG } 05:31:57 DEBUG } 05:31:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713501117" 05:31:57 INFO [http_cmd]: http status code OK 05:31:57 DEBUG --- status code --- 05:31:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:31:57 DEBUG --- http response --- 05:31:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:31:57 DEBUG ----- output ----- 05:31:57 DEBUG { 05:31:57 DEBUG "status": "success", 05:31:57 DEBUG "data": { 05:31:57 DEBUG "resultType": "vector", 05:31:57 DEBUG "result": [ 05:31:57 DEBUG { 05:31:57 DEBUG "metric": { 05:31:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:31:57 DEBUG }, 05:31:57 DEBUG "value": [ 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG [...] 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:31:57 DEBUG }, 05:31:57 DEBUG "value": [ 05:31:57 DEBUG 1713501117, 05:31:57 DEBUG "55.172" 05:31:57 DEBUG ] 05:31:57 DEBUG } 05:31:57 DEBUG ] 05:31:57 DEBUG } 05:31:57 DEBUG } 05:31:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501117" 05:31:57 INFO [http_cmd]: http status code OK 05:31:57 DEBUG --- status code --- 05:31:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:31:57 DEBUG --- http response --- 05:31:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:31:57 DEBUG ----- output ----- 05:31:57 DEBUG { 05:31:57 DEBUG "status": "success", 05:31:57 DEBUG "data": { 05:31:57 DEBUG "resultType": "vector", 05:31:57 DEBUG "result": [ 05:31:57 DEBUG { 05:31:57 DEBUG "metric": { 05:31:57 DEBUG "pod": "ds-cts-0" 05:31:57 DEBUG }, 05:31:57 DEBUG "value": [ 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG [...] 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG "pod": "ds-cts-2" 05:31:57 DEBUG }, 05:31:57 DEBUG "value": [ 05:31:57 DEBUG 1713501117, 05:31:57 DEBUG "31.4" 05:31:57 DEBUG ] 05:31:57 DEBUG } 05:31:57 DEBUG ] 05:31:57 DEBUG } 05:31:57 DEBUG } 05:31:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501117" 05:31:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501117" 05:31:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501117" 05:31:57 INFO [http_cmd]: http status code OK 05:31:57 DEBUG --- status code --- 05:31:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:31:57 DEBUG --- http response --- 05:31:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:31:57 DEBUG ----- output ----- 05:31:57 DEBUG { 05:31:57 DEBUG "status": "success", 05:31:57 DEBUG "data": { 05:31:57 DEBUG "resultType": "vector", 05:31:57 DEBUG "result": [ 05:31:57 DEBUG { 05:31:57 DEBUG "metric": { 05:31:57 DEBUG "instance": "10.142.0.107:9100" 05:31:57 DEBUG }, 05:31:57 DEBUG "value": [ 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG [...] 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG "instance": "10.142.0.43:9100" 05:31:57 DEBUG }, 05:31:57 DEBUG "value": [ 05:31:57 DEBUG 1713501117, 05:31:57 DEBUG "0.7762999999999977" 05:31:57 DEBUG ] 05:31:57 DEBUG } 05:31:57 DEBUG ] 05:31:57 DEBUG } 05:31:57 DEBUG } 05:31:57 INFO [http_cmd]: http status code OK 05:31:57 DEBUG --- status code --- 05:31:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:31:57 DEBUG --- http response --- 05:31:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:31:57 DEBUG ----- output ----- 05:31:57 DEBUG { 05:31:57 DEBUG "status": "success", 05:31:57 DEBUG "data": { 05:31:57 DEBUG "resultType": "vector", 05:31:57 DEBUG "result": [ 05:31:57 DEBUG { 05:31:57 DEBUG "metric": { 05:31:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:31:57 DEBUG }, 05:31:57 DEBUG "value": [ 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG [...] 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:31:57 DEBUG }, 05:31:57 DEBUG "value": [ 05:31:57 DEBUG 1713501117, 05:31:57 DEBUG "2002.3333333333333" 05:31:57 DEBUG ] 05:31:57 DEBUG } 05:31:57 DEBUG ] 05:31:57 DEBUG } 05:31:57 DEBUG } 05:31:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713501117" 05:31:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501117" 05:31:57 INFO [http_cmd]: http status code OK 05:31:57 DEBUG --- status code --- 05:31:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:31:57 DEBUG --- http response --- 05:31:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:31:57 DEBUG ----- output ----- 05:31:57 DEBUG { 05:31:57 DEBUG "status": "success", 05:31:57 DEBUG "data": { 05:31:57 DEBUG "resultType": "vector", 05:31:57 DEBUG "result": [ 05:31:57 DEBUG { 05:31:57 DEBUG "metric": { 05:31:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:31:57 DEBUG }, 05:31:57 DEBUG "value": [ 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG [...] 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:31:57 DEBUG }, 05:31:57 DEBUG "value": [ 05:31:57 DEBUG 1713501117, 05:31:57 DEBUG "2152.4" 05:31:57 DEBUG ] 05:31:57 DEBUG } 05:31:57 DEBUG ] 05:31:57 DEBUG } 05:31:57 DEBUG } 05:31:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501117" 05:31:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501117" 05:31:57 INFO [http_cmd]: http status code OK 05:31:57 INFO [http_cmd]: http status code OK 05:31:57 DEBUG --- status code --- 05:31:57 DEBUG --- status code --- 05:31:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:31:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:31:57 DEBUG --- http response --- 05:31:57 DEBUG --- http response --- 05:31:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:31:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:31:57 DEBUG ----- output ----- 05:31:57 DEBUG ----- output ----- 05:31:57 DEBUG { 05:31:57 DEBUG { 05:31:57 DEBUG "status": "success", 05:31:57 DEBUG "status": "success", 05:31:57 DEBUG "data": { 05:31:57 DEBUG "data": { 05:31:57 DEBUG "resultType": "vector", 05:31:57 DEBUG "resultType": "vector", 05:31:57 DEBUG "result": [ 05:31:57 DEBUG "result": [ 05:31:57 DEBUG { 05:31:57 DEBUG { 05:31:57 DEBUG "metric": { 05:31:57 DEBUG "metric": { 05:31:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:31:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:31:57 DEBUG }, 05:31:57 DEBUG }, 05:31:57 DEBUG "value": [ 05:31:57 DEBUG "value": [ 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG [...] 05:31:57 INFO [http_cmd]: http status code OK 05:31:57 DEBUG [...] 05:31:57 DEBUG 05:31:57 DEBUG --- status code --- 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG --- http response --- 05:31:57 DEBUG 05:31:57 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 05:31:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:31:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:31:57 DEBUG }, 05:31:57 DEBUG }, 05:31:57 DEBUG ----- output ----- 05:31:57 DEBUG "value": [ 05:31:57 DEBUG "value": [ 05:31:57 DEBUG { 05:31:57 DEBUG 1713501117, 05:31:57 DEBUG 1713501117, 05:31:57 DEBUG "status": "success", 05:31:57 DEBUG "0.00002740691043274296" 05:31:57 DEBUG "2229927936" 05:31:57 DEBUG "data": { 05:31:57 DEBUG ] 05:31:57 DEBUG ] 05:31:57 DEBUG "resultType": "vector", 05:31:57 DEBUG } 05:31:57 DEBUG } 05:31:57 DEBUG "result": [ 05:31:57 DEBUG ] 05:31:57 DEBUG ] 05:31:57 INFO [http_cmd]: http status code OK 05:31:57 DEBUG { 05:31:57 DEBUG } 05:31:57 DEBUG } 05:31:57 DEBUG --- status code --- 05:31:57 DEBUG "metric": { 05:31:57 DEBUG } 05:31:57 DEBUG } 05:31:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:31:57 DEBUG "pod": "ds-cts-0" 05:31:57 DEBUG --- http response --- 05:31:57 DEBUG }, 05:31:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:31:57 DEBUG "value": [ 05:31:57 DEBUG ----- output ----- 05:31:57 DEBUG 05:31:57 DEBUG { 05:31:57 DEBUG 05:31:57 DEBUG "status": "success", 05:31:57 DEBUG 05:31:57 DEBUG "data": { 05:31:57 DEBUG [...] 05:31:57 DEBUG "resultType": "vector", 05:31:57 DEBUG 05:31:57 DEBUG "result": [ 05:31:57 DEBUG 05:31:57 DEBUG { 05:31:57 DEBUG 05:31:57 DEBUG "metric": { 05:31:57 DEBUG "pod": "ds-cts-2" 05:31:57 DEBUG "pod": "ds-idrepo-0" 05:31:57 DEBUG }, 05:31:57 DEBUG }, 05:31:57 DEBUG "value": [ 05:31:57 DEBUG "value": [ 05:31:57 DEBUG 1713501117, 05:31:57 DEBUG 05:31:57 DEBUG "0.0006" 05:31:57 DEBUG 05:31:57 DEBUG ] 05:31:57 DEBUG 05:31:57 DEBUG } 05:31:57 DEBUG [...] 05:31:57 DEBUG ] 05:31:57 DEBUG 05:31:57 DEBUG } 05:31:57 DEBUG 05:31:57 DEBUG } 05:31:57 DEBUG 05:31:57 DEBUG "pod": "ds-idrepo-2" 05:31:57 DEBUG }, 05:31:57 DEBUG "value": [ 05:31:57 DEBUG 1713501117, 05:31:57 DEBUG "0" 05:31:57 DEBUG ] 05:31:57 DEBUG } 05:31:57 DEBUG ] 05:31:57 DEBUG } 05:31:57 DEBUG } 05:31:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501117" 05:31:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501117" 05:31:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501117" 05:31:57 INFO [http_cmd]: http status code OK 05:31:57 DEBUG --- status code --- 05:31:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:31:57 DEBUG --- http response --- 05:31:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:31:57 DEBUG ----- output ----- 05:31:57 DEBUG { 05:31:57 DEBUG "status": "success", 05:31:57 DEBUG "data": { 05:31:57 DEBUG "resultType": "vector", 05:31:57 DEBUG "result": [ 05:31:57 DEBUG { 05:31:57 DEBUG "metric": { 05:31:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:31:57 DEBUG }, 05:31:57 DEBUG "value": [ 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG [...] 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:31:57 DEBUG }, 05:31:57 DEBUG "value": [ 05:31:57 DEBUG 1713501117, 05:31:57 DEBUG "2152.5666666666666" 05:31:57 DEBUG ] 05:31:57 DEBUG } 05:31:57 DEBUG ] 05:31:57 DEBUG } 05:31:57 DEBUG } 05:31:57 INFO [http_cmd]: http status code OK 05:31:57 DEBUG --- status code --- 05:31:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:31:57 DEBUG --- http response --- 05:31:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:31:57 DEBUG ----- output ----- 05:31:57 DEBUG { 05:31:57 DEBUG "status": "success", 05:31:57 DEBUG "data": { 05:31:57 DEBUG "resultType": "vector", 05:31:57 DEBUG "result": [ 05:31:57 DEBUG { 05:31:57 DEBUG "metric": { 05:31:57 DEBUG "pod": "ds-idrepo-0" 05:31:57 DEBUG }, 05:31:57 DEBUG "value": [ 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG [...] 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG "pod": "ds-idrepo-2" 05:31:57 DEBUG }, 05:31:57 DEBUG "value": [ 05:31:57 DEBUG 1713501117, 05:31:57 DEBUG "0" 05:31:57 DEBUG ] 05:31:57 DEBUG } 05:31:57 DEBUG ] 05:31:57 DEBUG } 05:31:57 DEBUG } 05:31:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501117" 05:31:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713501117" 05:31:57 INFO [http_cmd]: http status code OK 05:31:57 DEBUG --- status code --- 05:31:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:31:57 DEBUG --- http response --- 05:31:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:31:57 DEBUG ----- output ----- 05:31:57 DEBUG { 05:31:57 DEBUG "status": "success", 05:31:57 DEBUG "data": { 05:31:57 DEBUG "resultType": "vector", 05:31:57 DEBUG "result": [ 05:31:57 DEBUG { 05:31:57 DEBUG "metric": { 05:31:57 DEBUG "instance": "10.142.0.107:9100" 05:31:57 DEBUG }, 05:31:57 DEBUG "value": [ 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG [...] 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG "instance": "10.142.0.43:9100" 05:31:57 DEBUG }, 05:31:57 DEBUG "value": [ 05:31:57 DEBUG 1713501117, 05:31:57 DEBUG "0.46166666666666406" 05:31:57 DEBUG ] 05:31:57 DEBUG } 05:31:57 DEBUG ] 05:31:57 DEBUG } 05:31:57 DEBUG } 05:31:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501117" 05:31:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501117" 05:31:57 INFO [http_cmd]: http status code OK 05:31:57 DEBUG --- status code --- 05:31:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:31:57 DEBUG --- http response --- 05:31:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:31:57 DEBUG ----- output ----- 05:31:57 DEBUG { 05:31:57 DEBUG "status": "success", 05:31:57 DEBUG "data": { 05:31:57 DEBUG "resultType": "vector", 05:31:57 DEBUG "result": [ 05:31:57 DEBUG { 05:31:57 DEBUG "metric": { 05:31:57 DEBUG "pod": "ds-idrepo-0" 05:31:57 DEBUG }, 05:31:57 DEBUG "value": [ 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG [...] 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG "pod": "ds-idrepo-2" 05:31:57 DEBUG }, 05:31:57 DEBUG "value": [ 05:31:57 DEBUG 1713501117, 05:31:57 DEBUG "0" 05:31:57 DEBUG ] 05:31:57 DEBUG } 05:31:57 DEBUG ] 05:31:57 DEBUG } 05:31:57 DEBUG } 05:31:57 INFO [http_cmd]: http status code OK 05:31:57 DEBUG --- status code --- 05:31:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:31:57 DEBUG --- http response --- 05:31:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:31:57 DEBUG ----- output ----- 05:31:57 DEBUG { 05:31:57 DEBUG "status": "success", 05:31:57 DEBUG "data": { 05:31:57 DEBUG "resultType": "vector", 05:31:57 DEBUG "result": [ 05:31:57 DEBUG { 05:31:57 DEBUG "metric": { 05:31:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:31:57 DEBUG }, 05:31:57 DEBUG "value": [ 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG [...] 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:31:57 DEBUG }, 05:31:57 DEBUG "value": [ 05:31:57 DEBUG 1713501117, 05:31:57 DEBUG "7.16225258542253" 05:31:57 DEBUG ] 05:31:57 DEBUG } 05:31:57 DEBUG ] 05:31:57 DEBUG } 05:31:57 DEBUG } 05:31:57 INFO [http_cmd]: http status code OK 05:31:57 DEBUG --- status code --- 05:31:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:31:57 DEBUG --- http response --- 05:31:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:31:57 DEBUG ----- output ----- 05:31:57 DEBUG { 05:31:57 DEBUG "status": "success", 05:31:57 DEBUG "data": { 05:31:57 DEBUG "resultType": "vector", 05:31:57 DEBUG "result": [ 05:31:57 DEBUG { 05:31:57 DEBUG "metric": { 05:31:57 DEBUG "instance": "10.142.0.107:9100" 05:31:57 DEBUG }, 05:31:57 DEBUG "value": [ 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG [...] 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG "instance": "10.142.0.43:9100" 05:31:57 DEBUG }, 05:31:57 DEBUG "value": [ 05:31:57 DEBUG 1713501117, 05:31:57 DEBUG "63907772.266666666" 05:31:57 DEBUG ] 05:31:57 DEBUG } 05:31:57 DEBUG ] 05:31:57 DEBUG } 05:31:57 DEBUG } 05:31:57 INFO [http_cmd]: http status code OK 05:31:57 DEBUG --- status code --- 05:31:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:31:57 DEBUG --- http response --- 05:31:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:31:57 DEBUG ----- output ----- 05:31:57 DEBUG { 05:31:57 DEBUG "status": "success", 05:31:57 DEBUG "data": { 05:31:57 DEBUG "resultType": "vector", 05:31:57 DEBUG "result": [ 05:31:57 DEBUG { 05:31:57 DEBUG "metric": { 05:31:57 DEBUG "instance": "10.142.0.107:9100" 05:31:57 DEBUG }, 05:31:57 DEBUG "value": [ 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG [...] 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG "instance": "10.142.0.43:9100" 05:31:57 DEBUG }, 05:31:57 DEBUG "value": [ 05:31:57 DEBUG 1713501117, 05:31:57 DEBUG "63912891.5" 05:31:57 DEBUG ] 05:31:57 DEBUG } 05:31:57 DEBUG ] 05:31:57 DEBUG } 05:31:57 DEBUG } 05:31:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501117" 05:31:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501117" 05:31:57 INFO [http_cmd]: http status code OK 05:31:57 DEBUG --- status code --- 05:31:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:31:57 DEBUG --- http response --- 05:31:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:31:57 DEBUG ----- output ----- 05:31:57 DEBUG { 05:31:57 DEBUG "status": "success", 05:31:57 DEBUG "data": { 05:31:57 DEBUG "resultType": "vector", 05:31:57 DEBUG "result": [ 05:31:57 DEBUG { 05:31:57 DEBUG "metric": { 05:31:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:31:57 DEBUG }, 05:31:57 DEBUG "value": [ 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG [...] 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 05:31:57 DEBUG }, 05:31:57 DEBUG "value": [ 05:31:57 DEBUG 1713501117, 05:31:57 DEBUG "0" 05:31:57 DEBUG ] 05:31:57 DEBUG } 05:31:57 DEBUG ] 05:31:57 DEBUG } 05:31:57 DEBUG } 05:31:57 INFO [http_cmd]: http status code OK 05:31:57 DEBUG --- status code --- 05:31:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:31:57 DEBUG --- http response --- 05:31:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:31:57 DEBUG ----- output ----- 05:31:57 DEBUG { 05:31:57 DEBUG "status": "success", 05:31:57 DEBUG "data": { 05:31:57 DEBUG "resultType": "vector", 05:31:57 DEBUG "result": [ 05:31:57 DEBUG { 05:31:57 DEBUG "metric": { 05:31:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:31:57 DEBUG }, 05:31:57 DEBUG "value": [ 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG [...] 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG 05:31:57 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 05:31:57 DEBUG }, 05:31:57 DEBUG "value": [ 05:31:57 DEBUG 1713501117, 05:31:57 DEBUG "0" 05:31:57 DEBUG ] 05:31:57 DEBUG } 05:31:57 DEBUG ] 05:31:57 DEBUG } 05:31:57 DEBUG } 05:31:59 INFO 05:31:59 INFO [loop_until]: kubectl --namespace=xlou top pods 05:31:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:31:59 INFO [loop_until]: OK (rc = 0) 05:31:59 DEBUG --- stdout --- 05:31:59 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 13881m 9137Mi am-6fd554f8f7-hvvkw 13171m 8907Mi am-6fd554f8f7-pmj4b 14558m 8792Mi ds-cts-0 19560m 12886Mi ds-cts-1 18803m 13184Mi ds-cts-2 19648m 13192Mi ds-idrepo-0 5686m 3267Mi ds-idrepo-1 5644m 3198Mi ds-idrepo-2 5627m 3065Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 3m 1215Mi lodemon-6b89d774d5-rf7sk 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7063m 2035Mi 05:31:59 DEBUG --- stderr --- 05:31:59 DEBUG 05:31:59 INFO 05:31:59 INFO [loop_until]: kubectl --namespace=xlou top node 05:31:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:31:59 INFO [loop_until]: OK (rc = 0) 05:31:59 DEBUG --- stdout --- 05:31:59 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 69m 0% 6252Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5771m 19% 6706Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13186m 44% 11603Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 85m 0% 3252Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 18999m 63% 15480Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 157m 0% 5180Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 13691m 45% 11925Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 11212m 37% 2719Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 15286m 51% 11002Mi 9% gke-xlou-cdm-ds-0f601751-j488 6092m 20% 6177Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 18705m 62% 17319Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6373m 21% 6888Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 19975m 66% 15173Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 7236m 45% 5739Mi 9% 05:31:59 DEBUG --- stderr --- 05:31:59 DEBUG 05:32:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713501176" 05:32:57 INFO [http_cmd]: http status code OK 05:32:57 DEBUG --- status code --- 05:32:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:32:57 DEBUG --- http response --- 05:32:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:32:57 DEBUG ----- output ----- 05:32:57 DEBUG { 05:32:57 DEBUG "status": "success", 05:32:57 DEBUG "data": { 05:32:57 DEBUG "resultType": "vector", 05:32:57 DEBUG "result": [ 05:32:57 DEBUG { 05:32:57 DEBUG "metric": { 05:32:57 DEBUG "pod": "ds-idrepo-0" 05:32:57 DEBUG }, 05:32:57 DEBUG "value": [ 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG [...] 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG "pod": "ds-cts-2" 05:32:57 DEBUG }, 05:32:57 DEBUG "value": [ 05:32:57 DEBUG 1713501176, 05:32:57 DEBUG "0" 05:32:57 DEBUG ] 05:32:57 DEBUG } 05:32:57 DEBUG ] 05:32:57 DEBUG } 05:32:57 DEBUG } 05:32:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501177" 05:32:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501177" 05:32:57 INFO [http_cmd]: http status code OK 05:32:57 DEBUG --- status code --- 05:32:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:32:57 DEBUG --- http response --- 05:32:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:32:57 DEBUG ----- output ----- 05:32:57 DEBUG { 05:32:57 DEBUG "status": "success", 05:32:57 DEBUG "data": { 05:32:57 DEBUG "resultType": "vector", 05:32:57 DEBUG "result": [ 05:32:57 DEBUG { 05:32:57 DEBUG "metric": { 05:32:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:32:57 DEBUG }, 05:32:57 DEBUG "value": [ 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG [...] 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:32:57 DEBUG }, 05:32:57 DEBUG "value": [ 05:32:57 DEBUG 1713501177, 05:32:57 DEBUG "3653400" 05:32:57 DEBUG ] 05:32:57 DEBUG } 05:32:57 DEBUG ] 05:32:57 DEBUG } 05:32:57 DEBUG } 05:32:57 INFO [http_cmd]: http status code OK 05:32:57 DEBUG --- status code --- 05:32:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:32:57 DEBUG --- http response --- 05:32:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:32:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501177" 05:32:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501177" 05:32:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713501177" 05:32:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501177" 05:32:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713501177" 05:32:57 INFO [http_cmd]: http status code OK 05:32:57 DEBUG --- status code --- 05:32:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:32:57 DEBUG --- http response --- 05:32:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:32:57 DEBUG ----- output ----- 05:32:57 DEBUG { 05:32:57 DEBUG "status": "success", 05:32:57 DEBUG "data": { 05:32:57 DEBUG "resultType": "vector", 05:32:57 DEBUG "result": [ 05:32:57 DEBUG { 05:32:57 DEBUG "metric": { 05:32:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:32:57 DEBUG }, 05:32:57 DEBUG "value": [ 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG [...] 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:32:57 DEBUG }, 05:32:57 DEBUG "value": [ 05:32:57 DEBUG 1713501177, 05:32:57 DEBUG "14290.076335877862" 05:32:57 DEBUG ] 05:32:57 DEBUG } 05:32:57 DEBUG ] 05:32:57 DEBUG } 05:32:57 DEBUG } 05:32:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501177" 05:32:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501177" 05:32:57 INFO [http_cmd]: http status code OK 05:32:57 DEBUG --- status code --- 05:32:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:32:57 DEBUG --- http response --- 05:32:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:32:57 DEBUG ----- output ----- 05:32:57 DEBUG { 05:32:57 DEBUG "status": "success", 05:32:57 DEBUG "data": { 05:32:57 DEBUG "resultType": "vector", 05:32:57 DEBUG "result": [ 05:32:57 DEBUG { 05:32:57 DEBUG "metric": { 05:32:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:32:57 DEBUG }, 05:32:57 DEBUG "value": [ 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG [...] 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:32:57 DEBUG }, 05:32:57 DEBUG "value": [ 05:32:57 DEBUG 1713501177, 05:32:57 DEBUG "55.172" 05:32:57 DEBUG ] 05:32:57 DEBUG } 05:32:57 DEBUG ] 05:32:57 DEBUG } 05:32:57 DEBUG } 05:32:57 INFO [http_cmd]: http status code OK 05:32:57 INFO [http_cmd]: http status code OK 05:32:57 ERROR [http_cmd]: ERROR 05:32:57 DEBUG --- status code --- 05:32:57 INFO [http_cmd]: http status code OK 05:32:57 DEBUG --- status code --- 05:32:57 ERROR --- status code --- 05:32:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:32:57 DEBUG --- status code --- 05:32:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:32:57 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:32:57 DEBUG --- http response --- 05:32:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:32:57 DEBUG --- http response --- 05:32:57 ERROR --- http response --- 05:32:57 DEBUG --- http response --- 05:32:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:32:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:32:57 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:32:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:32:57 DEBUG ----- output ----- 05:32:57 DEBUG ----- output ----- 05:32:57 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:32:57 DEBUG ----- output ----- 05:32:57 DEBUG { 05:32:57 DEBUG { 05:32:57 DEBUG { 05:32:57 DEBUG "status": "success", 05:32:57 DEBUG "status": "success", 05:32:57 DEBUG "status": "success", 05:32:57 DEBUG "data": { 05:32:57 DEBUG "data": { 05:32:57 DEBUG "data": { 05:32:57 DEBUG "resultType": "vector", 05:32:57 DEBUG "resultType": "vector", 05:32:57 DEBUG "resultType": "vector", 05:32:57 INFO [http_cmd]: http status code OK 05:32:57 DEBUG "result": [ 05:32:57 DEBUG "result": [ 05:32:57 DEBUG "result": [ 05:32:57 DEBUG --- status code --- 05:32:57 DEBUG { 05:32:57 DEBUG { 05:32:57 DEBUG { 05:32:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:32:57 DEBUG "metric": { 05:32:57 DEBUG "metric": { 05:32:57 DEBUG "metric": { 05:32:57 DEBUG --- http response --- 05:32:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:32:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:32:57 DEBUG "instance": "10.142.0.107:9100" 05:32:57 DEBUG }, 05:32:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:32:57 DEBUG }, 05:32:57 DEBUG }, 05:32:57 DEBUG "value": [ 05:32:57 DEBUG ----- output ----- 05:32:57 DEBUG "value": [ 05:32:57 DEBUG "value": [ 05:32:57 DEBUG 05:32:57 DEBUG { 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG "status": "success", 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG "data": { 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG [...] 05:32:57 DEBUG "resultType": "vector", 05:32:57 DEBUG [...] 05:32:57 DEBUG [...] 05:32:57 DEBUG 05:32:57 DEBUG "result": [ 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG { 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG "metric": { 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:32:57 DEBUG "pod": "ds-cts-0" 05:32:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:32:57 DEBUG "instance": "10.142.0.43:9100" 05:32:57 DEBUG }, 05:32:57 DEBUG }, 05:32:57 DEBUG }, 05:32:57 DEBUG }, 05:32:57 DEBUG "value": [ 05:32:57 DEBUG "value": [ 05:32:57 DEBUG "value": [ 05:32:57 DEBUG "value": [ 05:32:57 DEBUG 1713501177, 05:32:57 DEBUG 05:32:57 DEBUG 1713501177, 05:32:57 DEBUG 1713501177, 05:32:57 DEBUG "2171.366666666667" 05:32:57 DEBUG 05:32:57 DEBUG "0" 05:32:57 DEBUG "515103777792" 05:32:57 DEBUG ] 05:32:57 DEBUG 05:32:57 DEBUG ] 05:32:57 DEBUG ] 05:32:57 DEBUG } 05:32:57 DEBUG [...] 05:32:57 DEBUG } 05:32:57 DEBUG } 05:32:57 DEBUG ] 05:32:57 DEBUG 05:32:57 DEBUG ] 05:32:57 DEBUG ] 05:32:57 DEBUG } 05:32:57 DEBUG 05:32:57 DEBUG } 05:32:57 DEBUG } 05:32:57 DEBUG } 05:32:57 DEBUG 05:32:57 DEBUG } 05:32:57 DEBUG } 05:32:57 DEBUG "pod": "ds-cts-2" 05:32:57 DEBUG }, 05:32:57 DEBUG "value": [ 05:32:57 DEBUG 1713501177, 05:32:57 DEBUG "333.26666666666665" 05:32:57 DEBUG ] 05:32:57 DEBUG } 05:32:57 DEBUG ] 05:32:57 DEBUG } 05:32:57 DEBUG } 05:32:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501177" 05:32:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501177" 05:32:57 INFO [http_cmd]: http status code OK 05:32:57 DEBUG --- status code --- 05:32:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:32:57 DEBUG --- http response --- 05:32:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:32:57 DEBUG ----- output ----- 05:32:57 DEBUG { 05:32:57 DEBUG "status": "success", 05:32:57 DEBUG "data": { 05:32:57 DEBUG "resultType": "vector", 05:32:57 DEBUG "result": [ 05:32:57 DEBUG { 05:32:57 DEBUG "metric": { 05:32:57 DEBUG "pod": "ds-cts-0" 05:32:57 DEBUG }, 05:32:57 DEBUG "value": [ 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG [...] 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG "pod": "ds-cts-2" 05:32:57 DEBUG }, 05:32:57 DEBUG "value": [ 05:32:57 DEBUG 1713501177, 05:32:57 DEBUG "30" 05:32:57 DEBUG ] 05:32:57 DEBUG } 05:32:57 DEBUG ] 05:32:57 DEBUG } 05:32:57 DEBUG } 05:32:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501177" 05:32:57 INFO [http_cmd]: http status code OK 05:32:57 DEBUG --- status code --- 05:32:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:32:57 DEBUG --- http response --- 05:32:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:32:57 DEBUG ----- output ----- 05:32:57 DEBUG { 05:32:57 DEBUG "status": "success", 05:32:57 DEBUG "data": { 05:32:57 DEBUG "resultType": "vector", 05:32:57 DEBUG "result": [ 05:32:57 DEBUG { 05:32:57 DEBUG "metric": { 05:32:57 DEBUG "pod": "ds-cts-0" 05:32:57 DEBUG }, 05:32:57 DEBUG "value": [ 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG [...] 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG "pod": "ds-cts-2" 05:32:57 DEBUG }, 05:32:57 DEBUG "value": [ 05:32:57 DEBUG 1713501177, 05:32:57 DEBUG "0.0006666666666666666" 05:32:57 DEBUG ] 05:32:57 DEBUG } 05:32:57 DEBUG ] 05:32:57 DEBUG } 05:32:57 DEBUG } 05:32:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501177" 05:32:57 INFO [http_cmd]: http status code OK 05:32:57 DEBUG --- status code --- 05:32:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:32:57 DEBUG --- http response --- 05:32:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:32:57 DEBUG ----- output ----- 05:32:57 DEBUG { 05:32:57 DEBUG "status": "success", 05:32:57 DEBUG "data": { 05:32:57 DEBUG "resultType": "vector", 05:32:57 DEBUG "result": [ 05:32:57 DEBUG { 05:32:57 DEBUG "metric": { 05:32:57 DEBUG "instance": "10.142.0.107:9100" 05:32:57 DEBUG }, 05:32:57 DEBUG "value": [ 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG [...] 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG "instance": "10.142.0.43:9100" 05:32:57 DEBUG }, 05:32:57 DEBUG "value": [ 05:32:57 DEBUG 1713501177, 05:32:57 DEBUG "0.41853333333333276" 05:32:57 DEBUG ] 05:32:57 DEBUG } 05:32:57 DEBUG ] 05:32:57 DEBUG } 05:32:57 DEBUG } 05:32:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501177" 05:32:57 INFO [http_cmd]: http status code OK 05:32:57 DEBUG --- status code --- 05:32:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:32:57 DEBUG --- http response --- 05:32:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:32:57 DEBUG ----- output ----- 05:32:57 DEBUG { 05:32:57 DEBUG "status": "success", 05:32:57 DEBUG "data": { 05:32:57 DEBUG "resultType": "vector", 05:32:57 DEBUG "result": [ 05:32:57 DEBUG { 05:32:57 DEBUG "metric": { 05:32:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:32:57 DEBUG }, 05:32:57 DEBUG "value": [ 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG [...] 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:32:57 DEBUG }, 05:32:57 DEBUG "value": [ 05:32:57 DEBUG 1713501177, 05:32:57 DEBUG "2171.4666666666667" 05:32:57 DEBUG ] 05:32:57 DEBUG } 05:32:57 DEBUG ] 05:32:57 DEBUG } 05:32:57 DEBUG } 05:32:57 INFO [http_cmd]: http status code OK 05:32:57 DEBUG --- status code --- 05:32:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:32:57 DEBUG --- http response --- 05:32:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:32:57 DEBUG ----- output ----- 05:32:57 DEBUG { 05:32:57 DEBUG "status": "success", 05:32:57 DEBUG "data": { 05:32:57 DEBUG "resultType": "vector", 05:32:57 DEBUG "result": [ 05:32:57 DEBUG { 05:32:57 DEBUG "metric": { 05:32:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:32:57 DEBUG }, 05:32:57 DEBUG "value": [ 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG [...] 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:32:57 DEBUG }, 05:32:57 DEBUG "value": [ 05:32:57 DEBUG 1713501177, 05:32:57 DEBUG "2006.2333333333333" 05:32:57 DEBUG ] 05:32:57 DEBUG } 05:32:57 DEBUG ] 05:32:57 DEBUG } 05:32:57 DEBUG } 05:32:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713501177" 05:32:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501177" 05:32:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501177" 05:32:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713501177" 05:32:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501177" 05:32:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501177" 05:32:57 INFO [http_cmd]: http status code OK 05:32:57 DEBUG --- status code --- 05:32:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:32:57 DEBUG --- http response --- 05:32:57 INFO [http_cmd]: http status code OK 05:32:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:32:57 DEBUG --- status code --- 05:32:57 INFO [http_cmd]: http status code OK 05:32:57 INFO [http_cmd]: http status code OK 05:32:57 DEBUG ----- output ----- 05:32:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:32:57 DEBUG --- status code --- 05:32:57 DEBUG --- status code --- 05:32:57 DEBUG { 05:32:57 DEBUG --- http response --- 05:32:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:32:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:32:57 DEBUG "status": "success", 05:32:57 DEBUG --- http response --- 05:32:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:32:57 DEBUG --- http response --- 05:32:57 DEBUG "data": { 05:32:57 DEBUG ----- output ----- 05:32:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:32:57 DEBUG "resultType": "vector", 05:32:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:32:57 DEBUG { 05:32:57 DEBUG ----- output ----- 05:32:57 DEBUG "result": [ 05:32:57 DEBUG ----- output ----- 05:32:57 DEBUG "status": "success", 05:32:57 DEBUG { 05:32:57 DEBUG { 05:32:57 DEBUG { 05:32:57 DEBUG "data": { 05:32:57 DEBUG "status": "success", 05:32:57 DEBUG "metric": { 05:32:57 DEBUG "status": "success", 05:32:57 DEBUG "resultType": "vector", 05:32:57 DEBUG "data": { 05:32:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:32:57 DEBUG "data": { 05:32:57 DEBUG "result": [ 05:32:57 DEBUG "resultType": "vector", 05:32:57 DEBUG }, 05:32:57 DEBUG "resultType": "vector", 05:32:57 DEBUG { 05:32:57 DEBUG "result": [ 05:32:57 DEBUG "value": [ 05:32:57 DEBUG "result": [ 05:32:57 DEBUG "metric": { 05:32:57 DEBUG { 05:32:57 INFO [http_cmd]: http status code OK 05:32:57 DEBUG 05:32:57 DEBUG { 05:32:57 DEBUG "pod": "ds-idrepo-0" 05:32:57 DEBUG "metric": { 05:32:57 DEBUG --- status code --- 05:32:57 DEBUG 05:32:57 INFO [http_cmd]: http status code OK 05:32:57 DEBUG "metric": { 05:32:57 DEBUG }, 05:32:57 DEBUG "pod": "ds-idrepo-0" 05:32:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:32:57 DEBUG 05:32:57 DEBUG --- status code --- 05:32:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:32:57 DEBUG "value": [ 05:32:57 DEBUG }, 05:32:57 DEBUG --- http response --- 05:32:57 DEBUG [...] 05:32:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:32:57 DEBUG }, 05:32:57 DEBUG 05:32:57 DEBUG "value": [ 05:32:57 DEBUG 05:32:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:32:57 DEBUG --- http response --- 05:32:57 DEBUG "value": [ 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG ----- output ----- 05:32:57 DEBUG 05:32:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG { 05:32:57 DEBUG 05:32:57 DEBUG ----- output ----- 05:32:57 DEBUG [...] 05:32:57 DEBUG 05:32:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:32:57 DEBUG "status": "success", 05:32:57 DEBUG 05:32:57 DEBUG { 05:32:57 DEBUG 05:32:57 DEBUG [...] 05:32:57 DEBUG }, 05:32:57 DEBUG "data": { 05:32:57 DEBUG [...] 05:32:57 DEBUG "status": "success", 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG "value": [ 05:32:57 DEBUG "resultType": "vector", 05:32:57 DEBUG 05:32:57 DEBUG "data": { 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG 1713501177, 05:32:57 DEBUG "result": [ 05:32:57 DEBUG 05:32:57 DEBUG "resultType": "vector", 05:32:57 DEBUG "pod": "ds-idrepo-2" 05:32:57 DEBUG 05:32:57 DEBUG "7.109846523801653" 05:32:57 DEBUG { 05:32:57 DEBUG 05:32:57 DEBUG "result": [ 05:32:57 DEBUG }, 05:32:57 DEBUG "pod": "ds-idrepo-2" 05:32:57 DEBUG ] 05:32:57 DEBUG "metric": { 05:32:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:32:57 DEBUG { 05:32:57 DEBUG "value": [ 05:32:57 DEBUG }, 05:32:57 DEBUG } 05:32:57 DEBUG "pod": "ds-idrepo-0" 05:32:57 DEBUG }, 05:32:57 DEBUG "metric": { 05:32:57 DEBUG 1713501177, 05:32:57 DEBUG "value": [ 05:32:57 DEBUG ] 05:32:57 DEBUG }, 05:32:57 DEBUG "value": [ 05:32:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:32:57 DEBUG "0" 05:32:57 DEBUG 1713501177, 05:32:57 DEBUG } 05:32:57 DEBUG "value": [ 05:32:57 DEBUG 1713501177, 05:32:57 DEBUG }, 05:32:57 DEBUG ] 05:32:57 DEBUG "0" 05:32:57 DEBUG } 05:32:57 DEBUG 05:32:57 DEBUG "2231672832" 05:32:57 DEBUG "value": [ 05:32:57 DEBUG } 05:32:57 DEBUG ] 05:32:57 DEBUG 05:32:57 DEBUG ] 05:32:57 DEBUG 05:32:57 DEBUG ] 05:32:57 DEBUG } 05:32:57 DEBUG 05:32:57 DEBUG } 05:32:57 DEBUG 05:32:57 DEBUG } 05:32:57 DEBUG ] 05:32:57 DEBUG [...] 05:32:57 DEBUG ] 05:32:57 DEBUG 05:32:57 DEBUG } 05:32:57 DEBUG } 05:32:57 DEBUG 05:32:57 DEBUG } 05:32:57 DEBUG [...] 05:32:57 DEBUG } 05:32:57 DEBUG 05:32:57 DEBUG } 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG "pod": "ds-idrepo-2" 05:32:57 DEBUG 05:32:57 DEBUG }, 05:32:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501177" 05:32:57 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:32:57 DEBUG "value": [ 05:32:57 DEBUG }, 05:32:57 DEBUG 1713501177, 05:32:57 DEBUG "value": [ 05:32:57 DEBUG "0" 05:32:57 DEBUG 1713501177, 05:32:57 DEBUG ] 05:32:57 DEBUG "7.077313185382931" 05:32:57 DEBUG } 05:32:57 DEBUG ] 05:32:57 DEBUG ] 05:32:57 DEBUG } 05:32:57 DEBUG } 05:32:57 DEBUG ] 05:32:57 DEBUG } 05:32:57 DEBUG } 05:32:57 DEBUG } 05:32:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501177" 05:32:57 INFO [http_cmd]: http status code OK 05:32:57 DEBUG --- status code --- 05:32:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:32:57 DEBUG --- http response --- 05:32:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:32:57 DEBUG ----- output ----- 05:32:57 DEBUG { 05:32:57 DEBUG "status": "success", 05:32:57 DEBUG "data": { 05:32:57 DEBUG "resultType": "vector", 05:32:57 DEBUG "result": [ 05:32:57 DEBUG { 05:32:57 DEBUG "metric": { 05:32:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:32:57 DEBUG }, 05:32:57 DEBUG "value": [ 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG [...] 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:32:57 DEBUG }, 05:32:57 DEBUG "value": [ 05:32:57 DEBUG 1713501177, 05:32:57 DEBUG "2171.6" 05:32:57 DEBUG ] 05:32:57 DEBUG } 05:32:57 DEBUG ] 05:32:57 DEBUG } 05:32:57 DEBUG } 05:32:57 INFO [http_cmd]: http status code OK 05:32:57 DEBUG --- status code --- 05:32:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:32:57 DEBUG --- http response --- 05:32:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:32:57 DEBUG ----- output ----- 05:32:57 DEBUG { 05:32:57 DEBUG "status": "success", 05:32:57 DEBUG "data": { 05:32:57 DEBUG "resultType": "vector", 05:32:57 DEBUG "result": [ 05:32:57 DEBUG { 05:32:57 DEBUG "metric": { 05:32:57 DEBUG "instance": "10.142.0.107:9100" 05:32:57 DEBUG }, 05:32:57 DEBUG "value": [ 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG [...] 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG "instance": "10.142.0.43:9100" 05:32:57 DEBUG }, 05:32:57 DEBUG "value": [ 05:32:57 DEBUG 1713501177, 05:32:57 DEBUG "0.23400000000000126" 05:32:57 DEBUG ] 05:32:57 DEBUG } 05:32:57 DEBUG ] 05:32:57 DEBUG } 05:32:57 DEBUG } 05:32:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501177" 05:32:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501177" 05:32:57 INFO [http_cmd]: http status code OK 05:32:57 DEBUG --- status code --- 05:32:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:32:57 DEBUG --- http response --- 05:32:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:32:57 DEBUG ----- output ----- 05:32:57 DEBUG { 05:32:57 DEBUG "status": "success", 05:32:57 DEBUG "data": { 05:32:57 INFO [http_cmd]: http status code OK 05:32:57 DEBUG "resultType": "vector", 05:32:57 DEBUG --- status code --- 05:32:57 DEBUG "result": [ 05:32:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:32:57 DEBUG { 05:32:57 DEBUG --- http response --- 05:32:57 DEBUG "metric": { 05:32:57 DEBUG "instance": "10.142.0.107:9100" 05:32:57 DEBUG }, 05:32:57 DEBUG "value": [ 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG [...] 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG "instance": "10.142.0.43:9100" 05:32:57 DEBUG }, 05:32:57 DEBUG "value": [ 05:32:57 DEBUG 1713501177, 05:32:57 DEBUG "64233673.666666664" 05:32:57 DEBUG ] 05:32:57 DEBUG } 05:32:57 DEBUG ] 05:32:57 DEBUG } 05:32:57 DEBUG } 05:32:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:32:57 DEBUG ----- output ----- 05:32:57 DEBUG { 05:32:57 DEBUG "status": "success", 05:32:57 DEBUG "data": { 05:32:57 DEBUG "resultType": "vector", 05:32:57 DEBUG "result": [ 05:32:57 DEBUG { 05:32:57 DEBUG "metric": { 05:32:57 DEBUG "instance": "10.142.0.107:9100" 05:32:57 DEBUG }, 05:32:57 DEBUG "value": [ 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG [...] 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG "instance": "10.142.0.43:9100" 05:32:57 DEBUG }, 05:32:57 DEBUG "value": [ 05:32:57 DEBUG 1713501177, 05:32:57 DEBUG "64240378.46666667" 05:32:57 DEBUG ] 05:32:57 DEBUG } 05:32:57 DEBUG ] 05:32:57 DEBUG } 05:32:57 DEBUG } 05:32:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501177" 05:32:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501177" 05:32:57 INFO [http_cmd]: http status code OK 05:32:57 DEBUG --- status code --- 05:32:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:32:57 DEBUG --- http response --- 05:32:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:32:57 DEBUG ----- output ----- 05:32:57 DEBUG { 05:32:57 DEBUG "status": "success", 05:32:57 DEBUG "data": { 05:32:57 DEBUG "resultType": "vector", 05:32:57 DEBUG "result": [ 05:32:57 DEBUG { 05:32:57 DEBUG "metric": { 05:32:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:32:57 DEBUG }, 05:32:57 DEBUG "value": [ 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG [...] 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:32:57 DEBUG }, 05:32:57 DEBUG "value": [ 05:32:57 DEBUG 1713501177, 05:32:57 DEBUG "0" 05:32:57 DEBUG ] 05:32:57 DEBUG } 05:32:57 DEBUG ] 05:32:57 DEBUG } 05:32:57 DEBUG } 05:32:57 INFO [http_cmd]: http status code OK 05:32:57 DEBUG --- status code --- 05:32:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:32:57 DEBUG --- http response --- 05:32:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:32:57 DEBUG ----- output ----- 05:32:57 DEBUG { 05:32:57 DEBUG "status": "success", 05:32:57 DEBUG "data": { 05:32:57 DEBUG "resultType": "vector", 05:32:57 DEBUG "result": [ 05:32:57 DEBUG { 05:32:57 DEBUG "metric": { 05:32:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:32:57 DEBUG }, 05:32:57 DEBUG "value": [ 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG [...] 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG 05:32:57 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:32:57 DEBUG }, 05:32:57 DEBUG "value": [ 05:32:57 DEBUG 1713501177, 05:32:57 DEBUG "0" 05:32:57 DEBUG ] 05:32:57 DEBUG } 05:32:57 DEBUG ] 05:32:57 DEBUG } 05:32:57 DEBUG } 05:32:59 INFO 05:32:59 INFO [loop_until]: kubectl --namespace=xlou top pods 05:32:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:32:59 INFO [loop_until]: OK (rc = 0) 05:32:59 DEBUG --- stdout --- 05:32:59 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 13782m 9141Mi am-6fd554f8f7-hvvkw 13779m 8907Mi am-6fd554f8f7-pmj4b 14124m 8793Mi ds-cts-0 19246m 12683Mi ds-cts-1 18505m 12662Mi ds-cts-2 19200m 12995Mi ds-idrepo-0 5817m 3267Mi ds-idrepo-1 5933m 3198Mi ds-idrepo-2 5546m 3066Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 3m 1215Mi lodemon-6b89d774d5-rf7sk 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7153m 2037Mi 05:32:59 DEBUG --- stderr --- 05:32:59 DEBUG 05:32:59 INFO 05:32:59 INFO [loop_until]: kubectl --namespace=xlou top node 05:32:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:32:59 INFO [loop_until]: OK (rc = 0) 05:32:59 DEBUG --- stdout --- 05:32:59 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 67m 0% 6228Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5804m 19% 6708Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13265m 44% 11614Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 81m 0% 3247Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 19184m 64% 15135Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 165m 0% 5174Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 13706m 45% 11918Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 11263m 37% 2720Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 15232m 51% 11000Mi 9% gke-xlou-cdm-ds-0f601751-j488 6146m 20% 6167Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 18259m 61% 16782Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6393m 21% 6884Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 19582m 65% 14952Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 7015m 44% 5744Mi 9% 05:32:59 DEBUG --- stderr --- 05:32:59 DEBUG 05:33:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713501237" 05:33:57 INFO [http_cmd]: http status code OK 05:33:57 DEBUG --- status code --- 05:33:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:33:57 DEBUG --- http response --- 05:33:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:33:57 DEBUG ----- output ----- 05:33:57 DEBUG { 05:33:57 DEBUG "status": "success", 05:33:57 DEBUG "data": { 05:33:57 DEBUG "resultType": "vector", 05:33:57 DEBUG "result": [ 05:33:57 DEBUG { 05:33:57 DEBUG "metric": { 05:33:57 DEBUG "pod": "ds-idrepo-0" 05:33:57 DEBUG }, 05:33:57 DEBUG "value": [ 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG [...] 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG "pod": "ds-cts-2" 05:33:57 DEBUG }, 05:33:57 DEBUG "value": [ 05:33:57 DEBUG 1713501237, 05:33:57 DEBUG "0" 05:33:57 DEBUG ] 05:33:57 DEBUG } 05:33:57 DEBUG ] 05:33:57 DEBUG } 05:33:57 DEBUG } 05:33:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501237" 05:33:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501237" 05:33:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713501237" 05:33:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501237" 05:33:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501237" 05:33:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501237" 05:33:57 INFO [http_cmd]: http status code OK 05:33:57 DEBUG --- status code --- 05:33:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:33:57 DEBUG --- http response --- 05:33:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:33:57 DEBUG ----- output ----- 05:33:57 DEBUG { 05:33:57 DEBUG "status": "success", 05:33:57 DEBUG "data": { 05:33:57 DEBUG "resultType": "vector", 05:33:57 DEBUG "result": [ 05:33:57 DEBUG { 05:33:57 DEBUG "metric": { 05:33:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:33:57 DEBUG }, 05:33:57 DEBUG "value": [ 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG [...] 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:33:57 DEBUG }, 05:33:57 DEBUG "value": [ 05:33:57 DEBUG 1713501237, 05:33:57 DEBUG "3663436.5" 05:33:57 DEBUG ] 05:33:57 DEBUG } 05:33:57 DEBUG ] 05:33:57 DEBUG } 05:33:57 DEBUG } 05:33:57 INFO [http_cmd]: http status code OK 05:33:57 DEBUG --- status code --- 05:33:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:33:57 DEBUG --- http response --- 05:33:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:33:57 DEBUG ----- output ----- 05:33:57 DEBUG { 05:33:57 DEBUG "status": "success", 05:33:57 DEBUG "data": { 05:33:57 DEBUG "resultType": "vector", 05:33:57 DEBUG "result": [ 05:33:57 DEBUG { 05:33:57 DEBUG "metric": { 05:33:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:33:57 DEBUG }, 05:33:57 DEBUG "value": [ 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG [...] 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:33:57 DEBUG }, 05:33:57 DEBUG "value": [ 05:33:57 DEBUG 1713501237, 05:33:57 DEBUG "0" 05:33:57 DEBUG ] 05:33:57 DEBUG } 05:33:57 DEBUG ] 05:33:57 DEBUG } 05:33:57 DEBUG } 05:33:57 INFO [http_cmd]: http status code OK 05:33:57 INFO [http_cmd]: http status code OK 05:33:57 DEBUG --- status code --- 05:33:57 INFO [http_cmd]: http status code OK 05:33:57 INFO [http_cmd]: http status code OK 05:33:57 DEBUG --- status code --- 05:33:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:33:57 DEBUG --- status code --- 05:33:57 DEBUG --- status code --- 05:33:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:33:57 DEBUG --- http response --- 05:33:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:33:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:33:57 DEBUG --- http response --- 05:33:57 DEBUG --- http response --- 05:33:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:33:57 DEBUG --- http response --- 05:33:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:33:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:33:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:33:57 DEBUG ----- output ----- 05:33:57 DEBUG ----- output ----- 05:33:57 DEBUG ----- output ----- 05:33:57 DEBUG { 05:33:57 DEBUG { 05:33:57 DEBUG { 05:33:57 DEBUG "status": "success", 05:33:57 DEBUG "status": "success", 05:33:57 DEBUG "status": "success", 05:33:57 DEBUG "data": { 05:33:57 DEBUG "data": { 05:33:57 DEBUG "data": { 05:33:57 DEBUG "resultType": "vector", 05:33:57 DEBUG "resultType": "vector", 05:33:57 DEBUG "resultType": "vector", 05:33:57 DEBUG "result": [ 05:33:57 DEBUG "result": [ 05:33:57 DEBUG "result": [ 05:33:57 DEBUG { 05:33:57 DEBUG { 05:33:57 DEBUG { 05:33:57 DEBUG "metric": { 05:33:57 DEBUG "metric": { 05:33:57 DEBUG "metric": { 05:33:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:33:57 DEBUG "pod": "ds-idrepo-1" 05:33:57 DEBUG "instance": "10.142.0.107:9100" 05:33:57 DEBUG }, 05:33:57 DEBUG }, 05:33:57 DEBUG }, 05:33:57 DEBUG "value": [ 05:33:57 DEBUG "value": [ 05:33:57 DEBUG "value": [ 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG [...] 05:33:57 DEBUG [...] 05:33:57 DEBUG [...] 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:33:57 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 05:33:57 DEBUG "instance": "10.142.0.43:9100" 05:33:57 DEBUG }, 05:33:57 DEBUG }, 05:33:57 DEBUG }, 05:33:57 DEBUG "value": [ 05:33:57 DEBUG "value": [ 05:33:57 DEBUG "value": [ 05:33:57 DEBUG 1713501237, 05:33:57 DEBUG 1713501237, 05:33:57 DEBUG 1713501237, 05:33:57 DEBUG "55.173500000000004" 05:33:57 DEBUG "0" 05:33:57 DEBUG "519305409536" 05:33:57 DEBUG ] 05:33:57 DEBUG ] 05:33:57 DEBUG ] 05:33:57 DEBUG } 05:33:57 DEBUG } 05:33:57 DEBUG } 05:33:57 DEBUG ] 05:33:57 DEBUG ] 05:33:57 DEBUG ] 05:33:57 DEBUG } 05:33:57 DEBUG } 05:33:57 DEBUG } 05:33:57 DEBUG } 05:33:57 DEBUG } 05:33:57 DEBUG } 05:33:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501237" 05:33:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501237" 05:33:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713501237" 05:33:57 INFO [http_cmd]: http status code OK 05:33:57 DEBUG --- status code --- 05:33:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:33:57 DEBUG --- http response --- 05:33:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:33:57 DEBUG ----- output ----- 05:33:57 DEBUG { 05:33:57 DEBUG "status": "success", 05:33:57 DEBUG "data": { 05:33:57 DEBUG "resultType": "vector", 05:33:57 DEBUG "result": [ 05:33:57 DEBUG { 05:33:57 DEBUG "metric": { 05:33:57 DEBUG "pod": "ds-cts-0" 05:33:57 DEBUG }, 05:33:57 DEBUG "value": [ 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG [...] 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG "pod": "ds-cts-2" 05:33:57 DEBUG }, 05:33:57 DEBUG "value": [ 05:33:57 DEBUG 1713501237, 05:33:57 DEBUG "368.93333333333334" 05:33:57 DEBUG ] 05:33:57 DEBUG } 05:33:57 DEBUG ] 05:33:57 DEBUG } 05:33:57 DEBUG } 05:33:57 INFO [http_cmd]: http status code OK 05:33:57 DEBUG --- status code --- 05:33:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:33:57 DEBUG --- http response --- 05:33:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:33:57 DEBUG ----- output ----- 05:33:57 DEBUG { 05:33:57 DEBUG "status": "success", 05:33:57 DEBUG "data": { 05:33:57 DEBUG "resultType": "vector", 05:33:57 DEBUG "result": [ 05:33:57 DEBUG { 05:33:57 DEBUG "metric": { 05:33:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:33:57 DEBUG }, 05:33:57 DEBUG "value": [ 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG [...] 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:33:57 DEBUG }, 05:33:57 DEBUG "value": [ 05:33:57 DEBUG 1713501237, 05:33:57 DEBUG "2161.133333333333" 05:33:57 DEBUG ] 05:33:57 DEBUG } 05:33:57 DEBUG ] 05:33:57 DEBUG } 05:33:57 DEBUG } 05:33:57 ERROR [http_cmd]: ERROR 05:33:57 ERROR --- status code --- 05:33:57 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:33:57 ERROR --- http response --- 05:33:57 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:33:57 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:33:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501237" 05:33:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501237" 05:33:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501237" 05:33:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713501237" 05:33:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501237" 05:33:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501237" 05:33:57 INFO [http_cmd]: http status code OK 05:33:57 INFO [http_cmd]: http status code OK 05:33:57 INFO [http_cmd]: http status code OK 05:33:57 DEBUG --- status code --- 05:33:57 DEBUG --- status code --- 05:33:57 DEBUG --- status code --- 05:33:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:33:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:33:57 INFO [http_cmd]: http status code OK 05:33:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:33:57 DEBUG --- http response --- 05:33:57 DEBUG --- http response --- 05:33:57 DEBUG --- status code --- 05:33:57 DEBUG --- http response --- 05:33:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:33:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:33:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:33:57 DEBUG ----- output ----- 05:33:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:33:57 DEBUG --- http response --- 05:33:57 DEBUG ----- output ----- 05:33:57 DEBUG { 05:33:57 DEBUG ----- output ----- 05:33:57 DEBUG { 05:33:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:33:57 DEBUG "status": "success", 05:33:57 DEBUG { 05:33:57 DEBUG "status": "success", 05:33:57 DEBUG ----- output ----- 05:33:57 DEBUG "data": { 05:33:57 DEBUG "status": "success", 05:33:57 DEBUG "data": { 05:33:57 DEBUG { 05:33:57 DEBUG "resultType": "vector", 05:33:57 INFO [http_cmd]: http status code OK 05:33:57 DEBUG "data": { 05:33:57 DEBUG "resultType": "vector", 05:33:57 DEBUG "status": "success", 05:33:57 DEBUG "result": [ 05:33:57 DEBUG --- status code --- 05:33:57 DEBUG "resultType": "vector", 05:33:57 DEBUG "result": [ 05:33:57 DEBUG "data": { 05:33:57 DEBUG { 05:33:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:33:57 DEBUG "result": [ 05:33:57 DEBUG { 05:33:57 DEBUG "resultType": "vector", 05:33:57 DEBUG "metric": { 05:33:57 DEBUG --- http response --- 05:33:57 DEBUG { 05:33:57 DEBUG "metric": { 05:33:57 DEBUG "result": [ 05:33:57 DEBUG "pod": "ds-cts-0" 05:33:57 DEBUG "metric": { 05:33:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:33:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:33:57 DEBUG { 05:33:57 DEBUG }, 05:33:57 DEBUG "pod": "ds-cts-0" 05:33:57 DEBUG ----- output ----- 05:33:57 DEBUG }, 05:33:57 DEBUG "metric": { 05:33:57 DEBUG "value": [ 05:33:57 DEBUG }, 05:33:57 DEBUG { 05:33:57 DEBUG "value": [ 05:33:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:33:57 DEBUG 05:33:57 DEBUG "value": [ 05:33:57 DEBUG "status": "success", 05:33:57 DEBUG 05:33:57 DEBUG }, 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG "data": { 05:33:57 DEBUG 05:33:57 DEBUG "value": [ 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG "resultType": "vector", 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG [...] 05:33:57 DEBUG 05:33:57 DEBUG "result": [ 05:33:57 DEBUG [...] 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG [...] 05:33:57 DEBUG { 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG "metric": { 05:33:57 DEBUG 05:33:57 DEBUG [...] 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG "instance": "10.142.0.107:9100" 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG "pod": "ds-cts-2" 05:33:57 DEBUG 05:33:57 DEBUG }, 05:33:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:33:57 DEBUG 05:33:57 DEBUG }, 05:33:57 DEBUG "pod": "ds-cts-2" 05:33:57 DEBUG "value": [ 05:33:57 DEBUG }, 05:33:57 DEBUG 05:33:57 DEBUG "value": [ 05:33:57 DEBUG }, 05:33:57 DEBUG 05:33:57 DEBUG "value": [ 05:33:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:33:57 DEBUG 1713501237, 05:33:57 DEBUG "value": [ 05:33:57 DEBUG 05:33:57 DEBUG 1713501237, 05:33:57 DEBUG }, 05:33:57 DEBUG "0.0008255911111111111" 05:33:57 DEBUG 1713501237, 05:33:57 DEBUG 05:33:57 DEBUG "1999.1333333333332" 05:33:57 DEBUG "value": [ 05:33:57 DEBUG ] 05:33:57 INFO [http_cmd]: http status code OK 05:33:57 DEBUG "28.03333333333333" 05:33:57 DEBUG [...] 05:33:57 DEBUG ] 05:33:57 DEBUG 1713501237, 05:33:57 DEBUG } 05:33:57 DEBUG --- status code --- 05:33:57 DEBUG ] 05:33:57 DEBUG 05:33:57 DEBUG } 05:33:57 DEBUG "7.08535935437724" 05:33:57 DEBUG ] 05:33:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:33:57 DEBUG } 05:33:57 DEBUG 05:33:57 DEBUG ] 05:33:57 DEBUG ] 05:33:57 DEBUG } 05:33:57 DEBUG --- http response --- 05:33:57 DEBUG ] 05:33:57 DEBUG 05:33:57 DEBUG } 05:33:57 DEBUG } 05:33:57 DEBUG } 05:33:57 DEBUG } 05:33:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:33:57 DEBUG "instance": "10.142.0.43:9100" 05:33:57 DEBUG } 05:33:57 DEBUG ] 05:33:57 DEBUG } 05:33:57 DEBUG ----- output ----- 05:33:57 DEBUG }, 05:33:57 DEBUG } 05:33:57 DEBUG { 05:33:57 DEBUG "value": [ 05:33:57 DEBUG } 05:33:57 DEBUG "status": "success", 05:33:57 DEBUG 1713501237, 05:33:57 DEBUG "data": { 05:33:57 DEBUG "0.5628999999999982" 05:33:57 DEBUG "resultType": "vector", 05:33:57 DEBUG ] 05:33:57 DEBUG "result": [ 05:33:57 DEBUG } 05:33:57 DEBUG { 05:33:57 DEBUG ] 05:33:57 DEBUG "metric": { 05:33:57 DEBUG } 05:33:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:33:57 DEBUG } 05:33:57 DEBUG }, 05:33:57 DEBUG "value": [ 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG [...] 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:33:57 DEBUG }, 05:33:57 DEBUG "value": [ 05:33:57 DEBUG 1713501237, 05:33:57 DEBUG "2160.9" 05:33:57 DEBUG ] 05:33:57 DEBUG } 05:33:57 DEBUG ] 05:33:57 DEBUG } 05:33:57 DEBUG } 05:33:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501237" 05:33:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713501237" 05:33:57 INFO [http_cmd]: http status code OK 05:33:57 DEBUG --- status code --- 05:33:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:33:57 DEBUG --- http response --- 05:33:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:33:57 DEBUG ----- output ----- 05:33:57 DEBUG { 05:33:57 DEBUG "status": "success", 05:33:57 DEBUG "data": { 05:33:57 DEBUG "resultType": "vector", 05:33:57 DEBUG "result": [ 05:33:57 DEBUG { 05:33:57 DEBUG "metric": { 05:33:57 DEBUG "pod": "ds-idrepo-0" 05:33:57 DEBUG }, 05:33:57 DEBUG "value": [ 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG [...] 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG "pod": "ds-idrepo-2" 05:33:57 DEBUG }, 05:33:57 DEBUG "value": [ 05:33:57 DEBUG 1713501237, 05:33:57 DEBUG "0" 05:33:57 DEBUG ] 05:33:57 DEBUG } 05:33:57 DEBUG ] 05:33:57 DEBUG } 05:33:57 DEBUG } 05:33:57 INFO [http_cmd]: http status code OK 05:33:57 DEBUG --- status code --- 05:33:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:33:57 DEBUG --- http response --- 05:33:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:33:57 DEBUG ----- output ----- 05:33:57 DEBUG { 05:33:57 DEBUG "status": "success", 05:33:57 DEBUG "data": { 05:33:57 DEBUG "resultType": "vector", 05:33:57 DEBUG "result": [ 05:33:57 DEBUG { 05:33:57 DEBUG "metric": { 05:33:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:33:57 DEBUG }, 05:33:57 DEBUG "value": [ 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG [...] 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:33:57 DEBUG }, 05:33:57 DEBUG "value": [ 05:33:57 DEBUG 1713501237, 05:33:57 DEBUG "2233241600" 05:33:57 DEBUG ] 05:33:57 DEBUG } 05:33:57 DEBUG ] 05:33:57 DEBUG } 05:33:57 DEBUG } 05:33:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501237" 05:33:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501237" 05:33:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501237" 05:33:57 INFO [http_cmd]: http status code OK 05:33:57 DEBUG --- status code --- 05:33:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:33:57 DEBUG --- http response --- 05:33:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:33:57 DEBUG ----- output ----- 05:33:57 DEBUG { 05:33:57 DEBUG "status": "success", 05:33:57 DEBUG "data": { 05:33:57 DEBUG "resultType": "vector", 05:33:57 DEBUG "result": [ 05:33:57 DEBUG { 05:33:57 DEBUG "metric": { 05:33:57 DEBUG "pod": "ds-idrepo-0" 05:33:57 DEBUG }, 05:33:57 DEBUG "value": [ 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG [...] 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG "pod": "ds-idrepo-2" 05:33:57 DEBUG }, 05:33:57 DEBUG "value": [ 05:33:57 DEBUG 1713501237, 05:33:57 DEBUG "0" 05:33:57 DEBUG ] 05:33:57 DEBUG } 05:33:57 DEBUG ] 05:33:57 DEBUG } 05:33:57 DEBUG } 05:33:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501237" 05:33:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501237" 05:33:57 INFO [http_cmd]: http status code OK 05:33:57 INFO [http_cmd]: http status code OK 05:33:57 DEBUG --- status code --- 05:33:57 DEBUG --- status code --- 05:33:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:33:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:33:57 DEBUG --- http response --- 05:33:57 DEBUG --- http response --- 05:33:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:33:57 DEBUG ----- output ----- 05:33:57 DEBUG { 05:33:57 DEBUG "status": "success", 05:33:57 DEBUG "data": { 05:33:57 DEBUG "resultType": "vector", 05:33:57 DEBUG "result": [ 05:33:57 DEBUG { 05:33:57 DEBUG "metric": { 05:33:57 DEBUG "pod": "ds-idrepo-0" 05:33:57 DEBUG }, 05:33:57 DEBUG "value": [ 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG [...] 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG "pod": "ds-idrepo-2" 05:33:57 DEBUG }, 05:33:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501237" 05:33:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:33:57 DEBUG "value": [ 05:33:57 DEBUG ----- output ----- 05:33:57 DEBUG 1713501237, 05:33:57 DEBUG { 05:33:57 DEBUG "0" 05:33:57 DEBUG "status": "success", 05:33:57 DEBUG ] 05:33:57 DEBUG "data": { 05:33:57 DEBUG } 05:33:57 DEBUG "resultType": "vector", 05:33:57 DEBUG ] 05:33:57 DEBUG "result": [ 05:33:57 DEBUG } 05:33:57 DEBUG { 05:33:57 DEBUG } 05:33:57 DEBUG "metric": { 05:33:57 DEBUG "pod": "ds-idrepo-1" 05:33:57 DEBUG }, 05:33:57 DEBUG "value": [ 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG [...] 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 05:33:57 DEBUG }, 05:33:57 DEBUG "value": [ 05:33:57 DEBUG 1713501237, 05:33:57 DEBUG "0.00003109540636042897" 05:33:57 DEBUG ] 05:33:57 DEBUG } 05:33:57 DEBUG ] 05:33:57 DEBUG } 05:33:57 DEBUG } 05:33:57 INFO [http_cmd]: http status code OK 05:33:57 DEBUG --- status code --- 05:33:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:33:57 DEBUG --- http response --- 05:33:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:33:57 DEBUG ----- output ----- 05:33:57 DEBUG { 05:33:57 DEBUG "status": "success", 05:33:57 DEBUG "data": { 05:33:57 DEBUG "resultType": "vector", 05:33:57 DEBUG "result": [ 05:33:57 DEBUG { 05:33:57 DEBUG "metric": { 05:33:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:33:57 DEBUG }, 05:33:57 DEBUG "value": [ 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG [...] 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:33:57 DEBUG }, 05:33:57 DEBUG "value": [ 05:33:57 DEBUG 1713501237, 05:33:57 DEBUG "2161.8333333333335" 05:33:57 DEBUG ] 05:33:57 DEBUG } 05:33:57 DEBUG ] 05:33:57 DEBUG } 05:33:57 DEBUG } 05:33:57 INFO [http_cmd]: http status code OK 05:33:57 DEBUG --- status code --- 05:33:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:33:57 DEBUG --- http response --- 05:33:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:33:57 DEBUG ----- output ----- 05:33:57 DEBUG { 05:33:57 DEBUG "status": "success", 05:33:57 DEBUG "data": { 05:33:57 DEBUG "resultType": "vector", 05:33:57 DEBUG "result": [ 05:33:57 DEBUG { 05:33:57 DEBUG "metric": { 05:33:57 DEBUG "instance": "10.142.0.107:9100" 05:33:57 DEBUG }, 05:33:57 DEBUG "value": [ 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG [...] 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG "instance": "10.142.0.43:9100" 05:33:57 DEBUG }, 05:33:57 DEBUG "value": [ 05:33:57 DEBUG 1713501237, 05:33:57 DEBUG "0.3313333333333374" 05:33:57 DEBUG ] 05:33:57 DEBUG } 05:33:57 DEBUG ] 05:33:57 DEBUG } 05:33:57 DEBUG } 05:33:57 INFO [http_cmd]: http status code OK 05:33:57 DEBUG --- status code --- 05:33:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:33:57 DEBUG --- http response --- 05:33:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:33:57 DEBUG ----- output ----- 05:33:57 DEBUG { 05:33:57 DEBUG "status": "success", 05:33:57 DEBUG "data": { 05:33:57 DEBUG "resultType": "vector", 05:33:57 DEBUG "result": [ 05:33:57 DEBUG { 05:33:57 DEBUG "metric": { 05:33:57 DEBUG "instance": "10.142.0.107:9100" 05:33:57 DEBUG }, 05:33:57 DEBUG "value": [ 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG [...] 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG "instance": "10.142.0.43:9100" 05:33:57 DEBUG }, 05:33:57 DEBUG "value": [ 05:33:57 DEBUG 1713501237, 05:33:57 DEBUG "64121242.2" 05:33:57 DEBUG ] 05:33:57 DEBUG } 05:33:57 DEBUG ] 05:33:57 DEBUG } 05:33:57 DEBUG } 05:33:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501237" 05:33:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501237" 05:33:57 INFO [http_cmd]: http status code OK 05:33:57 DEBUG --- status code --- 05:33:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:33:57 DEBUG --- http response --- 05:33:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:33:57 DEBUG ----- output ----- 05:33:57 DEBUG { 05:33:57 DEBUG "status": "success", 05:33:57 DEBUG "data": { 05:33:57 DEBUG "resultType": "vector", 05:33:57 DEBUG "result": [ 05:33:57 DEBUG { 05:33:57 DEBUG "metric": { 05:33:57 DEBUG "instance": "10.142.0.107:9100" 05:33:57 DEBUG }, 05:33:57 DEBUG "value": [ 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG [...] 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG "instance": "10.142.0.43:9100" 05:33:57 DEBUG }, 05:33:57 DEBUG "value": [ 05:33:57 DEBUG 1713501237, 05:33:57 DEBUG "64114434.66666667" 05:33:57 DEBUG ] 05:33:57 DEBUG } 05:33:57 DEBUG ] 05:33:57 DEBUG } 05:33:57 DEBUG } 05:33:57 INFO [http_cmd]: http status code OK 05:33:57 DEBUG --- status code --- 05:33:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:33:57 DEBUG --- http response --- 05:33:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:33:57 DEBUG ----- output ----- 05:33:57 DEBUG { 05:33:57 DEBUG "status": "success", 05:33:57 DEBUG "data": { 05:33:57 DEBUG "resultType": "vector", 05:33:57 DEBUG "result": [ 05:33:57 DEBUG { 05:33:57 DEBUG "metric": { 05:33:57 DEBUG "pod": "ds-idrepo-1" 05:33:57 DEBUG }, 05:33:57 DEBUG "value": [ 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG [...] 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 05:33:57 DEBUG }, 05:33:57 DEBUG "value": [ 05:33:57 DEBUG 1713501237, 05:33:57 DEBUG "0" 05:33:57 DEBUG ] 05:33:57 DEBUG } 05:33:57 DEBUG ] 05:33:57 DEBUG } 05:33:57 DEBUG } 05:33:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501237" 05:33:57 INFO [http_cmd]: http status code OK 05:33:57 DEBUG --- status code --- 05:33:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:33:57 DEBUG --- http response --- 05:33:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:33:57 DEBUG ----- output ----- 05:33:57 DEBUG { 05:33:57 DEBUG "status": "success", 05:33:57 DEBUG "data": { 05:33:57 DEBUG "resultType": "vector", 05:33:57 DEBUG "result": [ 05:33:57 DEBUG { 05:33:57 DEBUG "metric": { 05:33:57 DEBUG "pod": "ds-idrepo-1" 05:33:57 DEBUG }, 05:33:57 DEBUG "value": [ 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG [...] 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG 05:33:57 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 05:33:57 DEBUG }, 05:33:57 DEBUG "value": [ 05:33:57 DEBUG 1713501237, 05:33:57 DEBUG "0" 05:33:57 DEBUG ] 05:33:57 DEBUG } 05:33:57 DEBUG ] 05:33:57 DEBUG } 05:33:57 DEBUG } 05:33:59 INFO 05:33:59 INFO [loop_until]: kubectl --namespace=xlou top pods 05:33:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:33:59 INFO [loop_until]: OK (rc = 0) 05:33:59 DEBUG --- stdout --- 05:33:59 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 13708m 9138Mi am-6fd554f8f7-hvvkw 13778m 8906Mi am-6fd554f8f7-pmj4b 14598m 8794Mi ds-cts-0 19244m 12923Mi ds-cts-1 19300m 13046Mi ds-cts-2 19661m 12920Mi ds-idrepo-0 5900m 3267Mi ds-idrepo-1 5848m 3199Mi ds-idrepo-2 5727m 3065Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 4m 1215Mi lodemon-6b89d774d5-rf7sk 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7099m 2038Mi 05:33:59 DEBUG --- stderr --- 05:33:59 DEBUG 05:33:59 INFO 05:33:59 INFO [loop_until]: kubectl --namespace=xlou top node 05:33:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:33:59 INFO [loop_until]: OK (rc = 0) 05:33:59 DEBUG --- stdout --- 05:33:59 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 6242Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5831m 19% 6712Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13247m 44% 11615Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 82m 0% 3243Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 19563m 65% 15497Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 131m 0% 5163Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 13744m 46% 11934Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 11308m 37% 2729Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 15373m 51% 10987Mi 9% gke-xlou-cdm-ds-0f601751-j488 6184m 20% 6166Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 18754m 62% 17052Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6158m 20% 6891Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 19925m 66% 14904Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 7278m 45% 5744Mi 9% 05:33:59 DEBUG --- stderr --- 05:33:59 DEBUG 05:34:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713501297" 05:34:57 INFO [http_cmd]: http status code OK 05:34:57 DEBUG --- status code --- 05:34:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:34:57 DEBUG --- http response --- 05:34:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:34:57 DEBUG ----- output ----- 05:34:57 DEBUG { 05:34:57 DEBUG "status": "success", 05:34:57 DEBUG "data": { 05:34:57 DEBUG "resultType": "vector", 05:34:57 DEBUG "result": [ 05:34:57 DEBUG { 05:34:57 DEBUG "metric": { 05:34:57 DEBUG "pod": "ds-idrepo-0" 05:34:57 DEBUG }, 05:34:57 DEBUG "value": [ 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG [...] 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG "pod": "ds-cts-2" 05:34:57 DEBUG }, 05:34:57 DEBUG "value": [ 05:34:57 DEBUG 1713501297, 05:34:57 DEBUG "0" 05:34:57 DEBUG ] 05:34:57 DEBUG } 05:34:57 DEBUG ] 05:34:57 DEBUG } 05:34:57 DEBUG } 05:34:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501297" 05:34:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713501297" 05:34:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501297" 05:34:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501297" 05:34:57 INFO [http_cmd]: http status code OK 05:34:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501297" 05:34:57 DEBUG --- status code --- 05:34:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:34:57 DEBUG --- http response --- 05:34:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 05:34:57 DEBUG ----- output ----- 05:34:57 DEBUG { 05:34:57 DEBUG "status": "success", 05:34:57 DEBUG "data": { 05:34:57 DEBUG "resultType": "vector", 05:34:57 DEBUG "result": [ 05:34:57 DEBUG { 05:34:57 DEBUG "metric": { 05:34:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:34:57 DEBUG }, 05:34:57 DEBUG "value": [ 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG [...] 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:34:57 DEBUG }, 05:34:57 DEBUG "value": [ 05:34:57 DEBUG 1713501297, 05:34:57 DEBUG "67487904.07371178" 05:34:57 DEBUG ] 05:34:57 DEBUG } 05:34:57 DEBUG ] 05:34:57 DEBUG } 05:34:57 DEBUG } 05:34:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501297" 05:34:57 ERROR [http_cmd]: ERROR 05:34:57 ERROR --- status code --- 05:34:57 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:34:57 ERROR --- http response --- 05:34:57 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:34:57 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:34:57 INFO [http_cmd]: http status code OK 05:34:57 INFO [http_cmd]: http status code OK 05:34:57 DEBUG --- status code --- 05:34:57 DEBUG --- status code --- 05:34:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:34:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:34:57 DEBUG --- http response --- 05:34:57 DEBUG --- http response --- 05:34:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:34:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:34:57 DEBUG ----- output ----- 05:34:57 DEBUG ----- output ----- 05:34:57 DEBUG { 05:34:57 DEBUG { 05:34:57 DEBUG "status": "success", 05:34:57 DEBUG "status": "success", 05:34:57 DEBUG "data": { 05:34:57 DEBUG "data": { 05:34:57 DEBUG "resultType": "vector", 05:34:57 DEBUG "resultType": "vector", 05:34:57 DEBUG "result": [ 05:34:57 DEBUG "result": [ 05:34:57 DEBUG { 05:34:57 DEBUG { 05:34:57 DEBUG "metric": { 05:34:57 DEBUG "metric": { 05:34:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:34:57 INFO [http_cmd]: http status code OK 05:34:57 INFO [http_cmd]: http status code OK 05:34:57 DEBUG "instance": "10.142.0.107:9100" 05:34:57 DEBUG }, 05:34:57 DEBUG --- status code --- 05:34:57 DEBUG --- status code --- 05:34:57 DEBUG }, 05:34:57 DEBUG "value": [ 05:34:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:34:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:34:57 DEBUG "value": [ 05:34:57 DEBUG 05:34:57 DEBUG --- http response --- 05:34:57 DEBUG --- http response --- 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:34:57 DEBUG 05:34:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:34:57 DEBUG 05:34:57 DEBUG ----- output ----- 05:34:57 DEBUG 05:34:57 DEBUG ----- output ----- 05:34:57 DEBUG [...] 05:34:57 DEBUG { 05:34:57 DEBUG [...] 05:34:57 DEBUG { 05:34:57 DEBUG 05:34:57 DEBUG "status": "success", 05:34:57 DEBUG 05:34:57 DEBUG "status": "success", 05:34:57 DEBUG 05:34:57 DEBUG "data": { 05:34:57 DEBUG 05:34:57 DEBUG "data": { 05:34:57 DEBUG 05:34:57 DEBUG "resultType": "vector", 05:34:57 DEBUG 05:34:57 DEBUG "resultType": "vector", 05:34:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:34:57 DEBUG "result": [ 05:34:57 DEBUG "instance": "10.142.0.43:9100" 05:34:57 DEBUG "result": [ 05:34:57 DEBUG }, 05:34:57 DEBUG { 05:34:57 DEBUG }, 05:34:57 DEBUG { 05:34:57 DEBUG "value": [ 05:34:57 DEBUG "metric": { 05:34:57 DEBUG "value": [ 05:34:57 DEBUG "metric": { 05:34:57 DEBUG 1713501297, 05:34:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:34:57 DEBUG 1713501297, 05:34:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:34:57 DEBUG "2171.9333333333334" 05:34:57 DEBUG }, 05:34:57 DEBUG "523519697920" 05:34:57 DEBUG }, 05:34:57 DEBUG ] 05:34:57 DEBUG "value": [ 05:34:57 DEBUG ] 05:34:57 DEBUG "value": [ 05:34:57 DEBUG } 05:34:57 DEBUG 05:34:57 DEBUG } 05:34:57 DEBUG 05:34:57 DEBUG ] 05:34:57 DEBUG 05:34:57 DEBUG ] 05:34:57 DEBUG 05:34:57 DEBUG } 05:34:57 DEBUG 05:34:57 DEBUG } 05:34:57 DEBUG 05:34:57 DEBUG } 05:34:57 DEBUG [...] 05:34:57 DEBUG } 05:34:57 DEBUG [...] 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:34:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:34:57 DEBUG }, 05:34:57 DEBUG }, 05:34:57 DEBUG "value": [ 05:34:57 DEBUG "value": [ 05:34:57 DEBUG 1713501297, 05:34:57 DEBUG 1713501297, 05:34:57 DEBUG "3673017" 05:34:57 DEBUG "0" 05:34:57 DEBUG ] 05:34:57 DEBUG ] 05:34:57 DEBUG } 05:34:57 DEBUG } 05:34:57 DEBUG ] 05:34:57 DEBUG ] 05:34:57 DEBUG } 05:34:57 DEBUG } 05:34:57 DEBUG } 05:34:57 DEBUG } 05:34:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501297" 05:34:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713501297" 05:34:57 INFO [http_cmd]: http status code OK 05:34:57 DEBUG --- status code --- 05:34:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:34:57 DEBUG --- http response --- 05:34:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:34:57 INFO [http_cmd]: http status code OK 05:34:57 DEBUG --- status code --- 05:34:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:34:57 DEBUG --- http response --- 05:34:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:34:57 DEBUG ----- output ----- 05:34:57 DEBUG { 05:34:57 DEBUG "status": "success", 05:34:57 DEBUG "data": { 05:34:57 DEBUG "resultType": "vector", 05:34:57 DEBUG "result": [ 05:34:57 DEBUG { 05:34:57 DEBUG "metric": { 05:34:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:34:57 DEBUG }, 05:34:57 DEBUG "value": [ 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG [...] 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:34:57 DEBUG }, 05:34:57 DEBUG "value": [ 05:34:57 DEBUG 1713501297, 05:34:57 DEBUG "55.174499999999995" 05:34:57 DEBUG ] 05:34:57 DEBUG } 05:34:57 DEBUG ] 05:34:57 DEBUG } 05:34:57 DEBUG } 05:34:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501297" 05:34:57 INFO [http_cmd]: http status code OK 05:34:57 DEBUG --- status code --- 05:34:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:34:57 DEBUG --- http response --- 05:34:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:34:57 DEBUG ----- output ----- 05:34:57 DEBUG { 05:34:57 DEBUG "status": "success", 05:34:57 DEBUG "data": { 05:34:57 DEBUG "resultType": "vector", 05:34:57 DEBUG "result": [ 05:34:57 DEBUG { 05:34:57 DEBUG "metric": { 05:34:57 DEBUG "pod": "ds-cts-0" 05:34:57 DEBUG }, 05:34:57 DEBUG "value": [ 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG [...] 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG "pod": "ds-cts-2" 05:34:57 DEBUG }, 05:34:57 DEBUG "value": [ 05:34:57 DEBUG 1713501297, 05:34:57 DEBUG "224.43333333333334" 05:34:57 DEBUG ] 05:34:57 DEBUG } 05:34:57 DEBUG ] 05:34:57 DEBUG } 05:34:57 DEBUG } 05:34:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713501297" 05:34:57 INFO [http_cmd]: http status code OK 05:34:57 DEBUG --- status code --- 05:34:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:34:57 DEBUG --- http response --- 05:34:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:34:57 DEBUG ----- output ----- 05:34:57 DEBUG { 05:34:57 DEBUG "status": "success", 05:34:57 DEBUG "data": { 05:34:57 DEBUG "resultType": "vector", 05:34:57 DEBUG "result": [ 05:34:57 DEBUG { 05:34:57 DEBUG "metric": { 05:34:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:34:57 DEBUG }, 05:34:57 DEBUG "value": [ 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG [...] 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:34:57 DEBUG }, 05:34:57 DEBUG "value": [ 05:34:57 DEBUG 1713501297, 05:34:57 DEBUG "7.098065916945318" 05:34:57 DEBUG ] 05:34:57 DEBUG } 05:34:57 DEBUG ] 05:34:57 DEBUG } 05:34:57 DEBUG } 05:34:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501297" 05:34:57 INFO [http_cmd]: http status code OK 05:34:57 DEBUG --- status code --- 05:34:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:34:57 DEBUG --- http response --- 05:34:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:34:57 DEBUG ----- output ----- 05:34:57 DEBUG { 05:34:57 DEBUG "status": "success", 05:34:57 DEBUG "data": { 05:34:57 DEBUG "resultType": "vector", 05:34:57 DEBUG "result": [ 05:34:57 DEBUG { 05:34:57 DEBUG "metric": { 05:34:57 DEBUG "pod": "ds-cts-0" 05:34:57 DEBUG }, 05:34:57 DEBUG "value": [ 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG [...] 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG "pod": "ds-cts-2" 05:34:57 DEBUG }, 05:34:57 DEBUG "value": [ 05:34:57 DEBUG 1713501297, 05:34:57 DEBUG "0.001029462222222222" 05:34:57 DEBUG ] 05:34:57 DEBUG } 05:34:57 DEBUG ] 05:34:57 DEBUG } 05:34:57 DEBUG } 05:34:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501297" 05:34:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501297" 05:34:57 INFO [http_cmd]: http status code OK 05:34:57 DEBUG --- status code --- 05:34:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:34:57 DEBUG --- http response --- 05:34:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:34:57 DEBUG ----- output ----- 05:34:57 DEBUG { 05:34:57 DEBUG "status": "success", 05:34:57 DEBUG "data": { 05:34:57 DEBUG "resultType": "vector", 05:34:57 DEBUG "result": [ 05:34:57 DEBUG { 05:34:57 DEBUG "metric": { 05:34:57 DEBUG "pod": "ds-cts-0" 05:34:57 DEBUG }, 05:34:57 DEBUG "value": [ 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG [...] 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG "pod": "ds-cts-2" 05:34:57 DEBUG }, 05:34:57 DEBUG "value": [ 05:34:57 DEBUG 1713501297, 05:34:57 DEBUG "30" 05:34:57 DEBUG ] 05:34:57 DEBUG } 05:34:57 DEBUG ] 05:34:57 DEBUG } 05:34:57 DEBUG } 05:34:57 INFO [http_cmd]: http status code OK 05:34:57 DEBUG --- status code --- 05:34:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:34:57 DEBUG --- http response --- 05:34:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:34:57 DEBUG ----- output ----- 05:34:57 DEBUG { 05:34:57 DEBUG "status": "success", 05:34:57 DEBUG "data": { 05:34:57 DEBUG "resultType": "vector", 05:34:57 DEBUG "result": [ 05:34:57 DEBUG { 05:34:57 DEBUG "metric": { 05:34:57 DEBUG "instance": "10.142.0.107:9100" 05:34:57 DEBUG }, 05:34:57 DEBUG "value": [ 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG [...] 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG "instance": "10.142.0.43:9100" 05:34:57 DEBUG }, 05:34:57 DEBUG "value": [ 05:34:57 DEBUG 1713501297, 05:34:57 DEBUG "0.43676666666668024" 05:34:57 DEBUG ] 05:34:57 DEBUG } 05:34:57 DEBUG ] 05:34:57 DEBUG } 05:34:57 DEBUG } 05:34:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501297" 05:34:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501297" 05:34:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501297" 05:34:57 INFO [http_cmd]: http status code OK 05:34:57 DEBUG --- status code --- 05:34:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:34:57 DEBUG --- http response --- 05:34:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:34:57 DEBUG ----- output ----- 05:34:57 DEBUG { 05:34:57 DEBUG "status": "success", 05:34:57 DEBUG "data": { 05:34:57 DEBUG "resultType": "vector", 05:34:57 DEBUG "result": [ 05:34:57 DEBUG { 05:34:57 DEBUG "metric": { 05:34:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:34:57 DEBUG }, 05:34:57 DEBUG "value": [ 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG [...] 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:34:57 DEBUG }, 05:34:57 DEBUG "value": [ 05:34:57 DEBUG 1713501297, 05:34:57 DEBUG "2004.0333333333333" 05:34:57 DEBUG ] 05:34:57 DEBUG } 05:34:57 DEBUG ] 05:34:57 DEBUG } 05:34:57 DEBUG } 05:34:57 INFO [http_cmd]: http status code OK 05:34:57 DEBUG --- status code --- 05:34:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501297" 05:34:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:34:57 DEBUG --- http response --- 05:34:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 05:34:57 DEBUG ----- output ----- 05:34:57 DEBUG { 05:34:57 DEBUG "status": "success", 05:34:57 DEBUG "data": { 05:34:57 DEBUG "resultType": "vector", 05:34:57 DEBUG "result": [ 05:34:57 DEBUG { 05:34:57 DEBUG "metric": { 05:34:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:34:57 DEBUG }, 05:34:57 DEBUG "value": [ 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG [...] 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:34:57 DEBUG }, 05:34:57 DEBUG "value": [ 05:34:57 DEBUG 1713501297, 05:34:57 DEBUG "7.06977150790448" 05:34:57 DEBUG ] 05:34:57 DEBUG } 05:34:57 DEBUG ] 05:34:57 DEBUG } 05:34:57 DEBUG } 05:34:57 INFO [http_cmd]: http status code OK 05:34:57 DEBUG --- status code --- 05:34:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:34:57 DEBUG --- http response --- 05:34:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:34:57 DEBUG ----- output ----- 05:34:57 DEBUG { 05:34:57 DEBUG "status": "success", 05:34:57 DEBUG "data": { 05:34:57 DEBUG "resultType": "vector", 05:34:57 DEBUG "result": [ 05:34:57 DEBUG { 05:34:57 DEBUG "metric": { 05:34:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:34:57 DEBUG }, 05:34:57 DEBUG "value": [ 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG [...] 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:34:57 DEBUG }, 05:34:57 DEBUG "value": [ 05:34:57 DEBUG 1713501297, 05:34:57 DEBUG "2172.366666666667" 05:34:57 DEBUG ] 05:34:57 DEBUG } 05:34:57 DEBUG ] 05:34:57 DEBUG } 05:34:57 DEBUG } 05:34:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501297" 05:34:57 INFO [http_cmd]: http status code OK 05:34:57 DEBUG --- status code --- 05:34:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:34:57 DEBUG --- http response --- 05:34:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:34:57 DEBUG ----- output ----- 05:34:57 DEBUG { 05:34:57 DEBUG "status": "success", 05:34:57 DEBUG "data": { 05:34:57 DEBUG "resultType": "vector", 05:34:57 DEBUG "result": [ 05:34:57 DEBUG { 05:34:57 DEBUG "metric": { 05:34:57 DEBUG "pod": "ds-idrepo-0" 05:34:57 DEBUG }, 05:34:57 DEBUG "value": [ 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG [...] 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG "pod": "ds-idrepo-2" 05:34:57 DEBUG }, 05:34:57 DEBUG "value": [ 05:34:57 DEBUG 1713501297, 05:34:57 DEBUG "0" 05:34:57 DEBUG ] 05:34:57 DEBUG } 05:34:57 DEBUG ] 05:34:57 DEBUG } 05:34:57 DEBUG } 05:34:57 INFO [http_cmd]: http status code OK 05:34:57 DEBUG --- status code --- 05:34:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:34:57 DEBUG --- http response --- 05:34:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713501297" 05:34:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:34:57 DEBUG ----- output ----- 05:34:57 DEBUG { 05:34:57 DEBUG "status": "success", 05:34:57 DEBUG "data": { 05:34:57 DEBUG "resultType": "vector", 05:34:57 DEBUG "result": [ 05:34:57 DEBUG { 05:34:57 DEBUG "metric": { 05:34:57 DEBUG "pod": "ds-idrepo-0" 05:34:57 DEBUG }, 05:34:57 DEBUG "value": [ 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG [...] 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG "pod": "ds-idrepo-2" 05:34:57 DEBUG }, 05:34:57 DEBUG "value": [ 05:34:57 DEBUG 1713501297, 05:34:57 DEBUG "0" 05:34:57 DEBUG ] 05:34:57 DEBUG } 05:34:57 DEBUG ] 05:34:57 DEBUG } 05:34:57 DEBUG } 05:34:57 INFO [http_cmd]: http status code OK 05:34:57 DEBUG --- status code --- 05:34:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:34:57 DEBUG --- http response --- 05:34:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:34:57 DEBUG ----- output ----- 05:34:57 DEBUG { 05:34:57 DEBUG "status": "success", 05:34:57 DEBUG "data": { 05:34:57 DEBUG "resultType": "vector", 05:34:57 DEBUG "result": [ 05:34:57 DEBUG { 05:34:57 DEBUG "metric": { 05:34:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:34:57 DEBUG }, 05:34:57 DEBUG "value": [ 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG [...] 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:34:57 DEBUG }, 05:34:57 DEBUG "value": [ 05:34:57 DEBUG 1713501297, 05:34:57 DEBUG "2234802176" 05:34:57 DEBUG ] 05:34:57 DEBUG } 05:34:57 DEBUG ] 05:34:57 DEBUG } 05:34:57 DEBUG } 05:34:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501297" 05:34:57 INFO [http_cmd]: http status code OK 05:34:57 DEBUG --- status code --- 05:34:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:34:57 DEBUG --- http response --- 05:34:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:34:57 DEBUG ----- output ----- 05:34:57 DEBUG { 05:34:57 DEBUG "status": "success", 05:34:57 DEBUG "data": { 05:34:57 DEBUG "resultType": "vector", 05:34:57 DEBUG "result": [ 05:34:57 DEBUG { 05:34:57 DEBUG "metric": { 05:34:57 DEBUG "pod": "ds-idrepo-0" 05:34:57 DEBUG }, 05:34:57 DEBUG "value": [ 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG [...] 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG "pod": "ds-idrepo-2" 05:34:57 DEBUG }, 05:34:57 DEBUG "value": [ 05:34:57 DEBUG 1713501297, 05:34:57 DEBUG "0" 05:34:57 DEBUG ] 05:34:57 DEBUG } 05:34:57 DEBUG ] 05:34:57 DEBUG } 05:34:57 DEBUG } 05:34:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501297" 05:34:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501297" 05:34:57 INFO [http_cmd]: http status code OK 05:34:57 DEBUG --- status code --- 05:34:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:34:57 DEBUG --- http response --- 05:34:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:34:57 DEBUG ----- output ----- 05:34:57 DEBUG { 05:34:57 DEBUG "status": "success", 05:34:57 DEBUG "data": { 05:34:57 DEBUG "resultType": "vector", 05:34:57 DEBUG "result": [ 05:34:57 DEBUG { 05:34:57 DEBUG "metric": { 05:34:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:34:57 DEBUG }, 05:34:57 DEBUG "value": [ 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG [...] 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:34:57 DEBUG }, 05:34:57 DEBUG "value": [ 05:34:57 DEBUG 1713501297, 05:34:57 DEBUG "2172.133333333333" 05:34:57 DEBUG ] 05:34:57 DEBUG } 05:34:57 DEBUG ] 05:34:57 DEBUG } 05:34:57 DEBUG } 05:34:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501297" 05:34:57 INFO [http_cmd]: http status code OK 05:34:57 DEBUG --- status code --- 05:34:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:34:57 DEBUG --- http response --- 05:34:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:34:57 DEBUG ----- output ----- 05:34:57 DEBUG { 05:34:57 DEBUG "status": "success", 05:34:57 DEBUG "data": { 05:34:57 DEBUG "resultType": "vector", 05:34:57 DEBUG "result": [ 05:34:57 DEBUG { 05:34:57 DEBUG "metric": { 05:34:57 DEBUG "instance": "10.142.0.107:9100" 05:34:57 DEBUG }, 05:34:57 DEBUG "value": [ 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG [...] 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG "instance": "10.142.0.43:9100" 05:34:57 DEBUG }, 05:34:57 DEBUG "value": [ 05:34:57 DEBUG 1713501297, 05:34:57 DEBUG "0.21933333333333654" 05:34:57 DEBUG ] 05:34:57 DEBUG } 05:34:57 DEBUG ] 05:34:57 DEBUG } 05:34:57 DEBUG } 05:34:57 INFO [http_cmd]: http status code OK 05:34:57 DEBUG --- status code --- 05:34:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:34:57 DEBUG --- http response --- 05:34:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:34:57 DEBUG ----- output ----- 05:34:57 DEBUG { 05:34:57 DEBUG "status": "success", 05:34:57 DEBUG "data": { 05:34:57 DEBUG "resultType": "vector", 05:34:57 DEBUG "result": [ 05:34:57 DEBUG { 05:34:57 DEBUG "metric": { 05:34:57 DEBUG "instance": "10.142.0.107:9100" 05:34:57 DEBUG }, 05:34:57 DEBUG "value": [ 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG [...] 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG "instance": "10.142.0.43:9100" 05:34:57 DEBUG }, 05:34:57 DEBUG "value": [ 05:34:57 DEBUG 1713501297, 05:34:57 DEBUG "64005794.4" 05:34:57 DEBUG ] 05:34:57 DEBUG } 05:34:57 DEBUG ] 05:34:57 DEBUG } 05:34:57 DEBUG } 05:34:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501297" 05:34:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501297" 05:34:57 INFO [http_cmd]: http status code OK 05:34:57 DEBUG --- status code --- 05:34:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:34:57 DEBUG --- http response --- 05:34:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:34:57 DEBUG ----- output ----- 05:34:57 DEBUG { 05:34:57 DEBUG "status": "success", 05:34:57 DEBUG "data": { 05:34:57 DEBUG "resultType": "vector", 05:34:57 DEBUG "result": [ 05:34:57 DEBUG { 05:34:57 DEBUG "metric": { 05:34:57 DEBUG "instance": "10.142.0.107:9100" 05:34:57 DEBUG }, 05:34:57 DEBUG "value": [ 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG [...] 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG "instance": "10.142.0.43:9100" 05:34:57 DEBUG }, 05:34:57 DEBUG "value": [ 05:34:57 DEBUG 1713501297, 05:34:57 DEBUG "63998694.233333334" 05:34:57 DEBUG ] 05:34:57 DEBUG } 05:34:57 DEBUG ] 05:34:57 DEBUG } 05:34:57 DEBUG } 05:34:57 INFO [http_cmd]: http status code OK 05:34:57 DEBUG --- status code --- 05:34:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:34:57 DEBUG --- http response --- 05:34:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 05:34:57 DEBUG ----- output ----- 05:34:57 DEBUG { 05:34:57 DEBUG "status": "success", 05:34:57 DEBUG "data": { 05:34:57 DEBUG "resultType": "vector", 05:34:57 DEBUG "result": [ 05:34:57 DEBUG { 05:34:57 DEBUG "metric": { 05:34:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:34:57 DEBUG }, 05:34:57 DEBUG "value": [ 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG [...] 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:34:57 DEBUG }, 05:34:57 DEBUG "value": [ 05:34:57 DEBUG 1713501297, 05:34:57 DEBUG "0" 05:34:57 DEBUG ] 05:34:57 DEBUG } 05:34:57 DEBUG ] 05:34:57 DEBUG } 05:34:57 DEBUG } 05:34:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501297" 05:34:57 INFO [http_cmd]: http status code OK 05:34:57 DEBUG --- status code --- 05:34:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:34:57 DEBUG --- http response --- 05:34:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 05:34:57 DEBUG ----- output ----- 05:34:57 DEBUG { 05:34:57 DEBUG "status": "success", 05:34:57 DEBUG "data": { 05:34:57 DEBUG "resultType": "vector", 05:34:57 DEBUG "result": [ 05:34:57 DEBUG { 05:34:57 DEBUG "metric": { 05:34:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:34:57 DEBUG }, 05:34:57 DEBUG "value": [ 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG [...] 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG 05:34:57 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:34:57 DEBUG }, 05:34:57 DEBUG "value": [ 05:34:57 DEBUG 1713501297, 05:34:57 DEBUG "0" 05:34:57 DEBUG ] 05:34:57 DEBUG } 05:34:57 DEBUG ] 05:34:57 DEBUG } 05:34:57 DEBUG } 05:34:59 INFO 05:34:59 INFO [loop_until]: kubectl --namespace=xlou top pods 05:34:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:34:59 INFO [loop_until]: OK (rc = 0) 05:34:59 DEBUG --- stdout --- 05:34:59 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 13774m 9139Mi am-6fd554f8f7-hvvkw 13207m 8908Mi am-6fd554f8f7-pmj4b 14480m 8794Mi ds-cts-0 19513m 12945Mi ds-cts-1 18741m 13006Mi ds-cts-2 21080m 12863Mi ds-idrepo-0 5897m 3267Mi ds-idrepo-1 5648m 3199Mi ds-idrepo-2 5648m 3066Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1171Mi idm-b85f9d978-wnhxb 4m 1215Mi lodemon-6b89d774d5-rf7sk 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7113m 2040Mi 05:34:59 DEBUG --- stderr --- 05:34:59 DEBUG 05:35:00 INFO 05:35:00 INFO [loop_until]: kubectl --namespace=xlou top node 05:35:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:35:00 INFO [loop_until]: OK (rc = 0) 05:35:00 DEBUG --- stdout --- 05:35:00 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 71m 0% 6248Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5820m 19% 6719Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13255m 44% 11617Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 84m 0% 3236Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 19749m 66% 15432Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 155m 0% 5158Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 13746m 46% 11917Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 11296m 37% 2723Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 14735m 49% 10986Mi 9% gke-xlou-cdm-ds-0f601751-j488 6184m 20% 6177Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 18552m 62% 16958Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6343m 21% 6883Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 19031m 63% 14957Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 7262m 45% 5742Mi 9% 05:35:00 DEBUG --- stderr --- 05:35:00 DEBUG 05:35:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713501357" 05:35:57 INFO [http_cmd]: http status code OK 05:35:57 DEBUG --- status code --- 05:35:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:35:57 DEBUG --- http response --- 05:35:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:35:57 DEBUG ----- output ----- 05:35:57 DEBUG { 05:35:57 DEBUG "status": "success", 05:35:57 DEBUG "data": { 05:35:57 DEBUG "resultType": "vector", 05:35:57 DEBUG "result": [ 05:35:57 DEBUG { 05:35:57 DEBUG "metric": { 05:35:57 DEBUG "pod": "ds-idrepo-0" 05:35:57 DEBUG }, 05:35:57 DEBUG "value": [ 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG [...] 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG "pod": "ds-cts-2" 05:35:57 DEBUG }, 05:35:57 DEBUG "value": [ 05:35:57 DEBUG 1713501357, 05:35:57 DEBUG "0" 05:35:57 DEBUG ] 05:35:57 DEBUG } 05:35:57 DEBUG ] 05:35:57 DEBUG } 05:35:57 DEBUG } 05:35:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501357" 05:35:57 INFO [http_cmd]: http status code OK 05:35:57 DEBUG --- status code --- 05:35:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:35:57 DEBUG --- http response --- 05:35:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:35:57 DEBUG ----- output ----- 05:35:57 DEBUG { 05:35:57 DEBUG "status": "success", 05:35:57 DEBUG "data": { 05:35:57 DEBUG "resultType": "vector", 05:35:57 DEBUG "result": [ 05:35:57 DEBUG { 05:35:57 DEBUG "metric": { 05:35:57 DEBUG "instance": "10.142.0.107:9100" 05:35:57 DEBUG }, 05:35:57 DEBUG "value": [ 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG [...] 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG "instance": "10.142.0.43:9100" 05:35:57 DEBUG }, 05:35:57 DEBUG "value": [ 05:35:57 DEBUG 1713501357, 05:35:57 DEBUG "528008365056" 05:35:57 DEBUG ] 05:35:57 DEBUG } 05:35:57 DEBUG ] 05:35:57 DEBUG } 05:35:57 DEBUG } 05:35:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501357" 05:35:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713501357" 05:35:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501357" 05:35:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501357" 05:35:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501357" 05:35:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501357" 05:35:57 INFO [http_cmd]: http status code OK 05:35:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501357" 05:35:57 DEBUG --- status code --- 05:35:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:35:57 DEBUG --- http response --- 05:35:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:35:57 DEBUG ----- output ----- 05:35:57 DEBUG { 05:35:57 DEBUG "status": "success", 05:35:57 DEBUG "data": { 05:35:57 DEBUG "resultType": "vector", 05:35:57 DEBUG "result": [ 05:35:57 DEBUG { 05:35:57 DEBUG "metric": { 05:35:57 DEBUG "pod": "ds-idrepo-0" 05:35:57 DEBUG }, 05:35:57 DEBUG "value": [ 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG [...] 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:35:57 DEBUG }, 05:35:57 DEBUG "value": [ 05:35:57 DEBUG 1713501357, 05:35:57 DEBUG "0" 05:35:57 DEBUG ] 05:35:57 DEBUG } 05:35:57 DEBUG ] 05:35:57 DEBUG } 05:35:57 DEBUG } 05:35:57 INFO [http_cmd]: http status code OK 05:35:57 DEBUG --- status code --- 05:35:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:35:57 DEBUG --- http response --- 05:35:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:35:57 DEBUG ----- output ----- 05:35:57 DEBUG { 05:35:57 DEBUG "status": "success", 05:35:57 DEBUG "data": { 05:35:57 DEBUG "resultType": "vector", 05:35:57 DEBUG "result": [ 05:35:57 DEBUG { 05:35:57 DEBUG "metric": { 05:35:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:35:57 DEBUG }, 05:35:57 DEBUG "value": [ 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG [...] 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:35:57 DEBUG }, 05:35:57 DEBUG "value": [ 05:35:57 DEBUG 1713501357, 05:35:57 DEBUG "0" 05:35:57 DEBUG ] 05:35:57 DEBUG } 05:35:57 DEBUG ] 05:35:57 DEBUG } 05:35:57 DEBUG } 05:35:57 INFO [http_cmd]: http status code OK 05:35:57 INFO [http_cmd]: http status code OK 05:35:57 ERROR [http_cmd]: ERROR 05:35:57 DEBUG --- status code --- 05:35:57 DEBUG --- status code --- 05:35:57 ERROR --- status code --- 05:35:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:35:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:35:57 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:35:57 DEBUG --- http response --- 05:35:57 DEBUG --- http response --- 05:35:57 ERROR --- http response --- 05:35:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:35:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:35:57 INFO [http_cmd]: http status code OK 05:35:57 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:35:57 DEBUG ----- output ----- 05:35:57 DEBUG ----- output ----- 05:35:57 DEBUG --- status code --- 05:35:57 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:35:57 DEBUG { 05:35:57 DEBUG { 05:35:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:35:57 DEBUG "status": "success", 05:35:57 DEBUG "status": "success", 05:35:57 DEBUG --- http response --- 05:35:57 DEBUG "data": { 05:35:57 DEBUG "data": { 05:35:57 DEBUG "resultType": "vector", 05:35:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:35:57 DEBUG "resultType": "vector", 05:35:57 DEBUG "result": [ 05:35:57 DEBUG ----- output ----- 05:35:57 DEBUG "result": [ 05:35:57 DEBUG { 05:35:57 DEBUG { 05:35:57 DEBUG { 05:35:57 DEBUG "metric": { 05:35:57 DEBUG "status": "success", 05:35:57 DEBUG "metric": { 05:35:57 DEBUG "pod": "ds-cts-0" 05:35:57 DEBUG "data": { 05:35:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:35:57 DEBUG }, 05:35:57 DEBUG "resultType": "vector", 05:35:57 DEBUG }, 05:35:57 DEBUG "value": [ 05:35:57 DEBUG "result": [ 05:35:57 DEBUG "value": [ 05:35:57 DEBUG 05:35:57 DEBUG { 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG "metric": { 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:35:57 DEBUG 05:35:57 DEBUG [...] 05:35:57 DEBUG }, 05:35:57 DEBUG [...] 05:35:57 INFO [http_cmd]: http status code OK 05:35:57 DEBUG 05:35:57 DEBUG "value": [ 05:35:57 DEBUG 05:35:57 DEBUG --- status code --- 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG --- http response --- 05:35:57 DEBUG "pod": "ds-cts-2" 05:35:57 DEBUG 05:35:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:35:57 DEBUG }, 05:35:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:35:57 DEBUG [...] 05:35:57 DEBUG }, 05:35:57 DEBUG "value": [ 05:35:57 DEBUG 05:35:57 DEBUG "value": [ 05:35:57 DEBUG 1713501357, 05:35:57 DEBUG 05:35:57 DEBUG 1713501357, 05:35:57 DEBUG "311.76666666666665" 05:35:57 DEBUG 05:35:57 DEBUG "3682940" 05:35:57 DEBUG ] 05:35:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:35:57 DEBUG ] 05:35:57 DEBUG } 05:35:57 DEBUG }, 05:35:57 DEBUG } 05:35:57 DEBUG ] 05:35:57 DEBUG "value": [ 05:35:57 DEBUG ] 05:35:57 DEBUG } 05:35:57 DEBUG 1713501357, 05:35:57 DEBUG } 05:35:57 DEBUG } 05:35:57 DEBUG "2164.7666666666664" 05:35:57 DEBUG } 05:35:57 DEBUG ] 05:35:57 DEBUG } 05:35:57 DEBUG ] 05:35:57 DEBUG } 05:35:57 DEBUG } 05:35:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713501357" 05:35:57 INFO [http_cmd]: http status code OK 05:35:57 DEBUG --- status code --- 05:35:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:35:57 DEBUG --- http response --- 05:35:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:35:57 DEBUG ----- output ----- 05:35:57 DEBUG { 05:35:57 DEBUG "status": "success", 05:35:57 DEBUG "data": { 05:35:57 DEBUG "resultType": "vector", 05:35:57 DEBUG "result": [ 05:35:57 DEBUG { 05:35:57 DEBUG "metric": { 05:35:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:35:57 DEBUG }, 05:35:57 DEBUG "value": [ 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG [...] 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:35:57 DEBUG }, 05:35:57 DEBUG "value": [ 05:35:57 DEBUG 1713501357, 05:35:57 DEBUG "55.175" 05:35:57 DEBUG ] 05:35:57 DEBUG } 05:35:57 DEBUG ] 05:35:57 DEBUG } 05:35:57 DEBUG } 05:35:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713501357" 05:35:57 INFO [http_cmd]: http status code OK 05:35:57 DEBUG --- status code --- 05:35:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:35:57 DEBUG --- http response --- 05:35:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:35:57 DEBUG ----- output ----- 05:35:57 DEBUG { 05:35:57 DEBUG "status": "success", 05:35:57 DEBUG "data": { 05:35:57 DEBUG "resultType": "vector", 05:35:57 DEBUG "result": [ 05:35:57 DEBUG { 05:35:57 DEBUG "metric": { 05:35:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:35:57 DEBUG }, 05:35:57 DEBUG "value": [ 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG [...] 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:35:57 DEBUG }, 05:35:57 DEBUG "value": [ 05:35:57 DEBUG 1713501357, 05:35:57 DEBUG "7.087363463469643" 05:35:57 DEBUG ] 05:35:57 DEBUG } 05:35:57 DEBUG ] 05:35:57 DEBUG } 05:35:57 DEBUG } 05:35:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501357" 05:35:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501357" 05:35:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501357" 05:35:57 INFO [http_cmd]: http status code OK 05:35:57 DEBUG --- status code --- 05:35:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:35:57 DEBUG --- http response --- 05:35:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:35:57 DEBUG ----- output ----- 05:35:57 DEBUG { 05:35:57 DEBUG "status": "success", 05:35:57 DEBUG "data": { 05:35:57 DEBUG "resultType": "vector", 05:35:57 DEBUG "result": [ 05:35:57 DEBUG { 05:35:57 DEBUG "metric": { 05:35:57 DEBUG "pod": "ds-cts-0" 05:35:57 DEBUG }, 05:35:57 DEBUG "value": [ 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG [...] 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG "pod": "ds-cts-2" 05:35:57 DEBUG }, 05:35:57 DEBUG "value": [ 05:35:57 DEBUG 1713501357, 05:35:57 DEBUG "0.0007333333333333333" 05:35:57 DEBUG ] 05:35:57 DEBUG } 05:35:57 DEBUG ] 05:35:57 DEBUG } 05:35:57 DEBUG } 05:35:57 INFO [http_cmd]: http status code OK 05:35:57 INFO [http_cmd]: http status code OK 05:35:57 DEBUG --- status code --- 05:35:57 DEBUG --- status code --- 05:35:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:35:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:35:57 DEBUG --- http response --- 05:35:57 DEBUG --- http response --- 05:35:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:35:57 DEBUG ----- output ----- 05:35:57 DEBUG { 05:35:57 DEBUG "status": "success", 05:35:57 DEBUG "data": { 05:35:57 DEBUG "resultType": "vector", 05:35:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:35:57 DEBUG "result": [ 05:35:57 DEBUG ----- output ----- 05:35:57 DEBUG { 05:35:57 DEBUG { 05:35:57 DEBUG "metric": { 05:35:57 DEBUG "status": "success", 05:35:57 DEBUG "pod": "ds-cts-0" 05:35:57 DEBUG "data": { 05:35:57 DEBUG }, 05:35:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501357" 05:35:57 DEBUG "resultType": "vector", 05:35:57 DEBUG "value": [ 05:35:57 DEBUG "result": [ 05:35:57 DEBUG { 05:35:57 DEBUG 05:35:57 DEBUG "metric": { 05:35:57 DEBUG 05:35:57 DEBUG "instance": "10.142.0.107:9100" 05:35:57 DEBUG 05:35:57 DEBUG }, 05:35:57 DEBUG [...] 05:35:57 DEBUG "value": [ 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG "pod": "ds-cts-2" 05:35:57 DEBUG [...] 05:35:57 DEBUG }, 05:35:57 DEBUG 05:35:57 DEBUG "value": [ 05:35:57 DEBUG 05:35:57 DEBUG 1713501357, 05:35:57 DEBUG 05:35:57 DEBUG "34.13333333333333" 05:35:57 DEBUG "instance": "10.142.0.43:9100" 05:35:57 DEBUG ] 05:35:57 DEBUG }, 05:35:57 DEBUG } 05:35:57 DEBUG "value": [ 05:35:57 DEBUG ] 05:35:57 DEBUG 1713501357, 05:35:57 DEBUG } 05:35:57 DEBUG "0.6450666666666668" 05:35:57 DEBUG } 05:35:57 DEBUG ] 05:35:57 DEBUG } 05:35:57 DEBUG ] 05:35:57 DEBUG } 05:35:57 DEBUG } 05:35:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501357" 05:35:57 INFO [http_cmd]: http status code OK 05:35:57 DEBUG --- status code --- 05:35:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:35:57 DEBUG --- http response --- 05:35:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:35:57 DEBUG ----- output ----- 05:35:57 DEBUG { 05:35:57 DEBUG "status": "success", 05:35:57 DEBUG "data": { 05:35:57 DEBUG "resultType": "vector", 05:35:57 DEBUG "result": [ 05:35:57 DEBUG { 05:35:57 DEBUG "metric": { 05:35:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:35:57 DEBUG }, 05:35:57 DEBUG "value": [ 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG [...] 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:35:57 DEBUG }, 05:35:57 DEBUG "value": [ 05:35:57 DEBUG 1713501357, 05:35:57 DEBUG "2001.9333333333334" 05:35:57 DEBUG ] 05:35:57 DEBUG } 05:35:57 DEBUG ] 05:35:57 DEBUG } 05:35:57 DEBUG } 05:35:57 INFO [http_cmd]: http status code OK 05:35:57 DEBUG --- status code --- 05:35:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:35:57 DEBUG --- http response --- 05:35:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:35:57 DEBUG ----- output ----- 05:35:57 DEBUG { 05:35:57 DEBUG "status": "success", 05:35:57 DEBUG "data": { 05:35:57 DEBUG "resultType": "vector", 05:35:57 DEBUG "result": [ 05:35:57 DEBUG { 05:35:57 DEBUG "metric": { 05:35:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:35:57 DEBUG }, 05:35:57 DEBUG "value": [ 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG [...] 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:35:57 DEBUG }, 05:35:57 DEBUG "value": [ 05:35:57 DEBUG 1713501357, 05:35:57 DEBUG "2164.366666666667" 05:35:57 DEBUG ] 05:35:57 DEBUG } 05:35:57 DEBUG ] 05:35:57 DEBUG } 05:35:57 DEBUG } 05:35:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501357" 05:35:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501357" 05:35:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501357" 05:35:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501357" 05:35:57 INFO [http_cmd]: http status code OK 05:35:57 DEBUG --- status code --- 05:35:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:35:57 DEBUG --- http response --- 05:35:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:35:57 DEBUG ----- output ----- 05:35:57 DEBUG { 05:35:57 DEBUG "status": "success", 05:35:57 DEBUG "data": { 05:35:57 DEBUG "resultType": "vector", 05:35:57 DEBUG "result": [ 05:35:57 DEBUG { 05:35:57 DEBUG "metric": { 05:35:57 DEBUG "pod": "ds-idrepo-0" 05:35:57 DEBUG }, 05:35:57 DEBUG "value": [ 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG [...] 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG "pod": "ds-idrepo-2" 05:35:57 DEBUG }, 05:35:57 DEBUG "value": [ 05:35:57 DEBUG 1713501357, 05:35:57 DEBUG "0" 05:35:57 DEBUG ] 05:35:57 DEBUG } 05:35:57 DEBUG ] 05:35:57 DEBUG } 05:35:57 DEBUG } 05:35:57 INFO [http_cmd]: http status code OK 05:35:57 DEBUG --- status code --- 05:35:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:35:57 DEBUG --- http response --- 05:35:57 INFO [http_cmd]: http status code OK 05:35:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:35:57 DEBUG --- status code --- 05:35:57 DEBUG ----- output ----- 05:35:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:35:57 DEBUG { 05:35:57 DEBUG --- http response --- 05:35:57 DEBUG "status": "success", 05:35:57 DEBUG "data": { 05:35:57 DEBUG "resultType": "vector", 05:35:57 DEBUG "result": [ 05:35:57 DEBUG { 05:35:57 DEBUG "metric": { 05:35:57 DEBUG "pod": "ds-idrepo-0" 05:35:57 DEBUG }, 05:35:57 DEBUG "value": [ 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG [...] 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:35:57 DEBUG }, 05:35:57 DEBUG "value": [ 05:35:57 DEBUG 1713501357, 05:35:57 DEBUG "0.000024517135982301468" 05:35:57 DEBUG ] 05:35:57 DEBUG } 05:35:57 DEBUG ] 05:35:57 DEBUG } 05:35:57 DEBUG } 05:35:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:35:57 DEBUG ----- output ----- 05:35:57 DEBUG { 05:35:57 DEBUG "status": "success", 05:35:57 DEBUG "data": { 05:35:57 DEBUG "resultType": "vector", 05:35:57 DEBUG "result": [ 05:35:57 DEBUG { 05:35:57 DEBUG "metric": { 05:35:57 DEBUG "pod": "ds-idrepo-0" 05:35:57 DEBUG }, 05:35:57 DEBUG "value": [ 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG [...] 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG "pod": "ds-idrepo-2" 05:35:57 DEBUG }, 05:35:57 DEBUG "value": [ 05:35:57 DEBUG 1713501357, 05:35:57 DEBUG "0" 05:35:57 DEBUG ] 05:35:57 DEBUG } 05:35:57 DEBUG ] 05:35:57 DEBUG } 05:35:57 DEBUG } 05:35:57 INFO [http_cmd]: http status code OK 05:35:57 DEBUG --- status code --- 05:35:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:35:57 DEBUG --- http response --- 05:35:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:35:57 DEBUG ----- output ----- 05:35:57 DEBUG { 05:35:57 DEBUG "status": "success", 05:35:57 DEBUG "data": { 05:35:57 DEBUG "resultType": "vector", 05:35:57 DEBUG "result": [ 05:35:57 DEBUG { 05:35:57 DEBUG "metric": { 05:35:57 DEBUG "pod": "ds-idrepo-0" 05:35:57 DEBUG }, 05:35:57 DEBUG "value": [ 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG [...] 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG "pod": "ds-idrepo-2" 05:35:57 DEBUG }, 05:35:57 DEBUG "value": [ 05:35:57 DEBUG 1713501357, 05:35:57 DEBUG "0" 05:35:57 DEBUG ] 05:35:57 DEBUG } 05:35:57 DEBUG ] 05:35:57 DEBUG } 05:35:57 DEBUG } 05:35:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713501357" 05:35:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501357" 05:35:57 INFO [http_cmd]: http status code OK 05:35:57 DEBUG --- status code --- 05:35:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:35:57 DEBUG --- http response --- 05:35:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:35:57 DEBUG ----- output ----- 05:35:57 DEBUG { 05:35:57 DEBUG "status": "success", 05:35:57 DEBUG "data": { 05:35:57 DEBUG "resultType": "vector", 05:35:57 DEBUG "result": [ 05:35:57 DEBUG { 05:35:57 DEBUG "metric": { 05:35:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:35:57 DEBUG }, 05:35:57 DEBUG "value": [ 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG [...] 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:35:57 DEBUG }, 05:35:57 DEBUG "value": [ 05:35:57 DEBUG 1713501357, 05:35:57 DEBUG "2236903424" 05:35:57 DEBUG ] 05:35:57 DEBUG } 05:35:57 DEBUG ] 05:35:57 DEBUG } 05:35:57 DEBUG } 05:35:57 INFO [http_cmd]: http status code OK 05:35:57 DEBUG --- status code --- 05:35:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:35:57 DEBUG --- http response --- 05:35:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:35:57 DEBUG ----- output ----- 05:35:57 DEBUG { 05:35:57 DEBUG "status": "success", 05:35:57 DEBUG "data": { 05:35:57 DEBUG "resultType": "vector", 05:35:57 DEBUG "result": [ 05:35:57 DEBUG { 05:35:57 DEBUG "metric": { 05:35:57 DEBUG "instance": "10.142.0.107:9100" 05:35:57 DEBUG }, 05:35:57 DEBUG "value": [ 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG [...] 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG "instance": "10.142.0.43:9100" 05:35:57 DEBUG }, 05:35:57 DEBUG "value": [ 05:35:57 DEBUG 1713501357, 05:35:57 DEBUG "0.348" 05:35:57 DEBUG ] 05:35:57 DEBUG } 05:35:57 DEBUG ] 05:35:57 DEBUG } 05:35:57 DEBUG } 05:35:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501357" 05:35:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501357" 05:35:57 INFO [http_cmd]: http status code OK 05:35:57 DEBUG --- status code --- 05:35:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:35:57 DEBUG --- http response --- 05:35:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501357" 05:35:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:35:57 INFO [http_cmd]: http status code OK 05:35:57 DEBUG ----- output ----- 05:35:57 DEBUG --- status code --- 05:35:57 DEBUG { 05:35:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:35:57 DEBUG "status": "success", 05:35:57 DEBUG --- http response --- 05:35:57 DEBUG "data": { 05:35:57 DEBUG "resultType": "vector", 05:35:57 DEBUG "result": [ 05:35:57 DEBUG { 05:35:57 DEBUG "metric": { 05:35:57 DEBUG "pod": "ds-idrepo-0" 05:35:57 DEBUG }, 05:35:57 DEBUG "value": [ 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG [...] 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:35:57 DEBUG }, 05:35:57 DEBUG "value": [ 05:35:57 DEBUG 1713501357, 05:35:57 DEBUG "0" 05:35:57 DEBUG ] 05:35:57 DEBUG } 05:35:57 DEBUG ] 05:35:57 DEBUG } 05:35:57 DEBUG } 05:35:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:35:57 DEBUG ----- output ----- 05:35:57 DEBUG { 05:35:57 DEBUG "status": "success", 05:35:57 DEBUG "data": { 05:35:57 DEBUG "resultType": "vector", 05:35:57 DEBUG "result": [ 05:35:57 DEBUG { 05:35:57 DEBUG "metric": { 05:35:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:35:57 DEBUG }, 05:35:57 DEBUG "value": [ 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG [...] 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:35:57 DEBUG }, 05:35:57 DEBUG "value": [ 05:35:57 DEBUG 1713501357, 05:35:57 DEBUG "2163.9333333333334" 05:35:57 DEBUG ] 05:35:57 DEBUG } 05:35:57 DEBUG ] 05:35:57 DEBUG } 05:35:57 DEBUG } 05:35:57 INFO [http_cmd]: http status code OK 05:35:57 DEBUG --- status code --- 05:35:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:35:57 DEBUG --- http response --- 05:35:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:35:57 DEBUG ----- output ----- 05:35:57 DEBUG { 05:35:57 DEBUG "status": "success", 05:35:57 DEBUG "data": { 05:35:57 DEBUG "resultType": "vector", 05:35:57 DEBUG "result": [ 05:35:57 DEBUG { 05:35:57 DEBUG "metric": { 05:35:57 DEBUG "instance": "10.142.0.107:9100" 05:35:57 DEBUG }, 05:35:57 DEBUG "value": [ 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG [...] 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG "instance": "10.142.0.43:9100" 05:35:57 DEBUG }, 05:35:57 DEBUG "value": [ 05:35:57 DEBUG 1713501357, 05:35:57 DEBUG "64147454.7" 05:35:57 DEBUG ] 05:35:57 DEBUG } 05:35:57 DEBUG ] 05:35:57 DEBUG } 05:35:57 DEBUG } 05:35:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501357" 05:35:57 INFO [http_cmd]: http status code OK 05:35:57 DEBUG --- status code --- 05:35:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:35:57 DEBUG --- http response --- 05:35:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:35:57 DEBUG ----- output ----- 05:35:57 DEBUG { 05:35:57 DEBUG "status": "success", 05:35:57 DEBUG "data": { 05:35:57 DEBUG "resultType": "vector", 05:35:57 DEBUG "result": [ 05:35:57 DEBUG { 05:35:57 DEBUG "metric": { 05:35:57 DEBUG "instance": "10.142.0.107:9100" 05:35:57 DEBUG }, 05:35:57 DEBUG "value": [ 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG [...] 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG "instance": "10.142.0.43:9100" 05:35:57 DEBUG }, 05:35:57 DEBUG "value": [ 05:35:57 DEBUG 1713501357, 05:35:57 DEBUG "64140384.93333334" 05:35:57 DEBUG ] 05:35:57 DEBUG } 05:35:57 DEBUG ] 05:35:57 DEBUG } 05:35:57 DEBUG } 05:35:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501357" 05:35:57 INFO [http_cmd]: http status code OK 05:35:57 DEBUG --- status code --- 05:35:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:35:57 DEBUG --- http response --- 05:35:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:35:57 DEBUG ----- output ----- 05:35:57 DEBUG { 05:35:57 DEBUG "status": "success", 05:35:57 DEBUG "data": { 05:35:57 DEBUG "resultType": "vector", 05:35:57 DEBUG "result": [ 05:35:57 DEBUG { 05:35:57 DEBUG "metric": { 05:35:57 DEBUG "pod": "ds-idrepo-0" 05:35:57 DEBUG }, 05:35:57 DEBUG "value": [ 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG [...] 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG 05:35:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:35:57 DEBUG }, 05:35:57 DEBUG "value": [ 05:35:57 DEBUG 1713501357, 05:35:57 DEBUG "0" 05:35:57 DEBUG ] 05:35:57 DEBUG } 05:35:57 DEBUG ] 05:35:57 DEBUG } 05:35:57 DEBUG } 05:35:59 INFO 05:35:59 INFO [loop_until]: kubectl --namespace=xlou top pods 05:35:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:35:59 INFO [loop_until]: OK (rc = 0) 05:35:59 DEBUG --- stdout --- 05:35:59 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 13796m 9138Mi am-6fd554f8f7-hvvkw 12983m 8908Mi am-6fd554f8f7-pmj4b 14576m 8794Mi ds-cts-0 19471m 13206Mi ds-cts-1 19318m 13132Mi ds-cts-2 19092m 12896Mi ds-idrepo-0 5742m 3269Mi ds-idrepo-1 5847m 3198Mi ds-idrepo-2 5638m 3066Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 3m 1215Mi lodemon-6b89d774d5-rf7sk 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 6842m 2043Mi 05:35:59 DEBUG --- stderr --- 05:35:59 DEBUG 05:36:00 INFO 05:36:00 INFO [loop_until]: kubectl --namespace=xlou top node 05:36:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:36:00 INFO [loop_until]: OK (rc = 0) 05:36:00 DEBUG --- stdout --- 05:36:00 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 71m 0% 6247Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5827m 19% 6717Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 12995m 43% 11616Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 79m 0% 3233Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 19115m 64% 15756Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 160m 0% 5181Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 13664m 45% 11924Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 11271m 37% 2730Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 15598m 52% 11004Mi 9% gke-xlou-cdm-ds-0f601751-j488 6206m 20% 6174Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 19116m 64% 17177Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6385m 21% 6876Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 19424m 65% 14869Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 7269m 45% 5784Mi 9% 05:36:00 DEBUG --- stderr --- 05:36:00 DEBUG 05:36:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713501417" 05:36:57 INFO [http_cmd]: http status code OK 05:36:57 DEBUG --- status code --- 05:36:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:36:57 DEBUG --- http response --- 05:36:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:36:57 DEBUG ----- output ----- 05:36:57 DEBUG { 05:36:57 DEBUG "status": "success", 05:36:57 DEBUG "data": { 05:36:57 DEBUG "resultType": "vector", 05:36:57 DEBUG "result": [ 05:36:57 DEBUG { 05:36:57 DEBUG "metric": { 05:36:57 DEBUG "pod": "ds-idrepo-0" 05:36:57 DEBUG }, 05:36:57 DEBUG "value": [ 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG [...] 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG "pod": "ds-cts-2" 05:36:57 DEBUG }, 05:36:57 DEBUG "value": [ 05:36:57 DEBUG 1713501417, 05:36:57 DEBUG "0" 05:36:57 DEBUG ] 05:36:57 DEBUG } 05:36:57 DEBUG ] 05:36:57 DEBUG } 05:36:57 DEBUG } 05:36:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501417" 05:36:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501417" 05:36:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501417" 05:36:57 INFO [http_cmd]: http status code OK 05:36:57 INFO [http_cmd]: http status code OK 05:36:57 DEBUG --- status code --- 05:36:57 DEBUG --- status code --- 05:36:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:36:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:36:57 DEBUG --- http response --- 05:36:57 DEBUG --- http response --- 05:36:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:36:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:36:57 DEBUG ----- output ----- 05:36:57 DEBUG { 05:36:57 DEBUG "status": "success", 05:36:57 DEBUG "data": { 05:36:57 DEBUG "resultType": "vector", 05:36:57 DEBUG "result": [ 05:36:57 DEBUG { 05:36:57 DEBUG "metric": { 05:36:57 DEBUG "instance": "10.142.0.107:9100" 05:36:57 DEBUG }, 05:36:57 DEBUG "value": [ 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG [...] 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG "instance": "10.142.0.43:9100" 05:36:57 DEBUG }, 05:36:57 DEBUG "value": [ 05:36:57 DEBUG 1713501417, 05:36:57 DEBUG "532210457600" 05:36:57 DEBUG ] 05:36:57 DEBUG } 05:36:57 DEBUG ] 05:36:57 DEBUG } 05:36:57 DEBUG } 05:36:57 INFO [http_cmd]: http status code OK 05:36:57 DEBUG --- status code --- 05:36:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:36:57 DEBUG --- http response --- 05:36:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:36:57 DEBUG ----- output ----- 05:36:57 DEBUG { 05:36:57 DEBUG "status": "success", 05:36:57 DEBUG "data": { 05:36:57 DEBUG "resultType": "vector", 05:36:57 DEBUG "result": [ 05:36:57 DEBUG { 05:36:57 DEBUG "metric": { 05:36:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:36:57 DEBUG }, 05:36:57 DEBUG "value": [ 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG [...] 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:36:57 DEBUG }, 05:36:57 DEBUG "value": [ 05:36:57 DEBUG 1713501417, 05:36:57 DEBUG "0" 05:36:57 DEBUG ] 05:36:57 DEBUG } 05:36:57 DEBUG ] 05:36:57 DEBUG } 05:36:57 DEBUG } 05:36:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501417" 05:36:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713501417" 05:36:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713501417" 05:36:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501417" 05:36:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501417" 05:36:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501417" 05:36:57 ERROR [http_cmd]: ERROR 05:36:57 ERROR --- status code --- 05:36:57 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:36:57 ERROR --- http response --- 05:36:57 INFO [http_cmd]: http status code OK 05:36:57 DEBUG --- status code --- 05:36:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:36:57 DEBUG --- http response --- 05:36:57 INFO [http_cmd]: http status code OK 05:36:57 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:36:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:36:57 DEBUG --- status code --- 05:36:57 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:36:57 DEBUG ----- output ----- 05:36:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:36:57 DEBUG { 05:36:57 DEBUG --- http response --- 05:36:57 DEBUG "status": "success", 05:36:57 DEBUG "data": { 05:36:57 DEBUG "resultType": "vector", 05:36:57 DEBUG "result": [ 05:36:57 DEBUG { 05:36:57 DEBUG "metric": { 05:36:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:36:57 DEBUG }, 05:36:57 DEBUG "value": [ 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG [...] 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:36:57 DEBUG }, 05:36:57 DEBUG "value": [ 05:36:57 DEBUG 1713501417, 05:36:57 DEBUG "34413.78385486486" 05:36:57 DEBUG ] 05:36:57 DEBUG } 05:36:57 DEBUG ] 05:36:57 DEBUG } 05:36:57 DEBUG } 05:36:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:36:57 DEBUG ----- output ----- 05:36:57 DEBUG { 05:36:57 DEBUG "status": "success", 05:36:57 DEBUG "data": { 05:36:57 DEBUG "resultType": "vector", 05:36:57 DEBUG "result": [ 05:36:57 DEBUG { 05:36:57 DEBUG "metric": { 05:36:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:36:57 DEBUG }, 05:36:57 DEBUG "value": [ 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG [...] 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:36:57 DEBUG }, 05:36:57 DEBUG "value": [ 05:36:57 DEBUG 1713501417, 05:36:57 DEBUG "55.175" 05:36:57 DEBUG ] 05:36:57 DEBUG } 05:36:57 DEBUG ] 05:36:57 DEBUG } 05:36:57 DEBUG } 05:36:57 INFO [http_cmd]: http status code OK 05:36:57 DEBUG --- status code --- 05:36:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:36:57 DEBUG --- http response --- 05:36:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:36:57 DEBUG ----- output ----- 05:36:57 DEBUG { 05:36:57 DEBUG "status": "success", 05:36:57 INFO [http_cmd]: http status code OK 05:36:57 DEBUG "data": { 05:36:57 INFO [http_cmd]: http status code OK 05:36:57 DEBUG --- status code --- 05:36:57 DEBUG "resultType": "vector", 05:36:57 DEBUG --- status code --- 05:36:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:36:57 DEBUG "result": [ 05:36:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:36:57 DEBUG --- http response --- 05:36:57 DEBUG { 05:36:57 DEBUG --- http response --- 05:36:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:36:57 DEBUG "metric": { 05:36:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:36:57 DEBUG ----- output ----- 05:36:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:36:57 DEBUG ----- output ----- 05:36:57 DEBUG { 05:36:57 DEBUG }, 05:36:57 DEBUG { 05:36:57 DEBUG "status": "success", 05:36:57 DEBUG "value": [ 05:36:57 DEBUG "status": "success", 05:36:57 DEBUG "data": { 05:36:57 DEBUG 05:36:57 DEBUG "data": { 05:36:57 DEBUG "resultType": "vector", 05:36:57 DEBUG 05:36:57 DEBUG "resultType": "vector", 05:36:57 DEBUG "result": [ 05:36:57 DEBUG 05:36:57 DEBUG "result": [ 05:36:57 DEBUG { 05:36:57 DEBUG [...] 05:36:57 DEBUG { 05:36:57 DEBUG "metric": { 05:36:57 DEBUG 05:36:57 DEBUG "metric": { 05:36:57 DEBUG "pod": "ds-cts-0" 05:36:57 DEBUG 05:36:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:36:57 DEBUG }, 05:36:57 DEBUG 05:36:57 DEBUG }, 05:36:57 DEBUG "value": [ 05:36:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:36:57 DEBUG "value": [ 05:36:57 DEBUG 05:36:57 DEBUG }, 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG "value": [ 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG 1713501417, 05:36:57 DEBUG 05:36:57 DEBUG [...] 05:36:57 DEBUG "3692572" 05:36:57 DEBUG [...] 05:36:57 DEBUG 05:36:57 DEBUG ] 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG } 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG ] 05:36:57 DEBUG 05:36:57 DEBUG "pod": "ds-cts-2" 05:36:57 DEBUG } 05:36:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:36:57 DEBUG }, 05:36:57 DEBUG } 05:36:57 DEBUG }, 05:36:57 DEBUG "value": [ 05:36:57 DEBUG "value": [ 05:36:57 DEBUG 1713501417, 05:36:57 DEBUG 1713501417, 05:36:57 DEBUG "139.66666666666666" 05:36:57 DEBUG "2171.4" 05:36:57 DEBUG ] 05:36:57 DEBUG ] 05:36:57 DEBUG } 05:36:57 DEBUG } 05:36:57 DEBUG ] 05:36:57 DEBUG ] 05:36:57 DEBUG } 05:36:57 DEBUG } 05:36:57 DEBUG } 05:36:57 DEBUG } 05:36:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713501417" 05:36:57 INFO [http_cmd]: http status code OK 05:36:57 DEBUG --- status code --- 05:36:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:36:57 DEBUG --- http response --- 05:36:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:36:57 DEBUG ----- output ----- 05:36:57 DEBUG { 05:36:57 DEBUG "status": "success", 05:36:57 DEBUG "data": { 05:36:57 DEBUG "resultType": "vector", 05:36:57 DEBUG "result": [ 05:36:57 DEBUG { 05:36:57 DEBUG "metric": { 05:36:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:36:57 DEBUG }, 05:36:57 DEBUG "value": [ 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG [...] 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:36:57 DEBUG }, 05:36:57 DEBUG "value": [ 05:36:57 DEBUG 1713501417, 05:36:57 DEBUG "7.214022119354202" 05:36:57 DEBUG ] 05:36:57 DEBUG } 05:36:57 DEBUG ] 05:36:57 DEBUG } 05:36:57 DEBUG } 05:36:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501417" 05:36:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501417" 05:36:57 INFO [http_cmd]: http status code OK 05:36:57 DEBUG --- status code --- 05:36:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:36:57 DEBUG --- http response --- 05:36:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:36:57 DEBUG ----- output ----- 05:36:57 DEBUG { 05:36:57 DEBUG "status": "success", 05:36:57 DEBUG "data": { 05:36:57 DEBUG "resultType": "vector", 05:36:57 DEBUG "result": [ 05:36:57 DEBUG { 05:36:57 DEBUG "metric": { 05:36:57 DEBUG "pod": "ds-cts-0" 05:36:57 DEBUG }, 05:36:57 DEBUG "value": [ 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG [...] 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG "pod": "ds-cts-2" 05:36:57 DEBUG }, 05:36:57 DEBUG "value": [ 05:36:57 DEBUG 1713501417, 05:36:57 DEBUG "0.0005666666666666666" 05:36:57 DEBUG ] 05:36:57 DEBUG } 05:36:57 DEBUG ] 05:36:57 DEBUG } 05:36:57 DEBUG } 05:36:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501417" 05:36:57 INFO [http_cmd]: http status code OK 05:36:57 DEBUG --- status code --- 05:36:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:36:57 DEBUG --- http response --- 05:36:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:36:57 DEBUG ----- output ----- 05:36:57 DEBUG { 05:36:57 DEBUG "status": "success", 05:36:57 DEBUG "data": { 05:36:57 DEBUG "resultType": "vector", 05:36:57 DEBUG "result": [ 05:36:57 DEBUG { 05:36:57 DEBUG "metric": { 05:36:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:36:57 DEBUG }, 05:36:57 DEBUG "value": [ 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG [...] 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:36:57 DEBUG }, 05:36:57 DEBUG "value": [ 05:36:57 DEBUG 1713501417, 05:36:57 DEBUG "4.914872745045215" 05:36:57 DEBUG ] 05:36:57 DEBUG } 05:36:57 DEBUG ] 05:36:57 DEBUG } 05:36:57 DEBUG } 05:36:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501417" 05:36:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501417" 05:36:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501417" 05:36:57 INFO [http_cmd]: http status code OK 05:36:57 DEBUG --- status code --- 05:36:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:36:57 DEBUG --- http response --- 05:36:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:36:57 DEBUG ----- output ----- 05:36:57 DEBUG { 05:36:57 DEBUG "status": "success", 05:36:57 DEBUG "data": { 05:36:57 DEBUG "resultType": "vector", 05:36:57 DEBUG "result": [ 05:36:57 DEBUG { 05:36:57 DEBUG "metric": { 05:36:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:36:57 DEBUG }, 05:36:57 DEBUG "value": [ 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG [...] 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:36:57 DEBUG }, 05:36:57 DEBUG "value": [ 05:36:57 DEBUG 1713501417, 05:36:57 DEBUG "2171.6666666666665" 05:36:57 DEBUG ] 05:36:57 DEBUG } 05:36:57 DEBUG ] 05:36:57 DEBUG } 05:36:57 DEBUG } 05:36:57 INFO [http_cmd]: http status code OK 05:36:57 DEBUG --- status code --- 05:36:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:36:57 DEBUG --- http response --- 05:36:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:36:57 DEBUG ----- output ----- 05:36:57 DEBUG { 05:36:57 DEBUG "status": "success", 05:36:57 DEBUG "data": { 05:36:57 DEBUG "resultType": "vector", 05:36:57 DEBUG "result": [ 05:36:57 DEBUG { 05:36:57 DEBUG "metric": { 05:36:57 DEBUG "pod": "ds-cts-0" 05:36:57 DEBUG }, 05:36:57 DEBUG "value": [ 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG [...] 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG "pod": "ds-cts-2" 05:36:57 DEBUG }, 05:36:57 DEBUG "value": [ 05:36:57 DEBUG 1713501417, 05:36:57 DEBUG "29.966666666666665" 05:36:57 DEBUG ] 05:36:57 DEBUG } 05:36:57 DEBUG ] 05:36:57 DEBUG } 05:36:57 DEBUG } 05:36:57 INFO [http_cmd]: http status code OK 05:36:57 DEBUG --- status code --- 05:36:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:36:57 DEBUG --- http response --- 05:36:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:36:57 DEBUG ----- output ----- 05:36:57 DEBUG { 05:36:57 DEBUG "status": "success", 05:36:57 DEBUG "data": { 05:36:57 DEBUG "resultType": "vector", 05:36:57 DEBUG "result": [ 05:36:57 DEBUG { 05:36:57 DEBUG "metric": { 05:36:57 DEBUG "instance": "10.142.0.107:9100" 05:36:57 DEBUG }, 05:36:57 DEBUG "value": [ 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG [...] 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG "instance": "10.142.0.43:9100" 05:36:57 DEBUG }, 05:36:57 DEBUG "value": [ 05:36:57 DEBUG 1713501417, 05:36:57 DEBUG "0.6721999999999961" 05:36:57 DEBUG ] 05:36:57 DEBUG } 05:36:57 DEBUG ] 05:36:57 DEBUG } 05:36:57 DEBUG } 05:36:57 INFO [http_cmd]: http status code OK 05:36:57 DEBUG --- status code --- 05:36:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:36:57 DEBUG --- http response --- 05:36:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:36:57 DEBUG ----- output ----- 05:36:57 DEBUG { 05:36:57 DEBUG "status": "success", 05:36:57 DEBUG "data": { 05:36:57 DEBUG "resultType": "vector", 05:36:57 DEBUG "result": [ 05:36:57 DEBUG { 05:36:57 DEBUG "metric": { 05:36:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:36:57 DEBUG }, 05:36:57 DEBUG "value": [ 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG [...] 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:36:57 DEBUG }, 05:36:57 DEBUG "value": [ 05:36:57 DEBUG 1713501417, 05:36:57 DEBUG "2004.6" 05:36:57 DEBUG ] 05:36:57 DEBUG } 05:36:57 DEBUG ] 05:36:57 DEBUG } 05:36:57 DEBUG } 05:36:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501417" 05:36:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501417" 05:36:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501417" 05:36:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501417" 05:36:57 INFO [http_cmd]: http status code OK 05:36:57 DEBUG --- status code --- 05:36:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:36:57 DEBUG --- http response --- 05:36:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:36:57 DEBUG ----- output ----- 05:36:57 DEBUG { 05:36:57 DEBUG "status": "success", 05:36:57 DEBUG "data": { 05:36:57 DEBUG "resultType": "vector", 05:36:57 DEBUG "result": [ 05:36:57 DEBUG { 05:36:57 DEBUG "metric": { 05:36:57 DEBUG "pod": "ds-idrepo-0" 05:36:57 DEBUG }, 05:36:57 DEBUG "value": [ 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG [...] 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG "pod": "ds-idrepo-2" 05:36:57 DEBUG }, 05:36:57 DEBUG "value": [ 05:36:57 DEBUG 1713501417, 05:36:57 DEBUG "0" 05:36:57 DEBUG ] 05:36:57 DEBUG } 05:36:57 DEBUG ] 05:36:57 DEBUG } 05:36:57 DEBUG } 05:36:57 INFO [http_cmd]: http status code OK 05:36:57 INFO [http_cmd]: http status code OK 05:36:57 DEBUG --- status code --- 05:36:57 DEBUG --- status code --- 05:36:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:36:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:36:57 DEBUG --- http response --- 05:36:57 DEBUG --- http response --- 05:36:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:36:57 DEBUG ----- output ----- 05:36:57 DEBUG { 05:36:57 DEBUG "status": "success", 05:36:57 DEBUG "data": { 05:36:57 DEBUG "resultType": "vector", 05:36:57 DEBUG "result": [ 05:36:57 DEBUG { 05:36:57 DEBUG "metric": { 05:36:57 DEBUG "pod": "ds-idrepo-0" 05:36:57 DEBUG }, 05:36:57 DEBUG "value": [ 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG [...] 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG "pod": "ds-idrepo-2" 05:36:57 DEBUG }, 05:36:57 DEBUG "value": [ 05:36:57 DEBUG 1713501417, 05:36:57 DEBUG "0" 05:36:57 DEBUG ] 05:36:57 DEBUG } 05:36:57 DEBUG ] 05:36:57 DEBUG } 05:36:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:36:57 DEBUG } 05:36:57 DEBUG ----- output ----- 05:36:57 DEBUG { 05:36:57 DEBUG "status": "success", 05:36:57 DEBUG "data": { 05:36:57 DEBUG "resultType": "vector", 05:36:57 DEBUG "result": [ 05:36:57 DEBUG { 05:36:57 DEBUG "metric": { 05:36:57 DEBUG "instance": "10.142.0.107:9100" 05:36:57 DEBUG }, 05:36:57 DEBUG "value": [ 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG [...] 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG "instance": "10.142.0.43:9100" 05:36:57 DEBUG }, 05:36:57 DEBUG "value": [ 05:36:57 DEBUG 1713501417, 05:36:57 DEBUG "0.374999999999997" 05:36:57 DEBUG ] 05:36:57 DEBUG } 05:36:57 DEBUG ] 05:36:57 DEBUG } 05:36:57 DEBUG } 05:36:57 INFO [http_cmd]: http status code OK 05:36:57 DEBUG --- status code --- 05:36:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:36:57 DEBUG --- http response --- 05:36:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:36:57 DEBUG ----- output ----- 05:36:57 DEBUG { 05:36:57 DEBUG "status": "success", 05:36:57 DEBUG "data": { 05:36:57 DEBUG "resultType": "vector", 05:36:57 DEBUG "result": [ 05:36:57 DEBUG { 05:36:57 DEBUG "metric": { 05:36:57 DEBUG "pod": "ds-idrepo-0" 05:36:57 DEBUG }, 05:36:57 DEBUG "value": [ 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG [...] 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG "pod": "ds-idrepo-2" 05:36:57 DEBUG }, 05:36:57 DEBUG "value": [ 05:36:57 DEBUG 1713501417, 05:36:57 DEBUG "0" 05:36:57 DEBUG ] 05:36:57 DEBUG } 05:36:57 DEBUG ] 05:36:57 DEBUG } 05:36:57 DEBUG } 05:36:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713501417" 05:36:57 INFO [http_cmd]: http status code OK 05:36:57 DEBUG --- status code --- 05:36:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:36:57 DEBUG --- http response --- 05:36:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:36:57 DEBUG ----- output ----- 05:36:57 DEBUG { 05:36:57 DEBUG "status": "success", 05:36:57 DEBUG "data": { 05:36:57 DEBUG "resultType": "vector", 05:36:57 DEBUG "result": [ 05:36:57 DEBUG { 05:36:57 DEBUG "metric": { 05:36:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:36:57 DEBUG }, 05:36:57 DEBUG "value": [ 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG [...] 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:36:57 DEBUG }, 05:36:57 DEBUG "value": [ 05:36:57 DEBUG 1713501417, 05:36:57 DEBUG "2270887936" 05:36:57 DEBUG ] 05:36:57 DEBUG } 05:36:57 DEBUG ] 05:36:57 DEBUG } 05:36:57 DEBUG } 05:36:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501417" 05:36:57 INFO [http_cmd]: http status code OK 05:36:57 DEBUG --- status code --- 05:36:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:36:57 DEBUG --- http response --- 05:36:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501417" 05:36:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:36:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501417" 05:36:57 DEBUG ----- output ----- 05:36:57 DEBUG { 05:36:57 DEBUG "status": "success", 05:36:57 DEBUG "data": { 05:36:57 DEBUG "resultType": "vector", 05:36:57 DEBUG "result": [ 05:36:57 DEBUG { 05:36:57 DEBUG "metric": { 05:36:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:36:57 DEBUG }, 05:36:57 DEBUG "value": [ 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG [...] 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:36:57 DEBUG }, 05:36:57 DEBUG "value": [ 05:36:57 DEBUG 1713501417, 05:36:57 DEBUG "0" 05:36:57 DEBUG ] 05:36:57 DEBUG } 05:36:57 DEBUG ] 05:36:57 DEBUG } 05:36:57 DEBUG } 05:36:57 INFO [http_cmd]: http status code OK 05:36:57 DEBUG --- status code --- 05:36:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:36:57 DEBUG --- http response --- 05:36:57 INFO [http_cmd]: http status code OK 05:36:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:36:57 DEBUG --- status code --- 05:36:57 DEBUG ----- output ----- 05:36:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:36:57 DEBUG { 05:36:57 DEBUG --- http response --- 05:36:57 DEBUG "status": "success", 05:36:57 DEBUG "data": { 05:36:57 DEBUG "resultType": "vector", 05:36:57 DEBUG "result": [ 05:36:57 DEBUG { 05:36:57 DEBUG "metric": { 05:36:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:36:57 DEBUG }, 05:36:57 DEBUG "value": [ 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG [...] 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:36:57 DEBUG }, 05:36:57 DEBUG "value": [ 05:36:57 DEBUG 1713501417, 05:36:57 DEBUG "2170.5" 05:36:57 DEBUG ] 05:36:57 DEBUG } 05:36:57 DEBUG ] 05:36:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:36:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501417" 05:36:57 DEBUG } 05:36:57 DEBUG ----- output ----- 05:36:57 DEBUG } 05:36:57 DEBUG { 05:36:57 DEBUG "status": "success", 05:36:57 DEBUG "data": { 05:36:57 DEBUG "resultType": "vector", 05:36:57 DEBUG "result": [ 05:36:57 DEBUG { 05:36:57 DEBUG "metric": { 05:36:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:36:57 DEBUG }, 05:36:57 DEBUG "value": [ 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG [...] 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:36:57 DEBUG }, 05:36:57 DEBUG "value": [ 05:36:57 DEBUG 1713501417, 05:36:57 DEBUG "0" 05:36:57 DEBUG ] 05:36:57 DEBUG } 05:36:57 DEBUG ] 05:36:57 DEBUG } 05:36:57 DEBUG } 05:36:57 INFO [http_cmd]: http status code OK 05:36:57 DEBUG --- status code --- 05:36:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:36:57 DEBUG --- http response --- 05:36:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:36:57 DEBUG ----- output ----- 05:36:57 DEBUG { 05:36:57 DEBUG "status": "success", 05:36:57 DEBUG "data": { 05:36:57 DEBUG "resultType": "vector", 05:36:57 DEBUG "result": [ 05:36:57 DEBUG { 05:36:57 DEBUG "metric": { 05:36:57 DEBUG "instance": "10.142.0.107:9100" 05:36:57 DEBUG }, 05:36:57 DEBUG "value": [ 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG [...] 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG "instance": "10.142.0.43:9100" 05:36:57 DEBUG }, 05:36:57 DEBUG "value": [ 05:36:57 DEBUG 1713501417, 05:36:57 DEBUG "64190514.93333334" 05:36:57 DEBUG ] 05:36:57 DEBUG } 05:36:57 DEBUG ] 05:36:57 DEBUG } 05:36:57 DEBUG } 05:36:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501417" 05:36:57 INFO [http_cmd]: http status code OK 05:36:57 DEBUG --- status code --- 05:36:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:36:57 DEBUG --- http response --- 05:36:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:36:57 DEBUG ----- output ----- 05:36:57 DEBUG { 05:36:57 DEBUG "status": "success", 05:36:57 DEBUG "data": { 05:36:57 DEBUG "resultType": "vector", 05:36:57 DEBUG "result": [ 05:36:57 DEBUG { 05:36:57 DEBUG "metric": { 05:36:57 DEBUG "instance": "10.142.0.107:9100" 05:36:57 DEBUG }, 05:36:57 DEBUG "value": [ 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG [...] 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG 05:36:57 DEBUG "instance": "10.142.0.43:9100" 05:36:57 DEBUG }, 05:36:57 DEBUG "value": [ 05:36:57 DEBUG 1713501417, 05:36:57 DEBUG "64188510.06666666" 05:36:57 DEBUG ] 05:36:57 DEBUG } 05:36:57 DEBUG ] 05:36:57 DEBUG } 05:36:57 DEBUG } 05:36:59 INFO 05:36:59 INFO [loop_until]: kubectl --namespace=xlou top pods 05:36:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:36:59 INFO [loop_until]: OK (rc = 0) 05:36:59 DEBUG --- stdout --- 05:36:59 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 13841m 9146Mi am-6fd554f8f7-hvvkw 13428m 8908Mi am-6fd554f8f7-pmj4b 14322m 8793Mi ds-cts-0 19595m 13164Mi ds-cts-1 18444m 13562Mi ds-cts-2 19528m 12790Mi ds-idrepo-0 5731m 3268Mi ds-idrepo-1 5777m 3199Mi ds-idrepo-2 5683m 3066Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 4m 1215Mi lodemon-6b89d774d5-rf7sk 1m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7146m 2072Mi 05:36:59 DEBUG --- stderr --- 05:36:59 DEBUG 05:37:00 INFO 05:37:00 INFO [loop_until]: kubectl --namespace=xlou top node 05:37:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:37:00 INFO [loop_until]: OK (rc = 0) 05:37:00 DEBUG --- stdout --- 05:37:00 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 71m 0% 6246Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5843m 19% 6726Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13239m 44% 11617Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 79m 0% 3227Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 19121m 64% 15885Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 137m 0% 5162Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 13769m 46% 11921Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 11293m 37% 2730Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 15268m 51% 10990Mi 9% gke-xlou-cdm-ds-0f601751-j488 6221m 20% 6172Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 18537m 62% 17611Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6430m 21% 6879Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 19288m 64% 14734Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 7253m 45% 5784Mi 9% 05:37:00 DEBUG --- stderr --- 05:37:00 DEBUG 05:37:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713501477" 05:37:57 INFO [http_cmd]: http status code OK 05:37:57 DEBUG --- status code --- 05:37:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:37:57 DEBUG --- http response --- 05:37:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:37:57 DEBUG ----- output ----- 05:37:57 DEBUG { 05:37:57 DEBUG "status": "success", 05:37:57 DEBUG "data": { 05:37:57 DEBUG "resultType": "vector", 05:37:57 DEBUG "result": [ 05:37:57 DEBUG { 05:37:57 DEBUG "metric": { 05:37:57 DEBUG "pod": "ds-idrepo-0" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG [...] 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG "pod": "ds-cts-2" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 1713501477, 05:37:57 DEBUG "0" 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG } 05:37:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501477" 05:37:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501477" 05:37:57 INFO [http_cmd]: http status code OK 05:37:57 DEBUG --- status code --- 05:37:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:37:57 DEBUG --- http response --- 05:37:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:37:57 DEBUG ----- output ----- 05:37:57 DEBUG { 05:37:57 DEBUG "status": "success", 05:37:57 DEBUG "data": { 05:37:57 DEBUG "resultType": "vector", 05:37:57 DEBUG "result": [ 05:37:57 DEBUG { 05:37:57 DEBUG "metric": { 05:37:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG [...] 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 1713501477, 05:37:57 DEBUG "0" 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG } 05:37:57 INFO [http_cmd]: http status code OK 05:37:57 DEBUG --- status code --- 05:37:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:37:57 DEBUG --- http response --- 05:37:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:37:57 DEBUG ----- output ----- 05:37:57 DEBUG { 05:37:57 DEBUG "status": "success", 05:37:57 DEBUG "data": { 05:37:57 DEBUG "resultType": "vector", 05:37:57 DEBUG "result": [ 05:37:57 DEBUG { 05:37:57 DEBUG "metric": { 05:37:57 DEBUG "instance": "10.142.0.107:9100" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG [...] 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG "instance": "10.142.0.43:9100" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 1713501477, 05:37:57 DEBUG "536422865920" 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG } 05:37:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501477" 05:37:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501477" 05:37:57 INFO [http_cmd]: http status code OK 05:37:57 DEBUG --- status code --- 05:37:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:37:57 DEBUG --- http response --- 05:37:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:37:57 DEBUG ----- output ----- 05:37:57 DEBUG { 05:37:57 DEBUG "status": "success", 05:37:57 DEBUG "data": { 05:37:57 DEBUG "resultType": "vector", 05:37:57 DEBUG "result": [ 05:37:57 DEBUG { 05:37:57 DEBUG "metric": { 05:37:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG [...] 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 1713501477, 05:37:57 DEBUG "6122.923204173824" 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG } 05:37:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501477" 05:37:57 INFO [http_cmd]: http status code OK 05:37:57 DEBUG --- status code --- 05:37:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:37:57 DEBUG --- http response --- 05:37:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:37:57 INFO [http_cmd]: http status code OK 05:37:57 DEBUG --- status code --- 05:37:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:37:57 DEBUG --- http response --- 05:37:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:37:57 DEBUG ----- output ----- 05:37:57 DEBUG { 05:37:57 DEBUG "status": "success", 05:37:57 DEBUG "data": { 05:37:57 DEBUG "resultType": "vector", 05:37:57 DEBUG "result": [ 05:37:57 DEBUG { 05:37:57 DEBUG "metric": { 05:37:57 DEBUG "pod": "ds-cts-0" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG [...] 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG "pod": "ds-cts-2" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 1713501477, 05:37:57 DEBUG "151.73333333333332" 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG } 05:37:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713501477" 05:37:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713501477" 05:37:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501477" 05:37:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501477" 05:37:57 ERROR [http_cmd]: ERROR 05:37:57 ERROR --- status code --- 05:37:57 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:37:57 ERROR --- http response --- 05:37:57 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:37:57 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:37:57 INFO [http_cmd]: http status code OK 05:37:57 DEBUG --- status code --- 05:37:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:37:57 DEBUG --- http response --- 05:37:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:37:57 DEBUG ----- output ----- 05:37:57 DEBUG { 05:37:57 DEBUG "status": "success", 05:37:57 DEBUG "data": { 05:37:57 DEBUG "resultType": "vector", 05:37:57 DEBUG "result": [ 05:37:57 DEBUG { 05:37:57 DEBUG "metric": { 05:37:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG [...] 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 1713501477, 05:37:57 DEBUG "55.175" 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG } 05:37:57 INFO [http_cmd]: http status code OK 05:37:57 INFO [http_cmd]: http status code OK 05:37:57 DEBUG --- status code --- 05:37:57 DEBUG --- status code --- 05:37:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:37:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:37:57 DEBUG --- http response --- 05:37:57 DEBUG --- http response --- 05:37:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:37:57 DEBUG ----- output ----- 05:37:57 DEBUG { 05:37:57 DEBUG "status": "success", 05:37:57 DEBUG "data": { 05:37:57 DEBUG "resultType": "vector", 05:37:57 DEBUG "result": [ 05:37:57 DEBUG { 05:37:57 DEBUG "metric": { 05:37:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:37:57 DEBUG [...] 05:37:57 DEBUG ----- output ----- 05:37:57 DEBUG 05:37:57 DEBUG { 05:37:57 DEBUG 05:37:57 DEBUG "status": "success", 05:37:57 DEBUG 05:37:57 DEBUG "data": { 05:37:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:37:57 DEBUG "resultType": "vector", 05:37:57 DEBUG }, 05:37:57 DEBUG "result": [ 05:37:57 DEBUG "value": [ 05:37:57 DEBUG { 05:37:57 DEBUG 1713501477, 05:37:57 DEBUG "metric": { 05:37:57 DEBUG "2167.733333333333" 05:37:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:37:57 DEBUG ] 05:37:57 DEBUG }, 05:37:57 DEBUG } 05:37:57 DEBUG "value": [ 05:37:57 DEBUG ] 05:37:57 DEBUG 05:37:57 DEBUG } 05:37:57 DEBUG 05:37:57 DEBUG } 05:37:57 DEBUG 05:37:57 DEBUG [...] 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 1713501477, 05:37:57 DEBUG "3702573" 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG } 05:37:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713501477" 05:37:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501477" 05:37:57 INFO [http_cmd]: http status code OK 05:37:57 DEBUG --- status code --- 05:37:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:37:57 DEBUG --- http response --- 05:37:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:37:57 DEBUG ----- output ----- 05:37:57 DEBUG { 05:37:57 DEBUG "status": "success", 05:37:57 DEBUG "data": { 05:37:57 DEBUG "resultType": "vector", 05:37:57 DEBUG "result": [ 05:37:57 DEBUG { 05:37:57 DEBUG "metric": { 05:37:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG [...] 05:37:57 DEBUG 05:37:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501477" 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 1713501477, 05:37:57 DEBUG "7.109818622449345" 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG } 05:37:57 INFO [http_cmd]: http status code OK 05:37:57 DEBUG --- status code --- 05:37:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:37:57 DEBUG --- http response --- 05:37:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:37:57 DEBUG ----- output ----- 05:37:57 DEBUG { 05:37:57 DEBUG "status": "success", 05:37:57 DEBUG "data": { 05:37:57 DEBUG "resultType": "vector", 05:37:57 DEBUG "result": [ 05:37:57 DEBUG { 05:37:57 DEBUG "metric": { 05:37:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG [...] 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 1713501477, 05:37:57 DEBUG "5.0323105707752385" 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG } 05:37:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501477" 05:37:57 INFO [http_cmd]: http status code OK 05:37:57 DEBUG --- status code --- 05:37:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:37:57 DEBUG --- http response --- 05:37:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:37:57 DEBUG ----- output ----- 05:37:57 DEBUG { 05:37:57 DEBUG "status": "success", 05:37:57 DEBUG "data": { 05:37:57 DEBUG "resultType": "vector", 05:37:57 DEBUG "result": [ 05:37:57 DEBUG { 05:37:57 DEBUG "metric": { 05:37:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG [...] 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 1713501477, 05:37:57 DEBUG "1998.0333333333333" 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG } 05:37:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501477" 05:37:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501477" 05:37:57 INFO [http_cmd]: http status code OK 05:37:57 DEBUG --- status code --- 05:37:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:37:57 DEBUG --- http response --- 05:37:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:37:57 DEBUG ----- output ----- 05:37:57 DEBUG { 05:37:57 DEBUG "status": "success", 05:37:57 DEBUG "data": { 05:37:57 DEBUG "resultType": "vector", 05:37:57 DEBUG "result": [ 05:37:57 DEBUG { 05:37:57 DEBUG "metric": { 05:37:57 DEBUG "pod": "ds-idrepo-0" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG [...] 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG "pod": "ds-idrepo-2" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 1713501477, 05:37:57 DEBUG "0" 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG } 05:37:57 INFO [http_cmd]: http status code OK 05:37:57 DEBUG --- status code --- 05:37:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:37:57 DEBUG --- http response --- 05:37:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:37:57 DEBUG ----- output ----- 05:37:57 DEBUG { 05:37:57 DEBUG "status": "success", 05:37:57 DEBUG "data": { 05:37:57 DEBUG "resultType": "vector", 05:37:57 DEBUG "result": [ 05:37:57 DEBUG { 05:37:57 DEBUG "metric": { 05:37:57 DEBUG "pod": "ds-cts-0" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG [...] 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG "pod": "ds-cts-2" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 1713501477, 05:37:57 DEBUG "0.0002773577777777778" 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG } 05:37:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501477" 05:37:57 INFO [http_cmd]: http status code OK 05:37:57 DEBUG --- status code --- 05:37:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:37:57 DEBUG --- http response --- 05:37:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:37:57 DEBUG ----- output ----- 05:37:57 DEBUG { 05:37:57 DEBUG "status": "success", 05:37:57 DEBUG "data": { 05:37:57 DEBUG "resultType": "vector", 05:37:57 DEBUG "result": [ 05:37:57 DEBUG { 05:37:57 DEBUG "metric": { 05:37:57 DEBUG "instance": "10.142.0.107:9100" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG [...] 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG "instance": "10.142.0.43:9100" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 1713501477, 05:37:57 DEBUG "0.43716666666665177" 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG } 05:37:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501477" 05:37:57 INFO [http_cmd]: http status code OK 05:37:57 DEBUG --- status code --- 05:37:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:37:57 DEBUG --- http response --- 05:37:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:37:57 DEBUG ----- output ----- 05:37:57 DEBUG { 05:37:57 DEBUG "status": "success", 05:37:57 DEBUG "data": { 05:37:57 DEBUG "resultType": "vector", 05:37:57 DEBUG "result": [ 05:37:57 DEBUG { 05:37:57 DEBUG "metric": { 05:37:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG [...] 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 1713501477, 05:37:57 DEBUG "2167.6" 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG } 05:37:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501477" 05:37:57 INFO [http_cmd]: http status code OK 05:37:57 DEBUG --- status code --- 05:37:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:37:57 DEBUG --- http response --- 05:37:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:37:57 DEBUG ----- output ----- 05:37:57 DEBUG { 05:37:57 DEBUG "status": "success", 05:37:57 DEBUG "data": { 05:37:57 DEBUG "resultType": "vector", 05:37:57 DEBUG "result": [ 05:37:57 DEBUG { 05:37:57 DEBUG "metric": { 05:37:57 DEBUG "pod": "ds-idrepo-0" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG [...] 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG "pod": "ds-idrepo-2" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 1713501477, 05:37:57 DEBUG "0" 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG } 05:37:57 INFO [http_cmd]: http status code OK 05:37:57 DEBUG --- status code --- 05:37:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:37:57 DEBUG --- http response --- 05:37:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:37:57 DEBUG ----- output ----- 05:37:57 DEBUG { 05:37:57 DEBUG "status": "success", 05:37:57 DEBUG "data": { 05:37:57 DEBUG "resultType": "vector", 05:37:57 DEBUG "result": [ 05:37:57 DEBUG { 05:37:57 DEBUG "metric": { 05:37:57 DEBUG "pod": "ds-cts-0" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG [...] 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG "pod": "ds-cts-2" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 1713501477, 05:37:57 DEBUG "28.866666666666667" 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG } 05:37:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501477" 05:37:57 INFO [http_cmd]: http status code OK 05:37:57 DEBUG --- status code --- 05:37:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:37:57 DEBUG --- http response --- 05:37:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:37:57 DEBUG ----- output ----- 05:37:57 DEBUG { 05:37:57 DEBUG "status": "success", 05:37:57 DEBUG "data": { 05:37:57 DEBUG "resultType": "vector", 05:37:57 DEBUG "result": [ 05:37:57 DEBUG { 05:37:57 DEBUG "metric": { 05:37:57 DEBUG "instance": "10.142.0.107:9100" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG [...] 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG "instance": "10.142.0.43:9100" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 1713501477, 05:37:57 DEBUG "0.21200000000000233" 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG } 05:37:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501477" 05:37:57 INFO [http_cmd]: http status code OK 05:37:57 DEBUG --- status code --- 05:37:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:37:57 DEBUG --- http response --- 05:37:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:37:57 DEBUG ----- output ----- 05:37:57 DEBUG { 05:37:57 DEBUG "status": "success", 05:37:57 DEBUG "data": { 05:37:57 DEBUG "resultType": "vector", 05:37:57 DEBUG "result": [ 05:37:57 DEBUG { 05:37:57 DEBUG "metric": { 05:37:57 DEBUG "pod": "ds-idrepo-0" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG [...] 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG "pod": "ds-idrepo-2" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 1713501477, 05:37:57 DEBUG "0" 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG } 05:37:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501477" 05:37:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713501477" 05:37:57 INFO [http_cmd]: http status code OK 05:37:57 DEBUG --- status code --- 05:37:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:37:57 DEBUG --- http response --- 05:37:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:37:57 DEBUG ----- output ----- 05:37:57 DEBUG { 05:37:57 DEBUG "status": "success", 05:37:57 DEBUG "data": { 05:37:57 DEBUG "resultType": "vector", 05:37:57 DEBUG "result": [ 05:37:57 DEBUG { 05:37:57 DEBUG "metric": { 05:37:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG [...] 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 1713501477, 05:37:57 DEBUG "2167.3" 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG } 05:37:57 INFO [http_cmd]: http status code OK 05:37:57 DEBUG --- status code --- 05:37:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:37:57 DEBUG --- http response --- 05:37:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:37:57 DEBUG ----- output ----- 05:37:57 DEBUG { 05:37:57 DEBUG "status": "success", 05:37:57 DEBUG "data": { 05:37:57 DEBUG "resultType": "vector", 05:37:57 DEBUG "result": [ 05:37:57 DEBUG { 05:37:57 DEBUG "metric": { 05:37:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG [...] 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 1713501477, 05:37:57 DEBUG "2270064640" 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG } 05:37:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501477" 05:37:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501477" 05:37:57 INFO [http_cmd]: http status code OK 05:37:57 DEBUG --- status code --- 05:37:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:37:57 DEBUG --- http response --- 05:37:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:37:57 DEBUG ----- output ----- 05:37:57 DEBUG { 05:37:57 DEBUG "status": "success", 05:37:57 DEBUG "data": { 05:37:57 DEBUG "resultType": "vector", 05:37:57 DEBUG "result": [ 05:37:57 DEBUG { 05:37:57 DEBUG "metric": { 05:37:57 DEBUG "instance": "10.142.0.107:9100" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG [...] 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG "instance": "10.142.0.43:9100" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 1713501477, 05:37:57 DEBUG "64145524.93333333" 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG } 05:37:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501477" 05:37:57 INFO [http_cmd]: http status code OK 05:37:57 DEBUG --- status code --- 05:37:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:37:57 DEBUG --- http response --- 05:37:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:37:57 DEBUG ----- output ----- 05:37:57 DEBUG { 05:37:57 DEBUG "status": "success", 05:37:57 DEBUG "data": { 05:37:57 DEBUG "resultType": "vector", 05:37:57 DEBUG "result": [ 05:37:57 DEBUG { 05:37:57 DEBUG "metric": { 05:37:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG [...] 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 1713501477, 05:37:57 DEBUG "0" 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG } 05:37:57 INFO [http_cmd]: http status code OK 05:37:57 DEBUG --- status code --- 05:37:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:37:57 DEBUG --- http response --- 05:37:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:37:57 DEBUG ----- output ----- 05:37:57 DEBUG { 05:37:57 DEBUG "status": "success", 05:37:57 DEBUG "data": { 05:37:57 DEBUG "resultType": "vector", 05:37:57 DEBUG "result": [ 05:37:57 DEBUG { 05:37:57 DEBUG "metric": { 05:37:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG [...] 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 1713501477, 05:37:57 DEBUG "0" 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG } 05:37:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501477" 05:37:57 INFO [http_cmd]: http status code OK 05:37:57 DEBUG --- status code --- 05:37:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:37:57 DEBUG --- http response --- 05:37:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:37:57 DEBUG ----- output ----- 05:37:57 DEBUG { 05:37:57 DEBUG "status": "success", 05:37:57 DEBUG "data": { 05:37:57 DEBUG "resultType": "vector", 05:37:57 DEBUG "result": [ 05:37:57 DEBUG { 05:37:57 DEBUG "metric": { 05:37:57 DEBUG "instance": "10.142.0.107:9100" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG [...] 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG 05:37:57 DEBUG "instance": "10.142.0.43:9100" 05:37:57 DEBUG }, 05:37:57 DEBUG "value": [ 05:37:57 DEBUG 1713501477, 05:37:57 DEBUG "64150133.4" 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG ] 05:37:57 DEBUG } 05:37:57 DEBUG } 05:37:59 INFO 05:37:59 INFO [loop_until]: kubectl --namespace=xlou top pods 05:37:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:37:59 INFO [loop_until]: OK (rc = 0) 05:37:59 DEBUG --- stdout --- 05:37:59 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 14005m 9141Mi am-6fd554f8f7-hvvkw 13353m 8909Mi am-6fd554f8f7-pmj4b 14321m 8798Mi ds-cts-0 20048m 13200Mi ds-cts-1 18666m 13494Mi ds-cts-2 19929m 13250Mi ds-idrepo-0 5820m 3268Mi ds-idrepo-1 5756m 3198Mi ds-idrepo-2 5710m 3066Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 3m 1215Mi lodemon-6b89d774d5-rf7sk 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7121m 2074Mi 05:37:59 DEBUG --- stderr --- 05:37:59 DEBUG 05:38:00 INFO 05:38:00 INFO [loop_until]: kubectl --namespace=xlou top node 05:38:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:38:00 INFO [loop_until]: OK (rc = 0) 05:38:00 DEBUG --- stdout --- 05:38:00 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 66m 0% 6245Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5828m 19% 6724Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 12992m 43% 11632Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 85m 0% 3235Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 19882m 66% 15773Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 154m 0% 5161Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 13838m 46% 11928Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 11347m 38% 2737Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 15305m 51% 11003Mi 9% gke-xlou-cdm-ds-0f601751-j488 6231m 20% 6167Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 18764m 62% 17496Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6377m 21% 6879Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 19864m 66% 15183Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 7273m 45% 5771Mi 9% 05:38:00 DEBUG --- stderr --- 05:38:00 DEBUG 05:38:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713501537" 05:38:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501537" 05:38:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501537" 05:38:57 INFO [http_cmd]: http status code OK 05:38:57 DEBUG --- status code --- 05:38:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:38:57 DEBUG --- http response --- 05:38:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:38:57 DEBUG ----- output ----- 05:38:57 DEBUG { 05:38:57 DEBUG "status": "success", 05:38:57 DEBUG "data": { 05:38:57 DEBUG "resultType": "vector", 05:38:57 DEBUG "result": [ 05:38:57 DEBUG { 05:38:57 DEBUG "metric": { 05:38:57 DEBUG "pod": "ds-idrepo-0" 05:38:57 DEBUG }, 05:38:57 DEBUG "value": [ 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG [...] 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG "pod": "ds-cts-2" 05:38:57 DEBUG }, 05:38:57 DEBUG "value": [ 05:38:57 DEBUG 1713501537, 05:38:57 DEBUG "0" 05:38:57 DEBUG ] 05:38:57 DEBUG } 05:38:57 DEBUG ] 05:38:57 DEBUG } 05:38:57 DEBUG } 05:38:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501537" 05:38:57 INFO [http_cmd]: http status code OK 05:38:57 DEBUG --- status code --- 05:38:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:38:57 DEBUG --- http response --- 05:38:57 INFO [http_cmd]: http status code OK 05:38:57 INFO [http_cmd]: http status code OK 05:38:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:38:57 DEBUG --- status code --- 05:38:57 DEBUG --- status code --- 05:38:57 DEBUG ----- output ----- 05:38:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:38:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:38:57 DEBUG { 05:38:57 DEBUG --- http response --- 05:38:57 DEBUG --- http response --- 05:38:57 DEBUG "status": "success", 05:38:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:38:57 DEBUG "data": { 05:38:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:38:57 DEBUG ----- output ----- 05:38:57 DEBUG "resultType": "vector", 05:38:57 DEBUG ----- output ----- 05:38:57 DEBUG { 05:38:57 DEBUG "result": [ 05:38:57 DEBUG { 05:38:57 DEBUG "status": "success", 05:38:57 DEBUG { 05:38:57 DEBUG "status": "success", 05:38:57 DEBUG "data": { 05:38:57 DEBUG "metric": { 05:38:57 DEBUG "data": { 05:38:57 DEBUG "resultType": "vector", 05:38:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:38:57 DEBUG "resultType": "vector", 05:38:57 DEBUG "result": [ 05:38:57 DEBUG }, 05:38:57 DEBUG "result": [ 05:38:57 DEBUG { 05:38:57 DEBUG "value": [ 05:38:57 DEBUG { 05:38:57 DEBUG "metric": { 05:38:57 DEBUG 05:38:57 DEBUG "metric": { 05:38:57 DEBUG "instance": "10.142.0.107:9100" 05:38:57 DEBUG 05:38:57 DEBUG "pod": "ds-cts-0" 05:38:57 DEBUG }, 05:38:57 DEBUG 05:38:57 DEBUG }, 05:38:57 DEBUG "value": [ 05:38:57 DEBUG [...] 05:38:57 DEBUG "value": [ 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG [...] 05:38:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:38:57 DEBUG [...] 05:38:57 DEBUG 05:38:57 DEBUG }, 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG "value": [ 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 1713501537, 05:38:57 DEBUG 05:38:57 DEBUG "instance": "10.142.0.43:9100" 05:38:57 DEBUG "0" 05:38:57 DEBUG "pod": "ds-cts-2" 05:38:57 DEBUG }, 05:38:57 DEBUG ] 05:38:57 DEBUG }, 05:38:57 DEBUG "value": [ 05:38:57 DEBUG } 05:38:57 DEBUG "value": [ 05:38:57 DEBUG 1713501537, 05:38:57 DEBUG ] 05:38:57 DEBUG 1713501537, 05:38:57 DEBUG "540564499456" 05:38:57 DEBUG } 05:38:57 DEBUG "312.8" 05:38:57 DEBUG ] 05:38:57 DEBUG } 05:38:57 DEBUG ] 05:38:57 DEBUG } 05:38:57 DEBUG } 05:38:57 DEBUG ] 05:38:57 DEBUG ] 05:38:57 DEBUG } 05:38:57 DEBUG } 05:38:57 DEBUG } 05:38:57 DEBUG } 05:38:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501537" 05:38:57 INFO [http_cmd]: http status code OK 05:38:57 DEBUG --- status code --- 05:38:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:38:57 DEBUG --- http response --- 05:38:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:38:57 DEBUG ----- output ----- 05:38:57 DEBUG { 05:38:57 DEBUG "status": "success", 05:38:57 DEBUG "data": { 05:38:57 DEBUG "resultType": "vector", 05:38:57 DEBUG "result": [ 05:38:57 DEBUG { 05:38:57 DEBUG "metric": { 05:38:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:38:57 DEBUG }, 05:38:57 DEBUG "value": [ 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG [...] 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 05:38:57 DEBUG }, 05:38:57 DEBUG "value": [ 05:38:57 DEBUG 1713501537, 05:38:57 DEBUG "0" 05:38:57 DEBUG ] 05:38:57 DEBUG } 05:38:57 DEBUG ] 05:38:57 DEBUG } 05:38:57 DEBUG } 05:38:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713501537" 05:38:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501537" 05:38:57 INFO [http_cmd]: http status code OK 05:38:57 DEBUG --- status code --- 05:38:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:38:57 DEBUG --- http response --- 05:38:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:38:57 DEBUG ----- output ----- 05:38:57 DEBUG { 05:38:57 DEBUG "status": "success", 05:38:57 DEBUG "data": { 05:38:57 DEBUG "resultType": "vector", 05:38:57 DEBUG "result": [ 05:38:57 DEBUG { 05:38:57 DEBUG "metric": { 05:38:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:38:57 DEBUG }, 05:38:57 DEBUG "value": [ 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG [...] 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:38:57 DEBUG }, 05:38:57 DEBUG "value": [ 05:38:57 DEBUG 1713501537, 05:38:57 DEBUG "55.175" 05:38:57 DEBUG ] 05:38:57 DEBUG } 05:38:57 DEBUG ] 05:38:57 DEBUG } 05:38:57 DEBUG } 05:38:57 INFO [http_cmd]: http status code OK 05:38:57 DEBUG --- status code --- 05:38:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:38:57 DEBUG --- http response --- 05:38:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:38:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713501537" 05:38:57 ERROR [http_cmd]: ERROR 05:38:57 ERROR --- status code --- 05:38:57 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:38:57 ERROR --- http response --- 05:38:57 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:38:57 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:38:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501537" 05:38:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501537" 05:38:57 INFO [http_cmd]: http status code OK 05:38:57 DEBUG --- status code --- 05:38:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:38:57 DEBUG --- http response --- 05:38:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:38:57 DEBUG ----- output ----- 05:38:57 DEBUG { 05:38:57 DEBUG "status": "success", 05:38:57 DEBUG "data": { 05:38:57 DEBUG "resultType": "vector", 05:38:57 DEBUG "result": [ 05:38:57 DEBUG { 05:38:57 DEBUG "metric": { 05:38:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:38:57 DEBUG }, 05:38:57 DEBUG "value": [ 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG [...] 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:38:57 DEBUG }, 05:38:57 DEBUG "value": [ 05:38:57 DEBUG 1713501537, 05:38:57 DEBUG "2160.3333333333335" 05:38:57 DEBUG ] 05:38:57 DEBUG } 05:38:57 DEBUG ] 05:38:57 DEBUG } 05:38:57 DEBUG } 05:38:57 INFO [http_cmd]: http status code OK 05:38:57 DEBUG --- status code --- 05:38:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:38:57 DEBUG --- http response --- 05:38:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:38:57 DEBUG ----- output ----- 05:38:57 DEBUG { 05:38:57 DEBUG "status": "success", 05:38:57 DEBUG "data": { 05:38:57 DEBUG "resultType": "vector", 05:38:57 DEBUG "result": [ 05:38:57 DEBUG { 05:38:57 DEBUG "metric": { 05:38:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:38:57 DEBUG }, 05:38:57 DEBUG "value": [ 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG [...] 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:38:57 DEBUG }, 05:38:57 DEBUG "value": [ 05:38:57 DEBUG 1713501537, 05:38:57 DEBUG "3712755.5" 05:38:57 DEBUG ] 05:38:57 DEBUG } 05:38:57 DEBUG ] 05:38:57 DEBUG } 05:38:57 DEBUG } 05:38:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713501537" 05:38:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501537" 05:38:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501537" 05:38:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501537" 05:38:57 INFO [http_cmd]: http status code OK 05:38:57 DEBUG --- status code --- 05:38:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:38:57 DEBUG --- http response --- 05:38:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:38:57 DEBUG ----- output ----- 05:38:57 DEBUG { 05:38:57 DEBUG "status": "success", 05:38:57 DEBUG "data": { 05:38:57 DEBUG "resultType": "vector", 05:38:57 DEBUG "result": [ 05:38:57 DEBUG { 05:38:57 DEBUG "metric": { 05:38:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:38:57 DEBUG }, 05:38:57 DEBUG "value": [ 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG [...] 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:38:57 DEBUG }, 05:38:57 DEBUG "value": [ 05:38:57 DEBUG 1713501537, 05:38:57 DEBUG "7.093775718744011" 05:38:57 DEBUG ] 05:38:57 DEBUG } 05:38:57 DEBUG ] 05:38:57 DEBUG } 05:38:57 DEBUG } 05:38:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501537" 05:38:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501537" 05:38:57 INFO [http_cmd]: http status code OK 05:38:57 DEBUG --- status code --- 05:38:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:38:57 DEBUG --- http response --- 05:38:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:38:57 DEBUG ----- output ----- 05:38:57 DEBUG { 05:38:57 DEBUG "status": "success", 05:38:57 DEBUG "data": { 05:38:57 DEBUG "resultType": "vector", 05:38:57 DEBUG "result": [ 05:38:57 DEBUG { 05:38:57 DEBUG "metric": { 05:38:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:38:57 DEBUG }, 05:38:57 DEBUG "value": [ 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG [...] 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 05:38:57 DEBUG }, 05:38:57 DEBUG "value": [ 05:38:57 DEBUG 1713501537, 05:38:57 DEBUG "0.00001918069461252903" 05:38:57 DEBUG ] 05:38:57 DEBUG } 05:38:57 DEBUG ] 05:38:57 DEBUG } 05:38:57 DEBUG } 05:38:57 INFO [http_cmd]: http status code OK 05:38:57 DEBUG --- status code --- 05:38:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:38:57 DEBUG --- http response --- 05:38:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:38:57 DEBUG ----- output ----- 05:38:57 DEBUG { 05:38:57 DEBUG "status": "success", 05:38:57 DEBUG "data": { 05:38:57 DEBUG "resultType": "vector", 05:38:57 DEBUG "result": [ 05:38:57 DEBUG { 05:38:57 DEBUG "metric": { 05:38:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:38:57 DEBUG }, 05:38:57 DEBUG "value": [ 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG [...] 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:38:57 DEBUG }, 05:38:57 DEBUG "value": [ 05:38:57 DEBUG 1713501537, 05:38:57 DEBUG "1996.7666666666667" 05:38:57 DEBUG ] 05:38:57 DEBUG } 05:38:57 DEBUG ] 05:38:57 DEBUG } 05:38:57 DEBUG } 05:38:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501537" 05:38:57 INFO [http_cmd]: http status code OK 05:38:57 DEBUG --- status code --- 05:38:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:38:57 DEBUG --- http response --- 05:38:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:38:57 DEBUG ----- output ----- 05:38:57 DEBUG { 05:38:57 DEBUG "status": "success", 05:38:57 DEBUG "data": { 05:38:57 DEBUG "resultType": "vector", 05:38:57 DEBUG "result": [ 05:38:57 DEBUG { 05:38:57 DEBUG "metric": { 05:38:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:38:57 DEBUG }, 05:38:57 DEBUG "value": [ 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG [...] 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:38:57 DEBUG }, 05:38:57 DEBUG "value": [ 05:38:57 DEBUG 1713501537, 05:38:57 DEBUG "2159.9666666666667" 05:38:57 DEBUG ] 05:38:57 DEBUG } 05:38:57 DEBUG ] 05:38:57 DEBUG } 05:38:57 DEBUG } 05:38:57 INFO [http_cmd]: http status code OK 05:38:57 DEBUG --- status code --- 05:38:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:38:57 DEBUG --- http response --- 05:38:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:38:57 DEBUG ----- output ----- 05:38:57 DEBUG { 05:38:57 DEBUG "status": "success", 05:38:57 DEBUG "data": { 05:38:57 DEBUG "resultType": "vector", 05:38:57 DEBUG "result": [ 05:38:57 DEBUG { 05:38:57 DEBUG "metric": { 05:38:57 DEBUG "pod": "ds-idrepo-0" 05:38:57 DEBUG }, 05:38:57 DEBUG "value": [ 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG [...] 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG "pod": "ds-idrepo-2" 05:38:57 DEBUG }, 05:38:57 DEBUG "value": [ 05:38:57 DEBUG 1713501537, 05:38:57 DEBUG "0" 05:38:57 DEBUG ] 05:38:57 DEBUG } 05:38:57 DEBUG ] 05:38:57 DEBUG } 05:38:57 DEBUG } 05:38:57 INFO [http_cmd]: http status code OK 05:38:57 DEBUG --- status code --- 05:38:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:38:57 DEBUG --- http response --- 05:38:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:38:57 DEBUG ----- output ----- 05:38:57 DEBUG { 05:38:57 DEBUG "status": "success", 05:38:57 DEBUG "data": { 05:38:57 DEBUG "resultType": "vector", 05:38:57 DEBUG "result": [ 05:38:57 DEBUG { 05:38:57 DEBUG "metric": { 05:38:57 DEBUG "instance": "10.142.0.107:9100" 05:38:57 DEBUG }, 05:38:57 DEBUG "value": [ 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG [...] 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG "instance": "10.142.0.43:9100" 05:38:57 DEBUG }, 05:38:57 DEBUG "value": [ 05:38:57 DEBUG 1713501537, 05:38:57 DEBUG "0.6883333333333278" 05:38:57 DEBUG ] 05:38:57 DEBUG } 05:38:57 DEBUG ] 05:38:57 DEBUG } 05:38:57 DEBUG } 05:38:57 INFO [http_cmd]: http status code OK 05:38:57 DEBUG --- status code --- 05:38:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:38:57 DEBUG --- http response --- 05:38:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:38:57 DEBUG ----- output ----- 05:38:57 DEBUG { 05:38:57 DEBUG "status": "success", 05:38:57 DEBUG "data": { 05:38:57 DEBUG "resultType": "vector", 05:38:57 DEBUG "result": [ 05:38:57 DEBUG { 05:38:57 DEBUG "metric": { 05:38:57 DEBUG "pod": "ds-cts-0" 05:38:57 DEBUG }, 05:38:57 DEBUG "value": [ 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG [...] 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG "pod": "ds-cts-2" 05:38:57 DEBUG }, 05:38:57 DEBUG "value": [ 05:38:57 DEBUG 1713501537, 05:38:57 DEBUG "0.0005333333333333333" 05:38:57 DEBUG ] 05:38:57 DEBUG } 05:38:57 DEBUG ] 05:38:57 DEBUG } 05:38:57 DEBUG } 05:38:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501537" 05:38:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501537" 05:38:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501537" 05:38:57 INFO [http_cmd]: http status code OK 05:38:57 DEBUG --- status code --- 05:38:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:38:57 DEBUG --- http response --- 05:38:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:38:57 DEBUG ----- output ----- 05:38:57 DEBUG { 05:38:57 DEBUG "status": "success", 05:38:57 DEBUG "data": { 05:38:57 DEBUG "resultType": "vector", 05:38:57 DEBUG "result": [ 05:38:57 DEBUG { 05:38:57 DEBUG "metric": { 05:38:57 DEBUG "pod": "ds-idrepo-0" 05:38:57 DEBUG }, 05:38:57 DEBUG "value": [ 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG [...] 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG "pod": "ds-idrepo-2" 05:38:57 DEBUG }, 05:38:57 DEBUG "value": [ 05:38:57 DEBUG 1713501537, 05:38:57 DEBUG "0" 05:38:57 DEBUG ] 05:38:57 DEBUG } 05:38:57 DEBUG ] 05:38:57 DEBUG } 05:38:57 DEBUG } 05:38:57 INFO [http_cmd]: http status code OK 05:38:57 DEBUG --- status code --- 05:38:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:38:57 DEBUG --- http response --- 05:38:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:38:57 DEBUG ----- output ----- 05:38:57 DEBUG { 05:38:57 DEBUG "status": "success", 05:38:57 DEBUG "data": { 05:38:57 DEBUG "resultType": "vector", 05:38:57 DEBUG "result": [ 05:38:57 DEBUG { 05:38:57 DEBUG "metric": { 05:38:57 DEBUG "pod": "ds-cts-0" 05:38:57 DEBUG }, 05:38:57 DEBUG "value": [ 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG [...] 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG "pod": "ds-cts-2" 05:38:57 DEBUG }, 05:38:57 DEBUG "value": [ 05:38:57 DEBUG 1713501537, 05:38:57 DEBUG "30" 05:38:57 DEBUG ] 05:38:57 DEBUG } 05:38:57 DEBUG ] 05:38:57 DEBUG } 05:38:57 DEBUG } 05:38:57 INFO [http_cmd]: http status code OK 05:38:57 DEBUG --- status code --- 05:38:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:38:57 DEBUG --- http response --- 05:38:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:38:57 DEBUG ----- output ----- 05:38:57 DEBUG { 05:38:57 DEBUG "status": "success", 05:38:57 DEBUG "data": { 05:38:57 DEBUG "resultType": "vector", 05:38:57 DEBUG "result": [ 05:38:57 DEBUG { 05:38:57 DEBUG "metric": { 05:38:57 DEBUG "pod": "ds-idrepo-0" 05:38:57 DEBUG }, 05:38:57 DEBUG "value": [ 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG [...] 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG "pod": "ds-idrepo-2" 05:38:57 DEBUG }, 05:38:57 DEBUG "value": [ 05:38:57 DEBUG 1713501537, 05:38:57 DEBUG "0" 05:38:57 DEBUG ] 05:38:57 DEBUG } 05:38:57 DEBUG ] 05:38:57 DEBUG } 05:38:57 DEBUG } 05:38:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501537" 05:38:57 INFO [http_cmd]: http status code OK 05:38:57 DEBUG --- status code --- 05:38:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:38:57 DEBUG --- http response --- 05:38:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:38:57 DEBUG ----- output ----- 05:38:57 DEBUG { 05:38:57 DEBUG "status": "success", 05:38:57 DEBUG "data": { 05:38:57 DEBUG "resultType": "vector", 05:38:57 DEBUG "result": [ 05:38:57 DEBUG { 05:38:57 DEBUG "metric": { 05:38:57 DEBUG "instance": "10.142.0.107:9100" 05:38:57 DEBUG }, 05:38:57 DEBUG "value": [ 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG [...] 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG "instance": "10.142.0.43:9100" 05:38:57 DEBUG }, 05:38:57 DEBUG "value": [ 05:38:57 DEBUG 1713501537, 05:38:57 DEBUG "0.39233333333333514" 05:38:57 DEBUG ] 05:38:57 DEBUG } 05:38:57 DEBUG ] 05:38:57 DEBUG } 05:38:57 DEBUG } 05:38:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501537" 05:38:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713501537" 05:38:57 INFO [http_cmd]: http status code OK 05:38:57 DEBUG --- status code --- 05:38:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:38:57 DEBUG --- http response --- 05:38:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:38:57 DEBUG ----- output ----- 05:38:57 DEBUG { 05:38:57 DEBUG "status": "success", 05:38:57 DEBUG "data": { 05:38:57 DEBUG "resultType": "vector", 05:38:57 DEBUG "result": [ 05:38:57 DEBUG { 05:38:57 DEBUG "metric": { 05:38:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:38:57 DEBUG }, 05:38:57 DEBUG "value": [ 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG [...] 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:38:57 DEBUG }, 05:38:57 DEBUG "value": [ 05:38:57 DEBUG 1713501537, 05:38:57 DEBUG "2160.6666666666665" 05:38:57 DEBUG ] 05:38:57 DEBUG } 05:38:57 DEBUG ] 05:38:57 DEBUG } 05:38:57 DEBUG } 05:38:57 INFO [http_cmd]: http status code OK 05:38:57 DEBUG --- status code --- 05:38:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:38:57 DEBUG --- http response --- 05:38:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:38:57 DEBUG ----- output ----- 05:38:57 DEBUG { 05:38:57 DEBUG "status": "success", 05:38:57 DEBUG "data": { 05:38:57 DEBUG "resultType": "vector", 05:38:57 DEBUG "result": [ 05:38:57 DEBUG { 05:38:57 DEBUG "metric": { 05:38:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:38:57 DEBUG }, 05:38:57 DEBUG "value": [ 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG [...] 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:38:57 DEBUG }, 05:38:57 DEBUG "value": [ 05:38:57 DEBUG 1713501537, 05:38:57 DEBUG "2272649216" 05:38:57 DEBUG ] 05:38:57 DEBUG } 05:38:57 DEBUG ] 05:38:57 DEBUG } 05:38:57 DEBUG } 05:38:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501537" 05:38:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501537" 05:38:57 INFO [http_cmd]: http status code OK 05:38:57 DEBUG --- status code --- 05:38:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:38:57 DEBUG --- http response --- 05:38:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:38:57 DEBUG ----- output ----- 05:38:57 DEBUG { 05:38:57 DEBUG "status": "success", 05:38:57 DEBUG "data": { 05:38:57 DEBUG "resultType": "vector", 05:38:57 DEBUG "result": [ 05:38:57 DEBUG { 05:38:57 DEBUG "metric": { 05:38:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:38:57 DEBUG }, 05:38:57 DEBUG "value": [ 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG [...] 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 05:38:57 DEBUG }, 05:38:57 DEBUG "value": [ 05:38:57 DEBUG 1713501537, 05:38:57 DEBUG "0" 05:38:57 DEBUG ] 05:38:57 DEBUG } 05:38:57 DEBUG ] 05:38:57 DEBUG } 05:38:57 DEBUG } 05:38:57 INFO [http_cmd]: http status code OK 05:38:57 DEBUG --- status code --- 05:38:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:38:57 DEBUG --- http response --- 05:38:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:38:57 DEBUG ----- output ----- 05:38:57 DEBUG { 05:38:57 DEBUG "status": "success", 05:38:57 DEBUG "data": { 05:38:57 DEBUG "resultType": "vector", 05:38:57 DEBUG "result": [ 05:38:57 DEBUG { 05:38:57 DEBUG "metric": { 05:38:57 DEBUG "instance": "10.142.0.107:9100" 05:38:57 DEBUG }, 05:38:57 DEBUG "value": [ 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG [...] 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG "instance": "10.142.0.43:9100" 05:38:57 DEBUG }, 05:38:57 DEBUG "value": [ 05:38:57 DEBUG 1713501537, 05:38:57 DEBUG "64004577.33333333" 05:38:57 DEBUG ] 05:38:57 DEBUG } 05:38:57 DEBUG ] 05:38:57 DEBUG } 05:38:57 DEBUG } 05:38:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501537" 05:38:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501537" 05:38:57 INFO [http_cmd]: http status code OK 05:38:57 DEBUG --- status code --- 05:38:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:38:57 DEBUG --- http response --- 05:38:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:38:57 DEBUG ----- output ----- 05:38:57 DEBUG { 05:38:57 DEBUG "status": "success", 05:38:57 DEBUG "data": { 05:38:57 DEBUG "resultType": "vector", 05:38:57 DEBUG "result": [ 05:38:57 DEBUG { 05:38:57 DEBUG "metric": { 05:38:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:38:57 DEBUG }, 05:38:57 DEBUG "value": [ 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG [...] 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 05:38:57 DEBUG }, 05:38:57 DEBUG "value": [ 05:38:57 DEBUG 1713501537, 05:38:57 DEBUG "0" 05:38:57 DEBUG ] 05:38:57 DEBUG } 05:38:57 DEBUG ] 05:38:57 DEBUG } 05:38:57 DEBUG } 05:38:57 INFO [http_cmd]: http status code OK 05:38:57 DEBUG --- status code --- 05:38:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:38:57 DEBUG --- http response --- 05:38:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:38:57 DEBUG ----- output ----- 05:38:57 DEBUG { 05:38:57 DEBUG "status": "success", 05:38:57 DEBUG "data": { 05:38:57 DEBUG "resultType": "vector", 05:38:57 DEBUG "result": [ 05:38:57 DEBUG { 05:38:57 DEBUG "metric": { 05:38:57 DEBUG "instance": "10.142.0.107:9100" 05:38:57 DEBUG }, 05:38:57 DEBUG "value": [ 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG [...] 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG 05:38:57 DEBUG "instance": "10.142.0.43:9100" 05:38:57 DEBUG }, 05:38:57 DEBUG "value": [ 05:38:57 DEBUG 1713501537, 05:38:57 DEBUG "64008276.46666667" 05:38:57 DEBUG ] 05:38:57 DEBUG } 05:38:57 DEBUG ] 05:38:57 DEBUG } 05:38:57 DEBUG } 05:38:59 INFO 05:38:59 INFO [loop_until]: kubectl --namespace=xlou top pods 05:38:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:38:59 INFO [loop_until]: OK (rc = 0) 05:38:59 DEBUG --- stdout --- 05:38:59 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 13776m 9144Mi am-6fd554f8f7-hvvkw 13588m 8912Mi am-6fd554f8f7-pmj4b 14915m 8796Mi ds-cts-0 19931m 13147Mi ds-cts-1 18754m 13519Mi ds-cts-2 19695m 13244Mi ds-idrepo-0 5863m 3269Mi ds-idrepo-1 5783m 3199Mi ds-idrepo-2 5676m 3066Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 3m 1215Mi lodemon-6b89d774d5-rf7sk 1m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7017m 2076Mi 05:38:59 DEBUG --- stderr --- 05:38:59 DEBUG 05:39:00 INFO 05:39:00 INFO [loop_until]: kubectl --namespace=xlou top node 05:39:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:39:00 INFO [loop_until]: OK (rc = 0) 05:39:00 DEBUG --- stdout --- 05:39:00 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 67m 0% 6244Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5835m 19% 6719Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13310m 44% 11631Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 79m 0% 3240Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 19890m 66% 15991Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 155m 0% 5171Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 13773m 46% 11933Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 11311m 37% 2731Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 15330m 51% 11008Mi 9% gke-xlou-cdm-ds-0f601751-j488 6200m 20% 6162Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 18500m 61% 17150Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6360m 21% 6883Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 20165m 67% 15194Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 7256m 45% 5769Mi 9% 05:39:00 DEBUG --- stderr --- 05:39:00 DEBUG 05:39:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713501597" 05:39:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501597" 05:39:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501597" 05:39:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713501597" 05:39:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501597" 05:39:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501597" 05:39:57 INFO [http_cmd]: http status code OK 05:39:57 INFO [http_cmd]: http status code OK 05:39:57 DEBUG --- status code --- 05:39:57 DEBUG --- status code --- 05:39:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:39:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:39:57 DEBUG --- http response --- 05:39:57 DEBUG --- http response --- 05:39:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:39:57 DEBUG ----- output ----- 05:39:57 DEBUG { 05:39:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 05:39:57 DEBUG "status": "success", 05:39:57 DEBUG ----- output ----- 05:39:57 DEBUG "data": { 05:39:57 DEBUG { 05:39:57 DEBUG "resultType": "vector", 05:39:57 DEBUG "status": "success", 05:39:57 DEBUG "result": [ 05:39:57 DEBUG "data": { 05:39:57 DEBUG { 05:39:57 DEBUG "resultType": "vector", 05:39:57 DEBUG "metric": { 05:39:57 DEBUG "result": [ 05:39:57 DEBUG "pod": "ds-idrepo-0" 05:39:57 DEBUG { 05:39:57 DEBUG }, 05:39:57 DEBUG "metric": { 05:39:57 DEBUG "value": [ 05:39:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:39:57 DEBUG 05:39:57 DEBUG }, 05:39:57 DEBUG 05:39:57 DEBUG "value": [ 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG [...] 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG [...] 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG "pod": "ds-cts-2" 05:39:57 DEBUG 05:39:57 DEBUG }, 05:39:57 DEBUG 05:39:57 DEBUG "value": [ 05:39:57 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:39:57 DEBUG 1713501597, 05:39:57 DEBUG }, 05:39:57 DEBUG "0" 05:39:57 DEBUG ] 05:39:57 DEBUG "value": [ 05:39:57 DEBUG } 05:39:57 DEBUG 1713501597, 05:39:57 DEBUG ] 05:39:57 DEBUG "24757.977417771235" 05:39:57 DEBUG } 05:39:57 DEBUG ] 05:39:57 DEBUG } 05:39:57 DEBUG } 05:39:57 DEBUG ] 05:39:57 DEBUG } 05:39:57 DEBUG } 05:39:57 INFO [http_cmd]: http status code OK 05:39:57 INFO [http_cmd]: http status code OK 05:39:57 INFO [http_cmd]: http status code OK 05:39:57 DEBUG --- status code --- 05:39:57 DEBUG --- status code --- 05:39:57 DEBUG --- status code --- 05:39:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:39:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:39:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:39:57 DEBUG --- http response --- 05:39:57 DEBUG --- http response --- 05:39:57 DEBUG --- http response --- 05:39:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:39:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:39:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:39:57 DEBUG ----- output ----- 05:39:57 DEBUG ----- output ----- 05:39:57 DEBUG ----- output ----- 05:39:57 DEBUG { 05:39:57 DEBUG { 05:39:57 DEBUG { 05:39:57 DEBUG "status": "success", 05:39:57 DEBUG "status": "success", 05:39:57 DEBUG "status": "success", 05:39:57 DEBUG "data": { 05:39:57 DEBUG "data": { 05:39:57 DEBUG "data": { 05:39:57 DEBUG "resultType": "vector", 05:39:57 DEBUG "resultType": "vector", 05:39:57 DEBUG "resultType": "vector", 05:39:57 DEBUG "result": [ 05:39:57 DEBUG "result": [ 05:39:57 DEBUG "result": [ 05:39:57 DEBUG { 05:39:57 DEBUG { 05:39:57 DEBUG { 05:39:57 DEBUG "metric": { 05:39:57 DEBUG "metric": { 05:39:57 DEBUG "metric": { 05:39:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:39:57 DEBUG "instance": "10.142.0.107:9100" 05:39:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:39:57 DEBUG }, 05:39:57 DEBUG }, 05:39:57 DEBUG }, 05:39:57 DEBUG "value": [ 05:39:57 DEBUG "value": [ 05:39:57 DEBUG "value": [ 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG [...] 05:39:57 DEBUG [...] 05:39:57 DEBUG [...] 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:39:57 DEBUG "instance": "10.142.0.43:9100" 05:39:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:39:57 DEBUG }, 05:39:57 DEBUG }, 05:39:57 DEBUG }, 05:39:57 DEBUG "value": [ 05:39:57 DEBUG "value": [ 05:39:57 DEBUG "value": [ 05:39:57 DEBUG 1713501597, 05:39:57 DEBUG 1713501597, 05:39:57 DEBUG 1713501597, 05:39:57 DEBUG "55.175" 05:39:57 DEBUG "544659809280" 05:39:57 DEBUG "0" 05:39:57 DEBUG ] 05:39:57 DEBUG ] 05:39:57 DEBUG ] 05:39:57 DEBUG } 05:39:57 DEBUG } 05:39:57 DEBUG } 05:39:57 DEBUG ] 05:39:57 DEBUG ] 05:39:57 DEBUG ] 05:39:57 DEBUG } 05:39:57 DEBUG } 05:39:57 DEBUG } 05:39:57 DEBUG } 05:39:57 DEBUG } 05:39:57 DEBUG } 05:39:57 INFO [http_cmd]: http status code OK 05:39:57 DEBUG --- status code --- 05:39:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:39:57 DEBUG --- http response --- 05:39:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:39:57 DEBUG ----- output ----- 05:39:57 DEBUG { 05:39:57 DEBUG "status": "success", 05:39:57 DEBUG "data": { 05:39:57 DEBUG "resultType": "vector", 05:39:57 DEBUG "result": [ 05:39:57 DEBUG { 05:39:57 DEBUG "metric": { 05:39:57 DEBUG "pod": "ds-cts-0" 05:39:57 DEBUG }, 05:39:57 DEBUG "value": [ 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG [...] 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG "pod": "ds-cts-2" 05:39:57 DEBUG }, 05:39:57 DEBUG "value": [ 05:39:57 DEBUG 1713501597, 05:39:57 DEBUG "370.73333333333335" 05:39:57 DEBUG ] 05:39:57 DEBUG } 05:39:57 DEBUG ] 05:39:57 DEBUG } 05:39:57 DEBUG } 05:39:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501597" 05:39:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713501597" 05:39:57 INFO [http_cmd]: http status code OK 05:39:57 DEBUG --- status code --- 05:39:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:39:57 DEBUG --- http response --- 05:39:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:39:57 ERROR [http_cmd]: ERROR 05:39:57 ERROR --- status code --- 05:39:57 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:39:57 ERROR --- http response --- 05:39:57 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:39:57 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:39:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501597" 05:39:57 INFO [http_cmd]: http status code OK 05:39:57 DEBUG --- status code --- 05:39:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:39:57 DEBUG --- http response --- 05:39:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:39:57 DEBUG ----- output ----- 05:39:57 DEBUG { 05:39:57 DEBUG "status": "success", 05:39:57 DEBUG "data": { 05:39:57 DEBUG "resultType": "vector", 05:39:57 DEBUG "result": [ 05:39:57 DEBUG { 05:39:57 DEBUG "metric": { 05:39:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:39:57 DEBUG }, 05:39:57 DEBUG "value": [ 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG [...] 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:39:57 DEBUG }, 05:39:57 DEBUG "value": [ 05:39:57 DEBUG 1713501597, 05:39:57 DEBUG "2164.5666666666666" 05:39:57 DEBUG ] 05:39:57 DEBUG } 05:39:57 DEBUG ] 05:39:57 DEBUG } 05:39:57 DEBUG } 05:39:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501597" 05:39:57 INFO [http_cmd]: http status code OK 05:39:57 DEBUG --- status code --- 05:39:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:39:57 DEBUG --- http response --- 05:39:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:39:57 DEBUG ----- output ----- 05:39:57 DEBUG { 05:39:57 DEBUG "status": "success", 05:39:57 DEBUG "data": { 05:39:57 DEBUG "resultType": "vector", 05:39:57 DEBUG "result": [ 05:39:57 DEBUG { 05:39:57 DEBUG "metric": { 05:39:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:39:57 DEBUG }, 05:39:57 DEBUG "value": [ 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG [...] 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:39:57 DEBUG }, 05:39:57 DEBUG "value": [ 05:39:57 DEBUG 1713501597, 05:39:57 DEBUG "3722678.5" 05:39:57 DEBUG ] 05:39:57 DEBUG } 05:39:57 DEBUG ] 05:39:57 DEBUG } 05:39:57 DEBUG } 05:39:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501597" 05:39:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501597" 05:39:57 INFO [http_cmd]: http status code OK 05:39:57 DEBUG --- status code --- 05:39:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:39:57 DEBUG --- http response --- 05:39:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713501597" 05:39:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:39:57 DEBUG ----- output ----- 05:39:57 DEBUG { 05:39:57 DEBUG "status": "success", 05:39:57 DEBUG "data": { 05:39:57 DEBUG "resultType": "vector", 05:39:57 DEBUG "result": [ 05:39:57 DEBUG { 05:39:57 DEBUG "metric": { 05:39:57 DEBUG "pod": "ds-idrepo-0" 05:39:57 DEBUG }, 05:39:57 DEBUG "value": [ 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG [...] 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG "pod": "ds-idrepo-2" 05:39:57 DEBUG }, 05:39:57 DEBUG "value": [ 05:39:57 DEBUG 1713501597, 05:39:57 DEBUG "0" 05:39:57 DEBUG ] 05:39:57 DEBUG } 05:39:57 DEBUG ] 05:39:57 DEBUG } 05:39:57 DEBUG } 05:39:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501597" 05:39:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501597" 05:39:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501597" 05:39:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501597" 05:39:57 INFO [http_cmd]: http status code OK 05:39:57 DEBUG --- status code --- 05:39:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:39:57 DEBUG --- http response --- 05:39:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:39:57 DEBUG ----- output ----- 05:39:57 DEBUG { 05:39:57 DEBUG "status": "success", 05:39:57 DEBUG "data": { 05:39:57 DEBUG "resultType": "vector", 05:39:57 DEBUG "result": [ 05:39:57 DEBUG { 05:39:57 DEBUG "metric": { 05:39:57 DEBUG "pod": "ds-cts-0" 05:39:57 DEBUG }, 05:39:57 DEBUG "value": [ 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG [...] 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG "pod": "ds-cts-2" 05:39:57 DEBUG }, 05:39:57 DEBUG "value": [ 05:39:57 DEBUG 1713501597, 05:39:57 DEBUG "0.00055682" 05:39:57 DEBUG ] 05:39:57 DEBUG } 05:39:57 DEBUG ] 05:39:57 DEBUG } 05:39:57 DEBUG } 05:39:57 INFO [http_cmd]: http status code OK 05:39:57 INFO [http_cmd]: http status code OK 05:39:57 DEBUG --- status code --- 05:39:57 INFO [http_cmd]: http status code OK 05:39:57 INFO [http_cmd]: http status code OK 05:39:57 DEBUG --- status code --- 05:39:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:39:57 DEBUG --- status code --- 05:39:57 DEBUG --- status code --- 05:39:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:39:57 DEBUG --- http response --- 05:39:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:39:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:39:57 DEBUG --- http response --- 05:39:57 INFO [http_cmd]: http status code OK 05:39:57 DEBUG --- http response --- 05:39:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:39:57 DEBUG --- http response --- 05:39:57 DEBUG --- status code --- 05:39:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:39:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:39:57 DEBUG ----- output ----- 05:39:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:39:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 05:39:57 DEBUG ----- output ----- 05:39:57 DEBUG ----- output ----- 05:39:57 DEBUG { 05:39:57 DEBUG --- http response --- 05:39:57 DEBUG ----- output ----- 05:39:57 DEBUG { 05:39:57 DEBUG { 05:39:57 DEBUG "status": "success", 05:39:57 DEBUG { 05:39:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:39:57 DEBUG "status": "success", 05:39:57 DEBUG "status": "success", 05:39:57 DEBUG "data": { 05:39:57 DEBUG "status": "success", 05:39:57 DEBUG ----- output ----- 05:39:57 DEBUG "data": { 05:39:57 DEBUG "data": { 05:39:57 DEBUG "resultType": "vector", 05:39:57 DEBUG "data": { 05:39:57 DEBUG { 05:39:57 DEBUG "resultType": "vector", 05:39:57 DEBUG "resultType": "vector", 05:39:57 DEBUG "result": [ 05:39:57 DEBUG "resultType": "vector", 05:39:57 DEBUG "status": "success", 05:39:57 DEBUG "result": [ 05:39:57 DEBUG "result": [ 05:39:57 DEBUG { 05:39:57 DEBUG "result": [ 05:39:57 DEBUG "data": { 05:39:57 DEBUG { 05:39:57 DEBUG { 05:39:57 DEBUG "metric": { 05:39:57 DEBUG { 05:39:57 DEBUG "resultType": "vector", 05:39:57 DEBUG "metric": { 05:39:57 DEBUG "metric": { 05:39:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:39:57 DEBUG "metric": { 05:39:57 DEBUG "result": [ 05:39:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:39:57 DEBUG "instance": "10.142.0.107:9100" 05:39:57 DEBUG }, 05:39:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:39:57 DEBUG { 05:39:57 DEBUG }, 05:39:57 DEBUG }, 05:39:57 DEBUG "value": [ 05:39:57 DEBUG }, 05:39:57 DEBUG "metric": { 05:39:57 DEBUG "value": [ 05:39:57 DEBUG "value": [ 05:39:57 DEBUG 05:39:57 DEBUG "value": [ 05:39:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG }, 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG "value": [ 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG [...] 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG [...] 05:39:57 DEBUG [...] 05:39:57 DEBUG 05:39:57 DEBUG [...] 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG [...] 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501597" 05:39:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:39:57 DEBUG "instance": "10.142.0.43:9100" 05:39:57 DEBUG }, 05:39:57 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501597" 05:39:57 DEBUG }, 05:39:57 DEBUG "value": [ 05:39:57 DEBUG }, 05:39:57 DEBUG }, 05:39:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:39:57 DEBUG "value": [ 05:39:57 DEBUG 1713501597, 05:39:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501597" 05:39:57 DEBUG "value": [ 05:39:57 DEBUG "value": [ 05:39:57 DEBUG }, 05:39:57 DEBUG "value": [ 05:39:57 DEBUG "2008.3999999999999" 05:39:57 DEBUG 1713501597, 05:39:57 DEBUG 1713501597, 05:39:57 DEBUG 1713501597, 05:39:57 DEBUG 1713501597, 05:39:57 DEBUG ] 05:39:57 DEBUG "7.074108566856863" 05:39:57 DEBUG "2164.1666666666665" 05:39:57 DEBUG "7.102427732494941" 05:39:57 DEBUG "0.4488000000000037" 05:39:57 DEBUG } 05:39:57 DEBUG ] 05:39:57 DEBUG ] 05:39:57 DEBUG ] 05:39:57 DEBUG ] 05:39:57 DEBUG ] 05:39:57 DEBUG } 05:39:57 DEBUG } 05:39:57 DEBUG } 05:39:57 DEBUG } 05:39:57 DEBUG } 05:39:57 DEBUG ] 05:39:57 DEBUG ] 05:39:57 DEBUG ] 05:39:57 DEBUG ] 05:39:57 DEBUG } 05:39:57 DEBUG } 05:39:57 DEBUG } 05:39:57 DEBUG } 05:39:57 DEBUG } 05:39:57 DEBUG } 05:39:57 DEBUG } 05:39:57 DEBUG } 05:39:57 DEBUG } 05:39:57 INFO [http_cmd]: http status code OK 05:39:57 DEBUG --- status code --- 05:39:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:39:57 DEBUG --- http response --- 05:39:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:39:57 DEBUG ----- output ----- 05:39:57 DEBUG { 05:39:57 DEBUG "status": "success", 05:39:57 DEBUG "data": { 05:39:57 DEBUG "resultType": "vector", 05:39:57 DEBUG "result": [ 05:39:57 DEBUG { 05:39:57 DEBUG "metric": { 05:39:57 DEBUG "pod": "ds-idrepo-0" 05:39:57 DEBUG }, 05:39:57 DEBUG "value": [ 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG [...] 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG "pod": "ds-idrepo-2" 05:39:57 DEBUG }, 05:39:57 DEBUG "value": [ 05:39:57 DEBUG 1713501597, 05:39:57 DEBUG "0" 05:39:57 DEBUG ] 05:39:57 DEBUG } 05:39:57 DEBUG ] 05:39:57 DEBUG } 05:39:57 DEBUG } 05:39:57 INFO [http_cmd]: http status code OK 05:39:57 DEBUG --- status code --- 05:39:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:39:57 DEBUG --- http response --- 05:39:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:39:57 DEBUG ----- output ----- 05:39:57 DEBUG { 05:39:57 DEBUG "status": "success", 05:39:57 DEBUG "data": { 05:39:57 DEBUG "resultType": "vector", 05:39:57 DEBUG "result": [ 05:39:57 DEBUG { 05:39:57 DEBUG "metric": { 05:39:57 DEBUG "pod": "ds-idrepo-0" 05:39:57 DEBUG }, 05:39:57 DEBUG "value": [ 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG [...] 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG "pod": "ds-idrepo-2" 05:39:57 DEBUG }, 05:39:57 DEBUG "value": [ 05:39:57 DEBUG 1713501597, 05:39:57 DEBUG "0" 05:39:57 DEBUG ] 05:39:57 DEBUG } 05:39:57 DEBUG ] 05:39:57 DEBUG } 05:39:57 DEBUG } 05:39:57 INFO [http_cmd]: http status code OK 05:39:57 DEBUG --- status code --- 05:39:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:39:57 DEBUG --- http response --- 05:39:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:39:57 DEBUG ----- output ----- 05:39:57 DEBUG { 05:39:57 DEBUG "status": "success", 05:39:57 DEBUG "data": { 05:39:57 DEBUG "resultType": "vector", 05:39:57 DEBUG "result": [ 05:39:57 DEBUG { 05:39:57 DEBUG "metric": { 05:39:57 DEBUG "pod": "ds-cts-0" 05:39:57 DEBUG }, 05:39:57 DEBUG "value": [ 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG [...] 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG "pod": "ds-cts-2" 05:39:57 DEBUG }, 05:39:57 DEBUG "value": [ 05:39:57 DEBUG 1713501597, 05:39:57 DEBUG "30" 05:39:57 DEBUG ] 05:39:57 DEBUG } 05:39:57 DEBUG ] 05:39:57 DEBUG } 05:39:57 DEBUG } 05:39:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501597" 05:39:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501597" 05:39:57 INFO [http_cmd]: http status code OK 05:39:57 DEBUG --- status code --- 05:39:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:39:57 DEBUG --- http response --- 05:39:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:39:57 DEBUG ----- output ----- 05:39:57 DEBUG { 05:39:57 DEBUG "status": "success", 05:39:57 DEBUG "data": { 05:39:57 DEBUG "resultType": "vector", 05:39:57 DEBUG "result": [ 05:39:57 DEBUG { 05:39:57 DEBUG "metric": { 05:39:57 DEBUG "instance": "10.142.0.107:9100" 05:39:57 DEBUG }, 05:39:57 DEBUG "value": [ 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG [...] 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG "instance": "10.142.0.43:9100" 05:39:57 DEBUG }, 05:39:57 DEBUG "value": [ 05:39:57 DEBUG 1713501597, 05:39:57 DEBUG "0.2573333333333318" 05:39:57 DEBUG ] 05:39:57 DEBUG } 05:39:57 DEBUG ] 05:39:57 DEBUG } 05:39:57 DEBUG } 05:39:57 INFO [http_cmd]: http status code OK 05:39:57 DEBUG --- status code --- 05:39:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:39:57 DEBUG --- http response --- 05:39:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:39:57 DEBUG ----- output ----- 05:39:57 DEBUG { 05:39:57 DEBUG "status": "success", 05:39:57 DEBUG "data": { 05:39:57 DEBUG "resultType": "vector", 05:39:57 DEBUG "result": [ 05:39:57 DEBUG { 05:39:57 DEBUG "metric": { 05:39:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:39:57 DEBUG }, 05:39:57 DEBUG "value": [ 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG [...] 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:39:57 DEBUG }, 05:39:57 DEBUG "value": [ 05:39:57 DEBUG 1713501597, 05:39:57 DEBUG "0" 05:39:57 DEBUG ] 05:39:57 DEBUG } 05:39:57 DEBUG ] 05:39:57 DEBUG } 05:39:57 DEBUG } 05:39:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501597" 05:39:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713501597" 05:39:57 INFO [http_cmd]: http status code OK 05:39:57 DEBUG --- status code --- 05:39:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:39:57 DEBUG --- http response --- 05:39:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:39:57 DEBUG ----- output ----- 05:39:57 DEBUG { 05:39:57 DEBUG "status": "success", 05:39:57 DEBUG "data": { 05:39:57 DEBUG "resultType": "vector", 05:39:57 DEBUG "result": [ 05:39:57 DEBUG { 05:39:57 DEBUG "metric": { 05:39:57 DEBUG "instance": "10.142.0.107:9100" 05:39:57 DEBUG }, 05:39:57 DEBUG "value": [ 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG [...] 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG "instance": "10.142.0.43:9100" 05:39:57 DEBUG }, 05:39:57 DEBUG "value": [ 05:39:57 DEBUG 1713501597, 05:39:57 DEBUG "63941495.43333334" 05:39:57 DEBUG ] 05:39:57 DEBUG } 05:39:57 DEBUG ] 05:39:57 DEBUG } 05:39:57 DEBUG } 05:39:57 INFO [http_cmd]: http status code OK 05:39:57 DEBUG --- status code --- 05:39:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:39:57 DEBUG --- http response --- 05:39:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:39:57 DEBUG ----- output ----- 05:39:57 DEBUG { 05:39:57 DEBUG "status": "success", 05:39:57 DEBUG "data": { 05:39:57 DEBUG "resultType": "vector", 05:39:57 DEBUG "result": [ 05:39:57 DEBUG { 05:39:57 DEBUG "metric": { 05:39:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:39:57 DEBUG }, 05:39:57 DEBUG "value": [ 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG [...] 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:39:57 DEBUG }, 05:39:57 DEBUG "value": [ 05:39:57 DEBUG 1713501597, 05:39:57 DEBUG "2274373632" 05:39:57 DEBUG ] 05:39:57 DEBUG } 05:39:57 DEBUG ] 05:39:57 DEBUG } 05:39:57 DEBUG } 05:39:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501597" 05:39:57 INFO [http_cmd]: http status code OK 05:39:57 DEBUG --- status code --- 05:39:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:39:57 DEBUG --- http response --- 05:39:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:39:57 DEBUG ----- output ----- 05:39:57 DEBUG { 05:39:57 DEBUG "status": "success", 05:39:57 DEBUG "data": { 05:39:57 DEBUG "resultType": "vector", 05:39:57 DEBUG "result": [ 05:39:57 DEBUG { 05:39:57 DEBUG "metric": { 05:39:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:39:57 DEBUG }, 05:39:57 DEBUG "value": [ 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG [...] 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:39:57 DEBUG }, 05:39:57 DEBUG "value": [ 05:39:57 DEBUG 1713501597, 05:39:57 DEBUG "2162.5666666666666" 05:39:57 DEBUG ] 05:39:57 DEBUG } 05:39:57 DEBUG ] 05:39:57 DEBUG } 05:39:57 DEBUG } 05:39:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501597" 05:39:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501597" 05:39:57 INFO [http_cmd]: http status code OK 05:39:57 DEBUG --- status code --- 05:39:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:39:57 DEBUG --- http response --- 05:39:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:39:57 DEBUG ----- output ----- 05:39:57 DEBUG { 05:39:57 DEBUG "status": "success", 05:39:57 DEBUG "data": { 05:39:57 DEBUG "resultType": "vector", 05:39:57 DEBUG "result": [ 05:39:57 DEBUG { 05:39:57 DEBUG "metric": { 05:39:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:39:57 DEBUG }, 05:39:57 DEBUG "value": [ 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG [...] 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:39:57 DEBUG }, 05:39:57 DEBUG "value": [ 05:39:57 DEBUG 1713501597, 05:39:57 DEBUG "0" 05:39:57 DEBUG ] 05:39:57 DEBUG } 05:39:57 DEBUG ] 05:39:57 DEBUG } 05:39:57 DEBUG } 05:39:57 INFO [http_cmd]: http status code OK 05:39:57 DEBUG --- status code --- 05:39:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:39:57 DEBUG --- http response --- 05:39:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:39:57 DEBUG ----- output ----- 05:39:57 DEBUG { 05:39:57 DEBUG "status": "success", 05:39:57 DEBUG "data": { 05:39:57 DEBUG "resultType": "vector", 05:39:57 DEBUG "result": [ 05:39:57 DEBUG { 05:39:57 DEBUG "metric": { 05:39:57 DEBUG "instance": "10.142.0.107:9100" 05:39:57 DEBUG }, 05:39:57 DEBUG "value": [ 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG [...] 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG 05:39:57 DEBUG "instance": "10.142.0.43:9100" 05:39:57 DEBUG }, 05:39:57 DEBUG "value": [ 05:39:57 DEBUG 1713501597, 05:39:57 DEBUG "63940116.86666667" 05:39:57 DEBUG ] 05:39:57 DEBUG } 05:39:57 DEBUG ] 05:39:57 DEBUG } 05:39:57 DEBUG } 05:39:59 INFO 05:39:59 INFO [loop_until]: kubectl --namespace=xlou top pods 05:39:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:39:59 INFO [loop_until]: OK (rc = 0) 05:39:59 DEBUG --- stdout --- 05:39:59 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 13767m 9144Mi am-6fd554f8f7-hvvkw 13494m 8909Mi am-6fd554f8f7-pmj4b 14181m 8796Mi ds-cts-0 19770m 13141Mi ds-cts-1 18439m 13250Mi ds-cts-2 20621m 13366Mi ds-idrepo-0 5809m 3268Mi ds-idrepo-1 5752m 3198Mi ds-idrepo-2 5472m 3066Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 3m 1215Mi lodemon-6b89d774d5-rf7sk 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 6770m 2078Mi 05:39:59 DEBUG --- stderr --- 05:39:59 DEBUG 05:40:00 INFO 05:40:00 INFO [loop_until]: kubectl --namespace=xlou top node 05:40:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:40:00 INFO [loop_until]: OK (rc = 0) 05:40:00 DEBUG --- stdout --- 05:40:00 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 67m 0% 6233Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5807m 19% 6719Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 12921m 43% 11622Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 84m 0% 3233Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 19561m 65% 15621Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 137m 0% 5185Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 13738m 46% 11926Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 11277m 37% 2738Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 15514m 51% 11008Mi 9% gke-xlou-cdm-ds-0f601751-j488 6212m 20% 6164Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 18748m 62% 17337Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6346m 21% 6892Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 19517m 65% 15195Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 7228m 45% 5772Mi 9% 05:40:00 DEBUG --- stderr --- 05:40:00 DEBUG 05:40:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501657" 05:40:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713501657" 05:40:57 INFO [http_cmd]: http status code OK 05:40:57 INFO [http_cmd]: http status code OK 05:40:57 DEBUG --- status code --- 05:40:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501657" 05:40:57 DEBUG --- status code --- 05:40:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:40:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:40:57 DEBUG --- http response --- 05:40:57 DEBUG --- http response --- 05:40:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:40:57 DEBUG ----- output ----- 05:40:57 DEBUG { 05:40:57 DEBUG "status": "success", 05:40:57 DEBUG "data": { 05:40:57 DEBUG "resultType": "vector", 05:40:57 DEBUG "result": [ 05:40:57 DEBUG { 05:40:57 DEBUG "metric": { 05:40:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG [...] 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 1713501657, 05:40:57 DEBUG "55.175" 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG } 05:40:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 05:40:57 DEBUG ----- output ----- 05:40:57 DEBUG { 05:40:57 DEBUG "status": "success", 05:40:57 DEBUG "data": { 05:40:57 DEBUG "resultType": "vector", 05:40:57 DEBUG "result": [ 05:40:57 DEBUG { 05:40:57 DEBUG "metric": { 05:40:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG [...] 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG "pod": "ds-cts-0" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 1713501657, 05:40:57 DEBUG "385703361.8048456" 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG } 05:40:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501657" 05:40:57 INFO [http_cmd]: http status code OK 05:40:57 DEBUG --- status code --- 05:40:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:40:57 DEBUG --- http response --- 05:40:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:40:57 DEBUG ----- output ----- 05:40:57 DEBUG { 05:40:57 DEBUG "status": "success", 05:40:57 DEBUG "data": { 05:40:57 DEBUG "resultType": "vector", 05:40:57 DEBUG "result": [ 05:40:57 DEBUG { 05:40:57 DEBUG "metric": { 05:40:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG [...] 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 1713501657, 05:40:57 DEBUG "0" 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG } 05:40:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713501657" 05:40:57 INFO [http_cmd]: http status code OK 05:40:57 DEBUG --- status code --- 05:40:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:40:57 DEBUG --- http response --- 05:40:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:40:57 INFO [http_cmd]: http status code OK 05:40:57 DEBUG --- status code --- 05:40:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:40:57 DEBUG --- http response --- 05:40:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:40:57 DEBUG ----- output ----- 05:40:57 DEBUG { 05:40:57 DEBUG "status": "success", 05:40:57 DEBUG "data": { 05:40:57 DEBUG "resultType": "vector", 05:40:57 DEBUG "result": [ 05:40:57 DEBUG { 05:40:57 DEBUG "metric": { 05:40:57 DEBUG "pod": "ds-idrepo-0" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG [...] 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG "pod": "ds-cts-2" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 1713501657, 05:40:57 DEBUG "0" 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG } 05:40:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501657" 05:40:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501657" 05:40:57 INFO [http_cmd]: http status code OK 05:40:57 DEBUG --- status code --- 05:40:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:40:57 DEBUG --- http response --- 05:40:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:40:57 DEBUG ----- output ----- 05:40:57 DEBUG { 05:40:57 DEBUG "status": "success", 05:40:57 DEBUG "data": { 05:40:57 DEBUG "resultType": "vector", 05:40:57 DEBUG "result": [ 05:40:57 DEBUG { 05:40:57 DEBUG "metric": { 05:40:57 DEBUG "pod": "ds-cts-0" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG [...] 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG "pod": "ds-cts-2" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 1713501657, 05:40:57 DEBUG "351" 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG } 05:40:57 INFO [http_cmd]: http status code OK 05:40:57 DEBUG --- status code --- 05:40:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:40:57 DEBUG --- http response --- 05:40:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:40:57 DEBUG ----- output ----- 05:40:57 DEBUG { 05:40:57 DEBUG "status": "success", 05:40:57 DEBUG "data": { 05:40:57 DEBUG "resultType": "vector", 05:40:57 DEBUG "result": [ 05:40:57 DEBUG { 05:40:57 DEBUG "metric": { 05:40:57 DEBUG "instance": "10.142.0.107:9100" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG [...] 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG "instance": "10.142.0.43:9100" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 1713501657, 05:40:57 DEBUG "548788769792" 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG } 05:40:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713501657" 05:40:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501657" 05:40:57 ERROR [http_cmd]: ERROR 05:40:57 ERROR --- status code --- 05:40:57 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:40:57 ERROR --- http response --- 05:40:57 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:40:57 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:40:57 INFO [http_cmd]: http status code OK 05:40:57 DEBUG --- status code --- 05:40:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:40:57 DEBUG --- http response --- 05:40:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:40:57 DEBUG ----- output ----- 05:40:57 DEBUG { 05:40:57 DEBUG "status": "success", 05:40:57 DEBUG "data": { 05:40:57 DEBUG "resultType": "vector", 05:40:57 DEBUG "result": [ 05:40:57 DEBUG { 05:40:57 DEBUG "metric": { 05:40:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG [...] 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 1713501657, 05:40:57 DEBUG "2165.9666666666667" 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG } 05:40:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713501657" 05:40:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501657" 05:40:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501657" 05:40:57 INFO [http_cmd]: http status code OK 05:40:57 DEBUG --- status code --- 05:40:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:40:57 DEBUG --- http response --- 05:40:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:40:57 DEBUG ----- output ----- 05:40:57 DEBUG { 05:40:57 DEBUG "status": "success", 05:40:57 DEBUG "data": { 05:40:57 DEBUG "resultType": "vector", 05:40:57 DEBUG "result": [ 05:40:57 DEBUG { 05:40:57 DEBUG "metric": { 05:40:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG [...] 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 1713501657, 05:40:57 DEBUG "3731951" 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG } 05:40:57 INFO [http_cmd]: http status code OK 05:40:57 DEBUG --- status code --- 05:40:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:40:57 DEBUG --- http response --- 05:40:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:40:57 DEBUG ----- output ----- 05:40:57 DEBUG { 05:40:57 DEBUG "status": "success", 05:40:57 DEBUG "data": { 05:40:57 DEBUG "resultType": "vector", 05:40:57 DEBUG "result": [ 05:40:57 DEBUG { 05:40:57 DEBUG "metric": { 05:40:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG [...] 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 1713501657, 05:40:57 DEBUG "7.079114005624057" 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG } 05:40:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501657" 05:40:57 INFO [http_cmd]: http status code OK 05:40:57 DEBUG --- status code --- 05:40:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:40:57 DEBUG --- http response --- 05:40:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:40:57 DEBUG ----- output ----- 05:40:57 DEBUG { 05:40:57 DEBUG "status": "success", 05:40:57 DEBUG "data": { 05:40:57 DEBUG "resultType": "vector", 05:40:57 DEBUG "result": [ 05:40:57 DEBUG { 05:40:57 DEBUG "metric": { 05:40:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG [...] 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 1713501657, 05:40:57 DEBUG "1997.8666666666666" 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG } 05:40:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501657" 05:40:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501657" 05:40:57 INFO [http_cmd]: http status code OK 05:40:57 DEBUG --- status code --- 05:40:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:40:57 DEBUG --- http response --- 05:40:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:40:57 DEBUG ----- output ----- 05:40:57 DEBUG { 05:40:57 DEBUG "status": "success", 05:40:57 DEBUG "data": { 05:40:57 DEBUG "resultType": "vector", 05:40:57 DEBUG "result": [ 05:40:57 DEBUG { 05:40:57 DEBUG "metric": { 05:40:57 DEBUG "pod": "ds-idrepo-0" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG [...] 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG "pod": "ds-idrepo-2" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 1713501657, 05:40:57 DEBUG "0" 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG } 05:40:57 INFO [http_cmd]: http status code OK 05:40:57 DEBUG --- status code --- 05:40:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:40:57 DEBUG --- http response --- 05:40:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:40:57 DEBUG ----- output ----- 05:40:57 DEBUG { 05:40:57 DEBUG "status": "success", 05:40:57 DEBUG "data": { 05:40:57 DEBUG "resultType": "vector", 05:40:57 DEBUG "result": [ 05:40:57 DEBUG { 05:40:57 DEBUG "metric": { 05:40:57 DEBUG "pod": "ds-idrepo-0" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG [...] 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG "pod": "ds-idrepo-2" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 1713501657, 05:40:57 DEBUG "0" 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG } 05:40:57 INFO [http_cmd]: http status code OK 05:40:57 DEBUG --- status code --- 05:40:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:40:57 DEBUG --- http response --- 05:40:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 05:40:57 DEBUG ----- output ----- 05:40:57 DEBUG { 05:40:57 DEBUG "status": "success", 05:40:57 DEBUG "data": { 05:40:57 DEBUG "resultType": "vector", 05:40:57 DEBUG "result": [ 05:40:57 DEBUG { 05:40:57 DEBUG "metric": { 05:40:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG [...] 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG "pod": "ds-cts-0" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 1713501657, 05:40:57 DEBUG "10.399047217242725" 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG } 05:40:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501657" 05:40:57 INFO [http_cmd]: http status code OK 05:40:57 DEBUG --- status code --- 05:40:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:40:57 DEBUG --- http response --- 05:40:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501657" 05:40:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:40:57 DEBUG ----- output ----- 05:40:57 DEBUG { 05:40:57 DEBUG "status": "success", 05:40:57 DEBUG "data": { 05:40:57 DEBUG "resultType": "vector", 05:40:57 DEBUG "result": [ 05:40:57 DEBUG { 05:40:57 DEBUG "metric": { 05:40:57 DEBUG "pod": "ds-cts-0" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG [...] 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG "pod": "ds-cts-2" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 1713501657, 05:40:57 DEBUG "0.0006587555555555556" 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG } 05:40:57 INFO [http_cmd]: http status code OK 05:40:57 DEBUG --- status code --- 05:40:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:40:57 DEBUG --- http response --- 05:40:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:40:57 DEBUG ----- output ----- 05:40:57 DEBUG { 05:40:57 DEBUG "status": "success", 05:40:57 DEBUG "data": { 05:40:57 DEBUG "resultType": "vector", 05:40:57 DEBUG "result": [ 05:40:57 DEBUG { 05:40:57 DEBUG "metric": { 05:40:57 DEBUG "pod": "ds-cts-0" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG [...] 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG "pod": "ds-cts-2" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 1713501657, 05:40:57 DEBUG "26.833333333333332" 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG } 05:40:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501657" 05:40:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501657" 05:40:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501657" 05:40:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501657" 05:40:57 INFO [http_cmd]: http status code OK 05:40:57 DEBUG --- status code --- 05:40:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:40:57 DEBUG --- http response --- 05:40:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:40:57 DEBUG ----- output ----- 05:40:57 DEBUG { 05:40:57 DEBUG "status": "success", 05:40:57 DEBUG "data": { 05:40:57 DEBUG "resultType": "vector", 05:40:57 DEBUG "result": [ 05:40:57 DEBUG { 05:40:57 DEBUG "metric": { 05:40:57 DEBUG "pod": "ds-idrepo-0" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG [...] 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG "pod": "ds-idrepo-2" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 1713501657, 05:40:57 DEBUG "0" 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG } 05:40:57 INFO [http_cmd]: http status code OK 05:40:57 DEBUG --- status code --- 05:40:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:40:57 DEBUG --- http response --- 05:40:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:40:57 DEBUG ----- output ----- 05:40:57 DEBUG { 05:40:57 DEBUG "status": "success", 05:40:57 DEBUG "data": { 05:40:57 DEBUG "resultType": "vector", 05:40:57 DEBUG "result": [ 05:40:57 DEBUG { 05:40:57 DEBUG "metric": { 05:40:57 DEBUG "instance": "10.142.0.107:9100" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG [...] 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG "instance": "10.142.0.43:9100" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 1713501657, 05:40:57 DEBUG "0.4189999999999994" 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG } 05:40:57 INFO [http_cmd]: http status code OK 05:40:57 DEBUG --- status code --- 05:40:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:40:57 DEBUG --- http response --- 05:40:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:40:57 DEBUG ----- output ----- 05:40:57 DEBUG { 05:40:57 DEBUG "status": "success", 05:40:57 DEBUG "data": { 05:40:57 DEBUG "resultType": "vector", 05:40:57 DEBUG "result": [ 05:40:57 DEBUG { 05:40:57 DEBUG "metric": { 05:40:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG [...] 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 1713501657, 05:40:57 DEBUG "2166" 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG } 05:40:57 INFO [http_cmd]: http status code OK 05:40:57 DEBUG --- status code --- 05:40:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:40:57 DEBUG --- http response --- 05:40:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:40:57 DEBUG ----- output ----- 05:40:57 DEBUG { 05:40:57 DEBUG "status": "success", 05:40:57 DEBUG "data": { 05:40:57 DEBUG "resultType": "vector", 05:40:57 DEBUG "result": [ 05:40:57 DEBUG { 05:40:57 DEBUG "metric": { 05:40:57 DEBUG "instance": "10.142.0.107:9100" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG [...] 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG "instance": "10.142.0.43:9100" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 1713501657, 05:40:57 DEBUG "0.7063666666666629" 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG } 05:40:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501657" 05:40:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713501657" 05:40:57 INFO [http_cmd]: http status code OK 05:40:57 DEBUG --- status code --- 05:40:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:40:57 DEBUG --- http response --- 05:40:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:40:57 DEBUG ----- output ----- 05:40:57 DEBUG { 05:40:57 DEBUG "status": "success", 05:40:57 DEBUG "data": { 05:40:57 DEBUG "resultType": "vector", 05:40:57 DEBUG "result": [ 05:40:57 DEBUG { 05:40:57 DEBUG "metric": { 05:40:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG [...] 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG "pod": "ds-cts-0" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 1713501657, 05:40:57 DEBUG "35566655.49886425" 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG } 05:40:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501657" 05:40:57 INFO [http_cmd]: http status code OK 05:40:57 DEBUG --- status code --- 05:40:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:40:57 DEBUG --- http response --- 05:40:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:40:57 DEBUG ----- output ----- 05:40:57 DEBUG { 05:40:57 DEBUG "status": "success", 05:40:57 DEBUG "data": { 05:40:57 DEBUG "resultType": "vector", 05:40:57 DEBUG "result": [ 05:40:57 DEBUG { 05:40:57 DEBUG "metric": { 05:40:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG [...] 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 1713501657, 05:40:57 DEBUG "2277781504" 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG } 05:40:57 INFO [http_cmd]: http status code OK 05:40:57 DEBUG --- status code --- 05:40:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:40:57 DEBUG --- http response --- 05:40:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:40:57 DEBUG ----- output ----- 05:40:57 DEBUG { 05:40:57 DEBUG "status": "success", 05:40:57 DEBUG "data": { 05:40:57 DEBUG "resultType": "vector", 05:40:57 DEBUG "result": [ 05:40:57 DEBUG { 05:40:57 DEBUG "metric": { 05:40:57 DEBUG "instance": "10.142.0.107:9100" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG [...] 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG "instance": "10.142.0.43:9100" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 1713501657, 05:40:57 DEBUG "64046821.36666667" 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG } 05:40:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501657" 05:40:57 INFO [http_cmd]: http status code OK 05:40:57 DEBUG --- status code --- 05:40:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:40:57 DEBUG --- http response --- 05:40:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:40:57 DEBUG ----- output ----- 05:40:57 DEBUG { 05:40:57 DEBUG "status": "success", 05:40:57 DEBUG "data": { 05:40:57 DEBUG "resultType": "vector", 05:40:57 DEBUG "result": [ 05:40:57 DEBUG { 05:40:57 DEBUG "metric": { 05:40:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG [...] 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 1713501657, 05:40:57 DEBUG "2167.0333333333333" 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG } 05:40:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501657" 05:40:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501657" 05:40:57 INFO [http_cmd]: http status code OK 05:40:57 INFO [http_cmd]: http status code OK 05:40:57 DEBUG --- status code --- 05:40:57 DEBUG --- status code --- 05:40:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:40:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:40:57 DEBUG --- http response --- 05:40:57 DEBUG --- http response --- 05:40:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:40:57 DEBUG ----- output ----- 05:40:57 DEBUG { 05:40:57 DEBUG "status": "success", 05:40:57 DEBUG "data": { 05:40:57 DEBUG "resultType": "vector", 05:40:57 DEBUG "result": [ 05:40:57 DEBUG { 05:40:57 DEBUG "metric": { 05:40:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG [...] 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG "pod": "ds-cts-0" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 1713501657, 05:40:57 DEBUG "19375858.571966257" 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG } 05:40:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:40:57 DEBUG ----- output ----- 05:40:57 DEBUG { 05:40:57 DEBUG "status": "success", 05:40:57 DEBUG "data": { 05:40:57 DEBUG "resultType": "vector", 05:40:57 DEBUG "result": [ 05:40:57 DEBUG { 05:40:57 DEBUG "metric": { 05:40:57 DEBUG "instance": "10.142.0.107:9100" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG [...] 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG 05:40:57 DEBUG "instance": "10.142.0.43:9100" 05:40:57 DEBUG }, 05:40:57 DEBUG "value": [ 05:40:57 DEBUG 1713501657, 05:40:57 DEBUG "64042607.4" 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG ] 05:40:57 DEBUG } 05:40:57 DEBUG } 05:40:59 INFO 05:40:59 INFO [loop_until]: kubectl --namespace=xlou top pods 05:40:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:40:59 INFO [loop_until]: OK (rc = 0) 05:40:59 DEBUG --- stdout --- 05:40:59 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 13906m 9144Mi am-6fd554f8f7-hvvkw 13115m 8910Mi am-6fd554f8f7-pmj4b 14470m 8798Mi ds-cts-0 20048m 13280Mi ds-cts-1 18462m 13208Mi ds-cts-2 19835m 12974Mi ds-idrepo-0 5653m 3268Mi ds-idrepo-1 5900m 3199Mi ds-idrepo-2 5640m 3066Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 4m 1215Mi lodemon-6b89d774d5-rf7sk 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7030m 2080Mi 05:40:59 DEBUG --- stderr --- 05:40:59 DEBUG 05:41:00 INFO 05:41:00 INFO [loop_until]: kubectl --namespace=xlou top node 05:41:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:41:00 INFO [loop_until]: OK (rc = 0) 05:41:00 DEBUG --- stdout --- 05:41:00 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 71m 0% 6237Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5834m 19% 6725Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13352m 44% 11619Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 3224Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 20141m 67% 15818Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 153m 0% 5156Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 14078m 47% 11938Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 10925m 36% 2736Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 15230m 51% 11003Mi 9% gke-xlou-cdm-ds-0f601751-j488 6203m 20% 6173Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 18509m 61% 17211Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6410m 21% 6886Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 19943m 66% 14982Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 7270m 45% 5772Mi 9% 05:41:00 DEBUG --- stderr --- 05:41:00 DEBUG 05:41:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713501717" 05:41:57 INFO [http_cmd]: http status code OK 05:41:57 DEBUG --- status code --- 05:41:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:41:57 DEBUG --- http response --- 05:41:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:41:57 DEBUG ----- output ----- 05:41:57 DEBUG { 05:41:57 DEBUG "status": "success", 05:41:57 DEBUG "data": { 05:41:57 DEBUG "resultType": "vector", 05:41:57 DEBUG "result": [ 05:41:57 DEBUG { 05:41:57 DEBUG "metric": { 05:41:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:41:57 DEBUG }, 05:41:57 DEBUG "value": [ 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG [...] 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:41:57 DEBUG }, 05:41:57 DEBUG "value": [ 05:41:57 DEBUG 1713501717, 05:41:57 DEBUG "55.175" 05:41:57 DEBUG ] 05:41:57 DEBUG } 05:41:57 DEBUG ] 05:41:57 DEBUG } 05:41:57 DEBUG } 05:41:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501717" 05:41:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501717" 05:41:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713501717" 05:41:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501717" 05:41:57 INFO [http_cmd]: http status code OK 05:41:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501717" 05:41:57 INFO [http_cmd]: http status code OK 05:41:57 DEBUG --- status code --- 05:41:57 DEBUG --- status code --- 05:41:57 INFO [http_cmd]: http status code OK 05:41:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:41:57 INFO [http_cmd]: http status code OK 05:41:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:41:57 DEBUG --- status code --- 05:41:57 DEBUG --- http response --- 05:41:57 DEBUG --- status code --- 05:41:57 DEBUG --- http response --- 05:41:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:41:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:41:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:41:57 DEBUG --- http response --- 05:41:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 05:41:57 DEBUG --- http response --- 05:41:57 DEBUG ----- output ----- 05:41:57 DEBUG ----- output ----- 05:41:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:41:57 DEBUG { 05:41:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:41:57 DEBUG { 05:41:57 DEBUG "status": "success", 05:41:57 DEBUG ----- output ----- 05:41:57 DEBUG "status": "success", 05:41:57 DEBUG "data": { 05:41:57 DEBUG { 05:41:57 DEBUG "data": { 05:41:57 DEBUG "resultType": "vector", 05:41:57 DEBUG "status": "success", 05:41:57 DEBUG "resultType": "vector", 05:41:57 DEBUG "result": [ 05:41:57 DEBUG "data": { 05:41:57 DEBUG "result": [ 05:41:57 DEBUG { 05:41:57 DEBUG "resultType": "vector", 05:41:57 DEBUG { 05:41:57 DEBUG "metric": { 05:41:57 DEBUG "result": [ 05:41:57 DEBUG "metric": { 05:41:57 DEBUG "instance": "10.142.0.107:9100" 05:41:57 DEBUG { 05:41:57 DEBUG "pod": "ds-idrepo-1" 05:41:57 DEBUG }, 05:41:57 DEBUG "metric": { 05:41:57 DEBUG }, 05:41:57 DEBUG "value": [ 05:41:57 DEBUG "pod": "ds-idrepo-0" 05:41:57 DEBUG "value": [ 05:41:57 DEBUG 05:41:57 DEBUG }, 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG "value": [ 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG [...] 05:41:57 DEBUG 05:41:57 DEBUG [...] 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG [...] 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG "instance": "10.142.0.43:9100" 05:41:57 DEBUG 05:41:57 DEBUG "pod": "ds-cts-0" 05:41:57 DEBUG }, 05:41:57 DEBUG 05:41:57 DEBUG }, 05:41:57 DEBUG "value": [ 05:41:57 DEBUG "pod": "ds-cts-2" 05:41:57 DEBUG "value": [ 05:41:57 DEBUG 1713501717, 05:41:57 DEBUG }, 05:41:57 DEBUG 1713501717, 05:41:57 DEBUG "552718969856" 05:41:57 DEBUG "value": [ 05:41:57 DEBUG "0" 05:41:57 DEBUG ] 05:41:57 DEBUG 1713501717, 05:41:57 DEBUG ] 05:41:57 DEBUG } 05:41:57 DEBUG "0" 05:41:57 DEBUG } 05:41:57 DEBUG ] 05:41:57 DEBUG ] 05:41:57 DEBUG ] 05:41:57 DEBUG } 05:41:57 DEBUG } 05:41:57 DEBUG } 05:41:57 DEBUG } 05:41:57 DEBUG ] 05:41:57 DEBUG } 05:41:57 DEBUG } 05:41:57 DEBUG } 05:41:57 INFO [http_cmd]: http status code OK 05:41:57 DEBUG --- status code --- 05:41:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:41:57 DEBUG --- http response --- 05:41:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:41:57 DEBUG ----- output ----- 05:41:57 DEBUG { 05:41:57 DEBUG "status": "success", 05:41:57 DEBUG "data": { 05:41:57 DEBUG "resultType": "vector", 05:41:57 DEBUG "result": [ 05:41:57 DEBUG { 05:41:57 DEBUG "metric": { 05:41:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:41:57 DEBUG }, 05:41:57 DEBUG "value": [ 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG [...] 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:41:57 DEBUG }, 05:41:57 DEBUG "value": [ 05:41:57 DEBUG 1713501717, 05:41:57 DEBUG "0" 05:41:57 DEBUG ] 05:41:57 DEBUG } 05:41:57 DEBUG ] 05:41:57 DEBUG } 05:41:57 DEBUG } 05:41:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501717" 05:41:57 INFO [http_cmd]: http status code OK 05:41:57 DEBUG --- status code --- 05:41:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:41:57 DEBUG --- http response --- 05:41:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:41:57 DEBUG ----- output ----- 05:41:57 DEBUG { 05:41:57 DEBUG "status": "success", 05:41:57 DEBUG "data": { 05:41:57 DEBUG "resultType": "vector", 05:41:57 DEBUG "result": [ 05:41:57 DEBUG { 05:41:57 DEBUG "metric": { 05:41:57 DEBUG "pod": "ds-cts-0" 05:41:57 DEBUG }, 05:41:57 DEBUG "value": [ 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG [...] 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG "pod": "ds-cts-2" 05:41:57 DEBUG }, 05:41:57 DEBUG "value": [ 05:41:57 DEBUG 1713501717, 05:41:57 DEBUG "197.26666666666665" 05:41:57 DEBUG ] 05:41:57 DEBUG } 05:41:57 DEBUG ] 05:41:57 DEBUG } 05:41:57 DEBUG } 05:41:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713501717" 05:41:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501717" 05:41:57 ERROR [http_cmd]: ERROR 05:41:57 ERROR --- status code --- 05:41:57 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:41:57 ERROR --- http response --- 05:41:57 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:41:57 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:41:57 INFO [http_cmd]: http status code OK 05:41:57 DEBUG --- status code --- 05:41:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:41:57 DEBUG --- http response --- 05:41:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:41:57 DEBUG ----- output ----- 05:41:57 DEBUG { 05:41:57 DEBUG "status": "success", 05:41:57 DEBUG "data": { 05:41:57 DEBUG "resultType": "vector", 05:41:57 DEBUG "result": [ 05:41:57 DEBUG { 05:41:57 DEBUG "metric": { 05:41:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:41:57 DEBUG }, 05:41:57 DEBUG "value": [ 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG [...] 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:41:57 DEBUG }, 05:41:57 DEBUG "value": [ 05:41:57 DEBUG 1713501717, 05:41:57 DEBUG "2171.633333333333" 05:41:57 DEBUG ] 05:41:57 DEBUG } 05:41:57 DEBUG ] 05:41:57 DEBUG } 05:41:57 DEBUG } 05:41:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501717" 05:41:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501717" 05:41:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713501717" 05:41:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501717" 05:41:57 INFO [http_cmd]: http status code OK 05:41:57 DEBUG --- status code --- 05:41:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:41:57 DEBUG --- http response --- 05:41:57 INFO [http_cmd]: http status code OK 05:41:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:41:57 DEBUG ----- output ----- 05:41:57 DEBUG --- status code --- 05:41:57 DEBUG { 05:41:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:41:57 DEBUG "status": "success", 05:41:57 DEBUG --- http response --- 05:41:57 DEBUG "data": { 05:41:57 DEBUG "resultType": "vector", 05:41:57 DEBUG "result": [ 05:41:57 DEBUG { 05:41:57 DEBUG "metric": { 05:41:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:41:57 DEBUG }, 05:41:57 DEBUG "value": [ 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG [...] 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:41:57 DEBUG }, 05:41:57 DEBUG "value": [ 05:41:57 DEBUG 1713501717, 05:41:57 DEBUG "3742040.5" 05:41:57 DEBUG ] 05:41:57 DEBUG } 05:41:57 DEBUG ] 05:41:57 DEBUG } 05:41:57 DEBUG } 05:41:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:41:57 DEBUG ----- output ----- 05:41:57 DEBUG { 05:41:57 DEBUG "status": "success", 05:41:57 DEBUG "data": { 05:41:57 DEBUG "resultType": "vector", 05:41:57 DEBUG "result": [ 05:41:57 DEBUG { 05:41:57 DEBUG "metric": { 05:41:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:41:57 DEBUG }, 05:41:57 DEBUG "value": [ 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG [...] 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:41:57 DEBUG }, 05:41:57 DEBUG "value": [ 05:41:57 DEBUG 1713501717, 05:41:57 DEBUG "2000.9333333333334" 05:41:57 DEBUG ] 05:41:57 DEBUG } 05:41:57 DEBUG ] 05:41:57 DEBUG } 05:41:57 DEBUG } 05:41:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501717" 05:41:57 INFO [http_cmd]: http status code OK 05:41:57 DEBUG --- status code --- 05:41:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:41:57 DEBUG --- http response --- 05:41:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:41:57 DEBUG ----- output ----- 05:41:57 DEBUG { 05:41:57 DEBUG "status": "success", 05:41:57 DEBUG "data": { 05:41:57 DEBUG "resultType": "vector", 05:41:57 DEBUG "result": [ 05:41:57 DEBUG { 05:41:57 DEBUG "metric": { 05:41:57 DEBUG "pod": "ds-cts-0" 05:41:57 DEBUG }, 05:41:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501717" 05:41:57 DEBUG "value": [ 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG [...] 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG "pod": "ds-cts-2" 05:41:57 DEBUG }, 05:41:57 DEBUG "value": [ 05:41:57 DEBUG 1713501717, 05:41:57 DEBUG "0.0007" 05:41:57 DEBUG ] 05:41:57 DEBUG } 05:41:57 DEBUG ] 05:41:57 DEBUG } 05:41:57 DEBUG } 05:41:57 INFO [http_cmd]: http status code OK 05:41:57 DEBUG --- status code --- 05:41:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:41:57 DEBUG --- http response --- 05:41:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:41:57 DEBUG ----- output ----- 05:41:57 DEBUG { 05:41:57 DEBUG "status": "success", 05:41:57 DEBUG "data": { 05:41:57 DEBUG "resultType": "vector", 05:41:57 DEBUG "result": [ 05:41:57 DEBUG { 05:41:57 DEBUG "metric": { 05:41:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:41:57 DEBUG }, 05:41:57 DEBUG "value": [ 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG [...] 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:41:57 DEBUG }, 05:41:57 DEBUG "value": [ 05:41:57 DEBUG 1713501717, 05:41:57 DEBUG "7.156604904025203" 05:41:57 DEBUG ] 05:41:57 DEBUG } 05:41:57 DEBUG ] 05:41:57 DEBUG } 05:41:57 DEBUG } 05:41:57 INFO [http_cmd]: http status code OK 05:41:57 DEBUG --- status code --- 05:41:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:41:57 DEBUG --- http response --- 05:41:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501717" 05:41:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:41:57 DEBUG ----- output ----- 05:41:57 DEBUG { 05:41:57 DEBUG "status": "success", 05:41:57 DEBUG "data": { 05:41:57 DEBUG "resultType": "vector", 05:41:57 DEBUG "result": [ 05:41:57 DEBUG { 05:41:57 DEBUG "metric": { 05:41:57 DEBUG "pod": "ds-idrepo-0" 05:41:57 DEBUG }, 05:41:57 DEBUG "value": [ 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG [...] 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG "pod": "ds-idrepo-2" 05:41:57 DEBUG }, 05:41:57 DEBUG "value": [ 05:41:57 DEBUG 1713501717, 05:41:57 DEBUG "0" 05:41:57 DEBUG ] 05:41:57 DEBUG } 05:41:57 DEBUG ] 05:41:57 DEBUG } 05:41:57 DEBUG } 05:41:57 INFO [http_cmd]: http status code OK 05:41:57 DEBUG --- status code --- 05:41:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:41:57 DEBUG --- http response --- 05:41:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 05:41:57 DEBUG ----- output ----- 05:41:57 DEBUG { 05:41:57 DEBUG "status": "success", 05:41:57 DEBUG "data": { 05:41:57 DEBUG "resultType": "vector", 05:41:57 DEBUG "result": [ 05:41:57 DEBUG { 05:41:57 DEBUG "metric": { 05:41:57 DEBUG "pod": "ds-idrepo-1" 05:41:57 DEBUG }, 05:41:57 DEBUG "value": [ 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG [...] 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG "pod": "ds-cts-0" 05:41:57 DEBUG }, 05:41:57 DEBUG "value": [ 05:41:57 DEBUG 1713501717, 05:41:57 DEBUG "0" 05:41:57 DEBUG ] 05:41:57 DEBUG } 05:41:57 DEBUG ] 05:41:57 DEBUG } 05:41:57 DEBUG } 05:41:57 INFO [http_cmd]: http status code OK 05:41:57 DEBUG --- status code --- 05:41:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:41:57 DEBUG --- http response --- 05:41:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:41:57 DEBUG ----- output ----- 05:41:57 DEBUG { 05:41:57 DEBUG "status": "success", 05:41:57 DEBUG "data": { 05:41:57 DEBUG "resultType": "vector", 05:41:57 DEBUG "result": [ 05:41:57 DEBUG { 05:41:57 DEBUG "metric": { 05:41:57 DEBUG "pod": "ds-idrepo-0" 05:41:57 DEBUG }, 05:41:57 DEBUG "value": [ 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG [...] 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG "pod": "ds-idrepo-2" 05:41:57 DEBUG }, 05:41:57 DEBUG "value": [ 05:41:57 DEBUG 1713501717, 05:41:57 DEBUG "0" 05:41:57 DEBUG ] 05:41:57 DEBUG } 05:41:57 DEBUG ] 05:41:57 DEBUG } 05:41:57 DEBUG } 05:41:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501717" 05:41:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501717" 05:41:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501717" 05:41:57 INFO [http_cmd]: http status code OK 05:41:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501717" 05:41:57 DEBUG --- status code --- 05:41:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:41:57 DEBUG --- http response --- 05:41:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:41:57 DEBUG ----- output ----- 05:41:57 DEBUG { 05:41:57 DEBUG "status": "success", 05:41:57 DEBUG "data": { 05:41:57 DEBUG "resultType": "vector", 05:41:57 DEBUG "result": [ 05:41:57 DEBUG { 05:41:57 DEBUG "metric": { 05:41:57 DEBUG "instance": "10.142.0.107:9100" 05:41:57 DEBUG }, 05:41:57 DEBUG "value": [ 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG [...] 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG "instance": "10.142.0.43:9100" 05:41:57 DEBUG }, 05:41:57 DEBUG "value": [ 05:41:57 DEBUG 1713501717, 05:41:57 DEBUG "0.33500000000000274" 05:41:57 DEBUG ] 05:41:57 DEBUG } 05:41:57 DEBUG ] 05:41:57 DEBUG } 05:41:57 DEBUG } 05:41:57 INFO [http_cmd]: http status code OK 05:41:57 DEBUG --- status code --- 05:41:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:41:57 DEBUG --- http response --- 05:41:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:41:57 DEBUG ----- output ----- 05:41:57 DEBUG { 05:41:57 DEBUG "status": "success", 05:41:57 DEBUG "data": { 05:41:57 DEBUG "resultType": "vector", 05:41:57 DEBUG "result": [ 05:41:57 DEBUG { 05:41:57 DEBUG "metric": { 05:41:57 DEBUG "pod": "ds-idrepo-0" 05:41:57 DEBUG }, 05:41:57 DEBUG "value": [ 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG [...] 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG "pod": "ds-idrepo-2" 05:41:57 DEBUG }, 05:41:57 DEBUG "value": [ 05:41:57 DEBUG 1713501717, 05:41:57 DEBUG "0" 05:41:57 DEBUG ] 05:41:57 DEBUG } 05:41:57 DEBUG ] 05:41:57 DEBUG } 05:41:57 DEBUG } 05:41:57 INFO [http_cmd]: http status code OK 05:41:57 DEBUG --- status code --- 05:41:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:41:57 DEBUG --- http response --- 05:41:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:41:57 DEBUG ----- output ----- 05:41:57 DEBUG { 05:41:57 DEBUG "status": "success", 05:41:57 DEBUG "data": { 05:41:57 DEBUG "resultType": "vector", 05:41:57 DEBUG "result": [ 05:41:57 DEBUG { 05:41:57 DEBUG "metric": { 05:41:57 DEBUG "instance": "10.142.0.107:9100" 05:41:57 DEBUG }, 05:41:57 DEBUG "value": [ 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG [...] 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG "instance": "10.142.0.43:9100" 05:41:57 DEBUG }, 05:41:57 DEBUG "value": [ 05:41:57 DEBUG 1713501717, 05:41:57 DEBUG "0.5366999999999715" 05:41:57 DEBUG ] 05:41:57 DEBUG } 05:41:57 DEBUG ] 05:41:57 DEBUG } 05:41:57 DEBUG } 05:41:57 INFO [http_cmd]: http status code OK 05:41:57 DEBUG --- status code --- 05:41:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:41:57 DEBUG --- http response --- 05:41:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:41:57 DEBUG ----- output ----- 05:41:57 DEBUG { 05:41:57 DEBUG "status": "success", 05:41:57 DEBUG "data": { 05:41:57 DEBUG "resultType": "vector", 05:41:57 DEBUG "result": [ 05:41:57 DEBUG { 05:41:57 DEBUG "metric": { 05:41:57 DEBUG "pod": "ds-cts-0" 05:41:57 DEBUG }, 05:41:57 DEBUG "value": [ 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG [...] 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG "pod": "ds-cts-2" 05:41:57 DEBUG }, 05:41:57 DEBUG "value": [ 05:41:57 DEBUG 1713501717, 05:41:57 DEBUG "30" 05:41:57 DEBUG ] 05:41:57 DEBUG } 05:41:57 DEBUG ] 05:41:57 DEBUG } 05:41:57 DEBUG } 05:41:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501717" 05:41:57 INFO [http_cmd]: http status code OK 05:41:57 DEBUG --- status code --- 05:41:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:41:57 DEBUG --- http response --- 05:41:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:41:57 DEBUG ----- output ----- 05:41:57 DEBUG { 05:41:57 DEBUG "status": "success", 05:41:57 DEBUG "data": { 05:41:57 DEBUG "resultType": "vector", 05:41:57 DEBUG "result": [ 05:41:57 DEBUG { 05:41:57 DEBUG "metric": { 05:41:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:41:57 DEBUG }, 05:41:57 DEBUG "value": [ 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG [...] 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:41:57 DEBUG }, 05:41:57 DEBUG "value": [ 05:41:57 DEBUG 1713501717, 05:41:57 DEBUG "2172" 05:41:57 DEBUG ] 05:41:57 DEBUG } 05:41:57 DEBUG ] 05:41:57 DEBUG } 05:41:57 DEBUG } 05:41:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501717" 05:41:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713501717" 05:41:57 INFO [http_cmd]: http status code OK 05:41:57 DEBUG --- status code --- 05:41:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:41:57 DEBUG --- http response --- 05:41:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:41:57 DEBUG ----- output ----- 05:41:57 DEBUG { 05:41:57 DEBUG "status": "success", 05:41:57 DEBUG "data": { 05:41:57 DEBUG "resultType": "vector", 05:41:57 DEBUG "result": [ 05:41:57 DEBUG { 05:41:57 DEBUG "metric": { 05:41:57 DEBUG "pod": "ds-idrepo-1" 05:41:57 DEBUG }, 05:41:57 DEBUG "value": [ 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG [...] 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:41:57 DEBUG }, 05:41:57 DEBUG "value": [ 05:41:57 DEBUG 1713501717, 05:41:57 DEBUG "0" 05:41:57 DEBUG ] 05:41:57 DEBUG } 05:41:57 DEBUG ] 05:41:57 DEBUG } 05:41:57 DEBUG } 05:41:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501717" 05:41:57 INFO [http_cmd]: http status code OK 05:41:57 DEBUG --- status code --- 05:41:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:41:57 DEBUG --- http response --- 05:41:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:41:57 DEBUG ----- output ----- 05:41:57 DEBUG { 05:41:57 DEBUG "status": "success", 05:41:57 DEBUG "data": { 05:41:57 DEBUG "resultType": "vector", 05:41:57 DEBUG "result": [ 05:41:57 DEBUG { 05:41:57 DEBUG "metric": { 05:41:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:41:57 DEBUG }, 05:41:57 DEBUG "value": [ 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG [...] 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:41:57 DEBUG }, 05:41:57 DEBUG "value": [ 05:41:57 DEBUG 1713501717, 05:41:57 DEBUG "2277236736" 05:41:57 DEBUG ] 05:41:57 DEBUG } 05:41:57 DEBUG ] 05:41:57 DEBUG } 05:41:57 DEBUG } 05:41:57 INFO [http_cmd]: http status code OK 05:41:57 DEBUG --- status code --- 05:41:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:41:57 DEBUG --- http response --- 05:41:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:41:57 DEBUG ----- output ----- 05:41:57 DEBUG { 05:41:57 DEBUG "status": "success", 05:41:57 DEBUG "data": { 05:41:57 DEBUG "resultType": "vector", 05:41:57 DEBUG "result": [ 05:41:57 DEBUG { 05:41:57 DEBUG "metric": { 05:41:57 DEBUG "instance": "10.142.0.107:9100" 05:41:57 DEBUG }, 05:41:57 DEBUG "value": [ 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG [...] 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG "instance": "10.142.0.43:9100" 05:41:57 DEBUG }, 05:41:57 DEBUG "value": [ 05:41:57 DEBUG 1713501717, 05:41:57 DEBUG "64294848.5" 05:41:57 DEBUG ] 05:41:57 DEBUG } 05:41:57 DEBUG ] 05:41:57 DEBUG } 05:41:57 DEBUG } 05:41:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501717" 05:41:57 INFO [http_cmd]: http status code OK 05:41:57 DEBUG --- status code --- 05:41:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:41:57 DEBUG --- http response --- 05:41:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:41:57 DEBUG ----- output ----- 05:41:57 DEBUG { 05:41:57 DEBUG "status": "success", 05:41:57 DEBUG "data": { 05:41:57 DEBUG "resultType": "vector", 05:41:57 DEBUG "result": [ 05:41:57 DEBUG { 05:41:57 DEBUG "metric": { 05:41:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:41:57 DEBUG }, 05:41:57 DEBUG "value": [ 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG [...] 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:41:57 DEBUG }, 05:41:57 DEBUG "value": [ 05:41:57 DEBUG 1713501717, 05:41:57 DEBUG "2171.9" 05:41:57 DEBUG ] 05:41:57 DEBUG } 05:41:57 DEBUG ] 05:41:57 DEBUG } 05:41:57 DEBUG } 05:41:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501717" 05:41:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501717" 05:41:57 INFO [http_cmd]: http status code OK 05:41:57 INFO [http_cmd]: http status code OK 05:41:57 DEBUG --- status code --- 05:41:57 DEBUG --- status code --- 05:41:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:41:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:41:57 DEBUG --- http response --- 05:41:57 DEBUG --- http response --- 05:41:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:41:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:41:57 DEBUG ----- output ----- 05:41:57 DEBUG ----- output ----- 05:41:57 DEBUG { 05:41:57 DEBUG { 05:41:57 DEBUG "status": "success", 05:41:57 DEBUG "status": "success", 05:41:57 DEBUG "data": { 05:41:57 DEBUG "data": { 05:41:57 DEBUG "resultType": "vector", 05:41:57 DEBUG "resultType": "vector", 05:41:57 DEBUG "result": [ 05:41:57 DEBUG "result": [ 05:41:57 DEBUG { 05:41:57 DEBUG { 05:41:57 DEBUG "metric": { 05:41:57 DEBUG "metric": { 05:41:57 DEBUG "pod": "ds-idrepo-1" 05:41:57 DEBUG "instance": "10.142.0.107:9100" 05:41:57 DEBUG }, 05:41:57 DEBUG }, 05:41:57 DEBUG "value": [ 05:41:57 DEBUG "value": [ 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG [...] 05:41:57 DEBUG [...] 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG 05:41:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:41:57 DEBUG "instance": "10.142.0.43:9100" 05:41:57 DEBUG }, 05:41:57 DEBUG }, 05:41:57 DEBUG "value": [ 05:41:57 DEBUG "value": [ 05:41:57 DEBUG 1713501717, 05:41:57 DEBUG 1713501717, 05:41:57 DEBUG "0" 05:41:57 DEBUG "64292281.96666667" 05:41:57 DEBUG ] 05:41:57 DEBUG ] 05:41:57 DEBUG } 05:41:57 DEBUG } 05:41:57 DEBUG ] 05:41:57 DEBUG ] 05:41:57 DEBUG } 05:41:57 DEBUG } 05:41:57 DEBUG } 05:41:57 DEBUG } 05:41:59 INFO 05:41:59 INFO [loop_until]: kubectl --namespace=xlou top pods 05:41:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:42:00 INFO [loop_until]: OK (rc = 0) 05:42:00 DEBUG --- stdout --- 05:42:00 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 14012m 9144Mi am-6fd554f8f7-hvvkw 13498m 8910Mi am-6fd554f8f7-pmj4b 14197m 8799Mi ds-cts-0 19210m 12685Mi ds-cts-1 18623m 12844Mi ds-cts-2 20053m 13075Mi ds-idrepo-0 5696m 3269Mi ds-idrepo-1 5801m 3199Mi ds-idrepo-2 5635m 3066Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 4m 1215Mi lodemon-6b89d774d5-rf7sk 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7031m 2082Mi 05:42:00 DEBUG --- stderr --- 05:42:00 DEBUG 05:42:00 INFO 05:42:00 INFO [loop_until]: kubectl --namespace=xlou top node 05:42:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:42:00 INFO [loop_until]: OK (rc = 0) 05:42:00 DEBUG --- stdout --- 05:42:00 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 71m 0% 6240Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5833m 19% 6710Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13262m 44% 11615Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 81m 0% 3245Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 19413m 65% 15313Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 154m 0% 5163Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 13878m 46% 11938Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 11320m 37% 2723Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 15237m 51% 11010Mi 9% gke-xlou-cdm-ds-0f601751-j488 6205m 20% 6173Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 18770m 62% 16924Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6376m 21% 6886Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 19956m 66% 15006Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 7080m 44% 5772Mi 9% 05:42:00 DEBUG --- stderr --- 05:42:00 DEBUG 05:42:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713501777" 05:42:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501777" 05:42:57 INFO [http_cmd]: http status code OK 05:42:57 INFO [http_cmd]: http status code OK 05:42:57 DEBUG --- status code --- 05:42:57 DEBUG --- status code --- 05:42:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:42:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:42:57 DEBUG --- http response --- 05:42:57 DEBUG --- http response --- 05:42:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:42:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:42:57 DEBUG ----- output ----- 05:42:57 DEBUG ----- output ----- 05:42:57 DEBUG { 05:42:57 DEBUG { 05:42:57 DEBUG "status": "success", 05:42:57 DEBUG "status": "success", 05:42:57 DEBUG "data": { 05:42:57 DEBUG "data": { 05:42:57 DEBUG "resultType": "vector", 05:42:57 DEBUG "resultType": "vector", 05:42:57 DEBUG "result": [ 05:42:57 DEBUG "result": [ 05:42:57 DEBUG { 05:42:57 DEBUG { 05:42:57 DEBUG "metric": { 05:42:57 DEBUG "metric": { 05:42:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:42:57 DEBUG "instance": "10.142.0.107:9100" 05:42:57 DEBUG }, 05:42:57 DEBUG }, 05:42:57 DEBUG "value": [ 05:42:57 DEBUG "value": [ 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG [...] 05:42:57 DEBUG [...] 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501777" 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:42:57 DEBUG "instance": "10.142.0.43:9100" 05:42:57 DEBUG }, 05:42:57 DEBUG }, 05:42:57 DEBUG "value": [ 05:42:57 DEBUG "value": [ 05:42:57 DEBUG 1713501777, 05:42:57 DEBUG 1713501777, 05:42:57 DEBUG "55.175" 05:42:57 DEBUG "556750277632" 05:42:57 DEBUG ] 05:42:57 DEBUG ] 05:42:57 DEBUG } 05:42:57 DEBUG } 05:42:57 DEBUG ] 05:42:57 DEBUG ] 05:42:57 DEBUG } 05:42:57 DEBUG } 05:42:57 DEBUG } 05:42:57 DEBUG } 05:42:57 INFO [http_cmd]: http status code OK 05:42:57 DEBUG --- status code --- 05:42:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:42:57 DEBUG --- http response --- 05:42:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:42:57 DEBUG ----- output ----- 05:42:57 DEBUG { 05:42:57 DEBUG "status": "success", 05:42:57 DEBUG "data": { 05:42:57 DEBUG "resultType": "vector", 05:42:57 DEBUG "result": [ 05:42:57 DEBUG { 05:42:57 DEBUG "metric": { 05:42:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:42:57 DEBUG }, 05:42:57 DEBUG "value": [ 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG [...] 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:42:57 DEBUG }, 05:42:57 DEBUG "value": [ 05:42:57 DEBUG 1713501777, 05:42:57 DEBUG "0" 05:42:57 DEBUG ] 05:42:57 DEBUG } 05:42:57 DEBUG ] 05:42:57 DEBUG } 05:42:57 DEBUG } 05:42:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501777" 05:42:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501777" 05:42:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713501777" 05:42:57 INFO [http_cmd]: http status code OK 05:42:57 DEBUG --- status code --- 05:42:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:42:57 DEBUG --- http response --- 05:42:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:42:57 INFO [http_cmd]: http status code OK 05:42:57 INFO [http_cmd]: http status code OK 05:42:57 DEBUG --- status code --- 05:42:57 DEBUG --- status code --- 05:42:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:42:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:42:57 DEBUG --- http response --- 05:42:57 DEBUG --- http response --- 05:42:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:42:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 05:42:57 DEBUG ----- output ----- 05:42:57 DEBUG ----- output ----- 05:42:57 DEBUG { 05:42:57 DEBUG { 05:42:57 DEBUG "status": "success", 05:42:57 DEBUG "status": "success", 05:42:57 DEBUG "data": { 05:42:57 DEBUG "data": { 05:42:57 DEBUG "resultType": "vector", 05:42:57 DEBUG "resultType": "vector", 05:42:57 DEBUG "result": [ 05:42:57 DEBUG "result": [ 05:42:57 DEBUG { 05:42:57 DEBUG { 05:42:57 DEBUG "metric": { 05:42:57 DEBUG "metric": { 05:42:57 DEBUG "pod": "ds-idrepo-0" 05:42:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:42:57 DEBUG }, 05:42:57 DEBUG }, 05:42:57 DEBUG "value": [ 05:42:57 DEBUG "value": [ 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG [...] 05:42:57 DEBUG [...] 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG "pod": "ds-cts-2" 05:42:57 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 05:42:57 DEBUG }, 05:42:57 DEBUG }, 05:42:57 DEBUG "value": [ 05:42:57 DEBUG "value": [ 05:42:57 DEBUG 1713501777, 05:42:57 DEBUG 1713501777, 05:42:57 DEBUG "0" 05:42:57 DEBUG "181574.4325785501" 05:42:57 DEBUG ] 05:42:57 DEBUG ] 05:42:57 DEBUG } 05:42:57 DEBUG } 05:42:57 DEBUG ] 05:42:57 DEBUG ] 05:42:57 DEBUG } 05:42:57 DEBUG } 05:42:57 DEBUG } 05:42:57 DEBUG } 05:42:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501777" 05:42:57 INFO [http_cmd]: http status code OK 05:42:57 DEBUG --- status code --- 05:42:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:42:57 DEBUG --- http response --- 05:42:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:42:57 DEBUG ----- output ----- 05:42:57 DEBUG { 05:42:57 DEBUG "status": "success", 05:42:57 DEBUG "data": { 05:42:57 DEBUG "resultType": "vector", 05:42:57 DEBUG "result": [ 05:42:57 DEBUG { 05:42:57 DEBUG "metric": { 05:42:57 DEBUG "pod": "ds-cts-0" 05:42:57 DEBUG }, 05:42:57 DEBUG "value": [ 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG [...] 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG "pod": "ds-cts-2" 05:42:57 DEBUG }, 05:42:57 DEBUG "value": [ 05:42:57 DEBUG 1713501777, 05:42:57 DEBUG "165.5" 05:42:57 DEBUG ] 05:42:57 DEBUG } 05:42:57 DEBUG ] 05:42:57 DEBUG } 05:42:57 DEBUG } 05:42:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713501777" 05:42:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501777" 05:42:57 ERROR [http_cmd]: ERROR 05:42:57 ERROR --- status code --- 05:42:57 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:42:57 ERROR --- http response --- 05:42:57 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:42:57 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:42:57 INFO [http_cmd]: http status code OK 05:42:57 DEBUG --- status code --- 05:42:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:42:57 DEBUG --- http response --- 05:42:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:42:57 DEBUG ----- output ----- 05:42:57 DEBUG { 05:42:57 DEBUG "status": "success", 05:42:57 DEBUG "data": { 05:42:57 DEBUG "resultType": "vector", 05:42:57 DEBUG "result": [ 05:42:57 DEBUG { 05:42:57 DEBUG "metric": { 05:42:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:42:57 DEBUG }, 05:42:57 DEBUG "value": [ 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG [...] 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:42:57 DEBUG }, 05:42:57 DEBUG "value": [ 05:42:57 DEBUG 1713501777, 05:42:57 DEBUG "2168.7666666666664" 05:42:57 DEBUG ] 05:42:57 DEBUG } 05:42:57 DEBUG ] 05:42:57 DEBUG } 05:42:57 DEBUG } 05:42:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501777" 05:42:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501777" 05:42:57 INFO [http_cmd]: http status code OK 05:42:57 DEBUG --- status code --- 05:42:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:42:57 DEBUG --- http response --- 05:42:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:42:57 DEBUG ----- output ----- 05:42:57 DEBUG { 05:42:57 DEBUG "status": "success", 05:42:57 DEBUG "data": { 05:42:57 DEBUG "resultType": "vector", 05:42:57 DEBUG "result": [ 05:42:57 DEBUG { 05:42:57 DEBUG "metric": { 05:42:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:42:57 DEBUG }, 05:42:57 DEBUG "value": [ 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG [...] 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:42:57 DEBUG }, 05:42:57 DEBUG "value": [ 05:42:57 DEBUG 1713501777, 05:42:57 DEBUG "3752460" 05:42:57 DEBUG ] 05:42:57 DEBUG } 05:42:57 DEBUG ] 05:42:57 DEBUG } 05:42:57 DEBUG } 05:42:57 INFO [http_cmd]: http status code OK 05:42:57 DEBUG --- status code --- 05:42:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:42:57 DEBUG --- http response --- 05:42:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:42:57 DEBUG ----- output ----- 05:42:57 DEBUG { 05:42:57 DEBUG "status": "success", 05:42:57 DEBUG "data": { 05:42:57 DEBUG "resultType": "vector", 05:42:57 DEBUG "result": [ 05:42:57 DEBUG { 05:42:57 DEBUG "metric": { 05:42:57 DEBUG "pod": "ds-idrepo-0" 05:42:57 DEBUG }, 05:42:57 DEBUG "value": [ 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG [...] 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG "pod": "ds-idrepo-2" 05:42:57 DEBUG }, 05:42:57 DEBUG "value": [ 05:42:57 DEBUG 1713501777, 05:42:57 DEBUG "0" 05:42:57 DEBUG ] 05:42:57 DEBUG } 05:42:57 DEBUG ] 05:42:57 DEBUG } 05:42:57 DEBUG } 05:42:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501777" 05:42:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501777" 05:42:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713501777" 05:42:57 INFO [http_cmd]: http status code OK 05:42:57 INFO [http_cmd]: http status code OK 05:42:57 DEBUG --- status code --- 05:42:57 DEBUG --- status code --- 05:42:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:42:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:42:57 DEBUG --- http response --- 05:42:57 DEBUG --- http response --- 05:42:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:42:57 DEBUG ----- output ----- 05:42:57 DEBUG { 05:42:57 DEBUG "status": "success", 05:42:57 DEBUG "data": { 05:42:57 DEBUG "resultType": "vector", 05:42:57 DEBUG "result": [ 05:42:57 DEBUG { 05:42:57 DEBUG "metric": { 05:42:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:42:57 DEBUG }, 05:42:57 DEBUG "value": [ 05:42:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:42:57 DEBUG 05:42:57 DEBUG ----- output ----- 05:42:57 DEBUG 05:42:57 DEBUG { 05:42:57 DEBUG 05:42:57 DEBUG "status": "success", 05:42:57 DEBUG [...] 05:42:57 DEBUG "data": { 05:42:57 DEBUG 05:42:57 DEBUG "resultType": "vector", 05:42:57 DEBUG 05:42:57 DEBUG "result": [ 05:42:57 DEBUG 05:42:57 DEBUG { 05:42:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:42:57 DEBUG "metric": { 05:42:57 DEBUG }, 05:42:57 DEBUG "pod": "ds-cts-0" 05:42:57 DEBUG "value": [ 05:42:57 DEBUG }, 05:42:57 DEBUG 1713501777, 05:42:57 DEBUG "value": [ 05:42:57 DEBUG "2000.3999999999999" 05:42:57 DEBUG 05:42:57 DEBUG ] 05:42:57 DEBUG 05:42:57 DEBUG } 05:42:57 DEBUG 05:42:57 DEBUG ] 05:42:57 DEBUG [...] 05:42:57 DEBUG } 05:42:57 DEBUG 05:42:57 DEBUG } 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG "pod": "ds-cts-2" 05:42:57 DEBUG }, 05:42:57 DEBUG "value": [ 05:42:57 DEBUG 1713501777, 05:42:57 DEBUG "0.0005252533333333333" 05:42:57 DEBUG ] 05:42:57 DEBUG } 05:42:57 DEBUG ] 05:42:57 DEBUG } 05:42:57 DEBUG } 05:42:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501777" 05:42:57 INFO [http_cmd]: http status code OK 05:42:57 DEBUG --- status code --- 05:42:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:42:57 DEBUG --- http response --- 05:42:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:42:57 DEBUG ----- output ----- 05:42:57 DEBUG { 05:42:57 DEBUG "status": "success", 05:42:57 DEBUG "data": { 05:42:57 DEBUG "resultType": "vector", 05:42:57 DEBUG "result": [ 05:42:57 DEBUG { 05:42:57 DEBUG "metric": { 05:42:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:42:57 DEBUG }, 05:42:57 DEBUG "value": [ 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG [...] 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:42:57 DEBUG }, 05:42:57 DEBUG "value": [ 05:42:57 DEBUG 1713501777, 05:42:57 DEBUG "7.1174165113379315" 05:42:57 DEBUG ] 05:42:57 DEBUG } 05:42:57 DEBUG ] 05:42:57 DEBUG } 05:42:57 DEBUG } 05:42:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501777" 05:42:57 INFO [http_cmd]: http status code OK 05:42:57 DEBUG --- status code --- 05:42:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:42:57 DEBUG --- http response --- 05:42:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 05:42:57 DEBUG ----- output ----- 05:42:57 DEBUG { 05:42:57 DEBUG "status": "success", 05:42:57 DEBUG "data": { 05:42:57 DEBUG "resultType": "vector", 05:42:57 DEBUG "result": [ 05:42:57 DEBUG { 05:42:57 DEBUG "metric": { 05:42:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:42:57 DEBUG }, 05:42:57 DEBUG "value": [ 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG [...] 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 05:42:57 DEBUG }, 05:42:57 DEBUG "value": [ 05:42:57 DEBUG 1713501777, 05:42:57 DEBUG "0.000015413393013864386" 05:42:57 DEBUG ] 05:42:57 DEBUG } 05:42:57 DEBUG ] 05:42:57 DEBUG } 05:42:57 DEBUG } 05:42:57 INFO [http_cmd]: http status code OK 05:42:57 DEBUG --- status code --- 05:42:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:42:57 DEBUG --- http response --- 05:42:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:42:57 DEBUG ----- output ----- 05:42:57 DEBUG { 05:42:57 DEBUG "status": "success", 05:42:57 DEBUG "data": { 05:42:57 DEBUG "resultType": "vector", 05:42:57 DEBUG "result": [ 05:42:57 DEBUG { 05:42:57 DEBUG "metric": { 05:42:57 DEBUG "pod": "ds-idrepo-0" 05:42:57 DEBUG }, 05:42:57 DEBUG "value": [ 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG [...] 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG "pod": "ds-idrepo-2" 05:42:57 DEBUG }, 05:42:57 DEBUG "value": [ 05:42:57 DEBUG 1713501777, 05:42:57 DEBUG "0" 05:42:57 DEBUG ] 05:42:57 DEBUG } 05:42:57 DEBUG ] 05:42:57 DEBUG } 05:42:57 DEBUG } 05:42:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501777" 05:42:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501777" 05:42:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501777" 05:42:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501777" 05:42:57 INFO [http_cmd]: http status code OK 05:42:57 INFO [http_cmd]: http status code OK 05:42:57 DEBUG --- status code --- 05:42:57 DEBUG --- status code --- 05:42:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:42:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:42:57 DEBUG --- http response --- 05:42:57 DEBUG --- http response --- 05:42:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:42:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713501777" 05:42:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:42:57 DEBUG ----- output ----- 05:42:57 DEBUG ----- output ----- 05:42:57 DEBUG { 05:42:57 DEBUG { 05:42:57 DEBUG "status": "success", 05:42:57 DEBUG "status": "success", 05:42:57 DEBUG "data": { 05:42:57 DEBUG "data": { 05:42:57 DEBUG "resultType": "vector", 05:42:57 DEBUG "resultType": "vector", 05:42:57 DEBUG "result": [ 05:42:57 DEBUG "result": [ 05:42:57 DEBUG { 05:42:57 DEBUG { 05:42:57 DEBUG "metric": { 05:42:57 DEBUG "metric": { 05:42:57 DEBUG "pod": "ds-idrepo-0" 05:42:57 DEBUG "instance": "10.142.0.107:9100" 05:42:57 DEBUG }, 05:42:57 DEBUG }, 05:42:57 DEBUG "value": [ 05:42:57 DEBUG "value": [ 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 INFO [http_cmd]: http status code OK 05:42:57 DEBUG 05:42:57 DEBUG [...] 05:42:57 DEBUG --- status code --- 05:42:57 DEBUG [...] 05:42:57 DEBUG 05:42:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG --- http response --- 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:42:57 DEBUG "pod": "ds-idrepo-2" 05:42:57 DEBUG "instance": "10.142.0.43:9100" 05:42:57 DEBUG ----- output ----- 05:42:57 DEBUG }, 05:42:57 DEBUG }, 05:42:57 DEBUG { 05:42:57 DEBUG "value": [ 05:42:57 DEBUG "value": [ 05:42:57 DEBUG "status": "success", 05:42:57 DEBUG 1713501777, 05:42:57 DEBUG 1713501777, 05:42:57 DEBUG "data": { 05:42:57 DEBUG "0" 05:42:57 DEBUG "0.22933333333333603" 05:42:57 DEBUG "resultType": "vector", 05:42:57 DEBUG ] 05:42:57 DEBUG ] 05:42:57 DEBUG "result": [ 05:42:57 DEBUG } 05:42:57 DEBUG } 05:42:57 DEBUG { 05:42:57 DEBUG ] 05:42:57 DEBUG ] 05:42:57 DEBUG "metric": { 05:42:57 DEBUG } 05:42:57 DEBUG } 05:42:57 DEBUG "instance": "10.142.0.107:9100" 05:42:57 DEBUG } 05:42:57 DEBUG } 05:42:57 DEBUG }, 05:42:57 INFO [http_cmd]: http status code OK 05:42:57 DEBUG "value": [ 05:42:57 DEBUG --- status code --- 05:42:57 DEBUG 05:42:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:42:57 DEBUG 05:42:57 DEBUG --- http response --- 05:42:57 DEBUG 05:42:57 DEBUG [...] 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG "instance": "10.142.0.43:9100" 05:42:57 DEBUG }, 05:42:57 DEBUG "value": [ 05:42:57 DEBUG 1713501777, 05:42:57 DEBUG "0.4823666666666743" 05:42:57 DEBUG ] 05:42:57 DEBUG } 05:42:57 DEBUG ] 05:42:57 DEBUG } 05:42:57 DEBUG } 05:42:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:42:57 DEBUG ----- output ----- 05:42:57 DEBUG { 05:42:57 DEBUG "status": "success", 05:42:57 DEBUG "data": { 05:42:57 DEBUG "resultType": "vector", 05:42:57 DEBUG "result": [ 05:42:57 DEBUG { 05:42:57 DEBUG "metric": { 05:42:57 DEBUG "pod": "ds-cts-0" 05:42:57 DEBUG }, 05:42:57 DEBUG "value": [ 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG [...] 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG "pod": "ds-cts-2" 05:42:57 DEBUG }, 05:42:57 DEBUG "value": [ 05:42:57 DEBUG 1713501777, 05:42:57 DEBUG "28.5" 05:42:57 DEBUG ] 05:42:57 DEBUG } 05:42:57 DEBUG ] 05:42:57 DEBUG } 05:42:57 DEBUG } 05:42:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501777" 05:42:57 INFO [http_cmd]: http status code OK 05:42:57 DEBUG --- status code --- 05:42:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:42:57 DEBUG --- http response --- 05:42:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:42:57 DEBUG ----- output ----- 05:42:57 DEBUG { 05:42:57 DEBUG "status": "success", 05:42:57 DEBUG "data": { 05:42:57 DEBUG "resultType": "vector", 05:42:57 DEBUG "result": [ 05:42:57 DEBUG { 05:42:57 DEBUG "metric": { 05:42:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:42:57 DEBUG }, 05:42:57 DEBUG "value": [ 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG [...] 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:42:57 DEBUG }, 05:42:57 DEBUG "value": [ 05:42:57 DEBUG 1713501777, 05:42:57 DEBUG "2278830080" 05:42:57 DEBUG ] 05:42:57 DEBUG } 05:42:57 DEBUG ] 05:42:57 DEBUG } 05:42:57 DEBUG } 05:42:57 INFO [http_cmd]: http status code OK 05:42:57 DEBUG --- status code --- 05:42:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:42:57 DEBUG --- http response --- 05:42:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:42:57 DEBUG ----- output ----- 05:42:57 DEBUG { 05:42:57 DEBUG "status": "success", 05:42:57 DEBUG "data": { 05:42:57 DEBUG "resultType": "vector", 05:42:57 DEBUG "result": [ 05:42:57 DEBUG { 05:42:57 DEBUG "metric": { 05:42:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:42:57 DEBUG }, 05:42:57 DEBUG "value": [ 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG [...] 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:42:57 DEBUG }, 05:42:57 DEBUG "value": [ 05:42:57 DEBUG 1713501777, 05:42:57 DEBUG "2168.5666666666666" 05:42:57 DEBUG ] 05:42:57 DEBUG } 05:42:57 DEBUG ] 05:42:57 DEBUG } 05:42:57 DEBUG } 05:42:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501777" 05:42:57 INFO [http_cmd]: http status code OK 05:42:57 DEBUG --- status code --- 05:42:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:42:57 DEBUG --- http response --- 05:42:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 05:42:57 DEBUG ----- output ----- 05:42:57 DEBUG { 05:42:57 DEBUG "status": "success", 05:42:57 DEBUG "data": { 05:42:57 DEBUG "resultType": "vector", 05:42:57 DEBUG "result": [ 05:42:57 DEBUG { 05:42:57 DEBUG "metric": { 05:42:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:42:57 DEBUG }, 05:42:57 DEBUG "value": [ 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG [...] 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 05:42:57 DEBUG }, 05:42:57 DEBUG "value": [ 05:42:57 DEBUG 1713501777, 05:42:57 DEBUG "0" 05:42:57 DEBUG ] 05:42:57 DEBUG } 05:42:57 DEBUG ] 05:42:57 DEBUG } 05:42:57 DEBUG } 05:42:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501777" 05:42:57 INFO [http_cmd]: http status code OK 05:42:57 DEBUG --- status code --- 05:42:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:42:57 DEBUG --- http response --- 05:42:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:42:57 DEBUG ----- output ----- 05:42:57 DEBUG { 05:42:57 DEBUG "status": "success", 05:42:57 DEBUG "data": { 05:42:57 DEBUG "resultType": "vector", 05:42:57 DEBUG "result": [ 05:42:57 DEBUG { 05:42:57 DEBUG "metric": { 05:42:57 DEBUG "instance": "10.142.0.107:9100" 05:42:57 DEBUG }, 05:42:57 DEBUG "value": [ 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG [...] 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG "instance": "10.142.0.43:9100" 05:42:57 DEBUG }, 05:42:57 DEBUG "value": [ 05:42:57 DEBUG 1713501777, 05:42:57 DEBUG "64204218.53333333" 05:42:57 DEBUG ] 05:42:57 DEBUG } 05:42:57 DEBUG ] 05:42:57 DEBUG } 05:42:57 DEBUG } 05:42:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501777" 05:42:57 INFO [http_cmd]: http status code OK 05:42:57 DEBUG --- status code --- 05:42:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:42:57 DEBUG --- http response --- 05:42:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:42:57 DEBUG ----- output ----- 05:42:57 DEBUG { 05:42:57 DEBUG "status": "success", 05:42:57 DEBUG "data": { 05:42:57 DEBUG "resultType": "vector", 05:42:57 DEBUG "result": [ 05:42:57 DEBUG { 05:42:57 DEBUG "metric": { 05:42:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:42:57 DEBUG }, 05:42:57 DEBUG "value": [ 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG [...] 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:42:57 DEBUG }, 05:42:57 DEBUG "value": [ 05:42:57 DEBUG 1713501777, 05:42:57 DEBUG "2169" 05:42:57 DEBUG ] 05:42:57 DEBUG } 05:42:57 DEBUG ] 05:42:57 DEBUG } 05:42:57 DEBUG } 05:42:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501777" 05:42:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501777" 05:42:57 INFO [http_cmd]: http status code OK 05:42:57 DEBUG --- status code --- 05:42:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:42:57 DEBUG --- http response --- 05:42:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 05:42:57 DEBUG ----- output ----- 05:42:57 DEBUG { 05:42:57 DEBUG "status": "success", 05:42:57 DEBUG "data": { 05:42:57 DEBUG "resultType": "vector", 05:42:57 DEBUG "result": [ 05:42:57 DEBUG { 05:42:57 DEBUG "metric": { 05:42:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:42:57 DEBUG }, 05:42:57 DEBUG "value": [ 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG [...] 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 05:42:57 DEBUG }, 05:42:57 DEBUG "value": [ 05:42:57 DEBUG 1713501777, 05:42:57 DEBUG "0" 05:42:57 DEBUG ] 05:42:57 DEBUG } 05:42:57 DEBUG ] 05:42:57 DEBUG } 05:42:57 DEBUG } 05:42:57 INFO [http_cmd]: http status code OK 05:42:57 DEBUG --- status code --- 05:42:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:42:57 DEBUG --- http response --- 05:42:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:42:57 DEBUG ----- output ----- 05:42:57 DEBUG { 05:42:57 DEBUG "status": "success", 05:42:57 DEBUG "data": { 05:42:57 DEBUG "resultType": "vector", 05:42:57 DEBUG "result": [ 05:42:57 DEBUG { 05:42:57 DEBUG "metric": { 05:42:57 DEBUG "instance": "10.142.0.107:9100" 05:42:57 DEBUG }, 05:42:57 DEBUG "value": [ 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG [...] 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG 05:42:57 DEBUG "instance": "10.142.0.43:9100" 05:42:57 DEBUG }, 05:42:57 DEBUG "value": [ 05:42:57 DEBUG 1713501777, 05:42:57 DEBUG "64199503.33333333" 05:42:57 DEBUG ] 05:42:57 DEBUG } 05:42:57 DEBUG ] 05:42:57 DEBUG } 05:42:57 DEBUG } 05:43:00 INFO 05:43:00 INFO [loop_until]: kubectl --namespace=xlou top pods 05:43:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:43:00 INFO [loop_until]: OK (rc = 0) 05:43:00 DEBUG --- stdout --- 05:43:00 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 14262m 9146Mi am-6fd554f8f7-hvvkw 12766m 8911Mi am-6fd554f8f7-pmj4b 14317m 8796Mi ds-cts-0 19782m 13130Mi ds-cts-1 18459m 13044Mi ds-cts-2 19787m 13320Mi ds-idrepo-0 5773m 3269Mi ds-idrepo-1 5751m 3199Mi ds-idrepo-2 5683m 3066Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 3m 1215Mi lodemon-6b89d774d5-rf7sk 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7200m 2083Mi 05:43:00 DEBUG --- stderr --- 05:43:00 DEBUG 05:43:01 INFO 05:43:01 INFO [loop_until]: kubectl --namespace=xlou top node 05:43:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:43:01 INFO [loop_until]: OK (rc = 0) 05:43:01 DEBUG --- stdout --- 05:43:01 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 70m 0% 6238Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5792m 19% 6720Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13296m 44% 11627Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 78m 0% 3235Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 19188m 64% 15468Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 152m 0% 5175Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 13759m 46% 11928Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 11299m 37% 2731Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 15230m 51% 11002Mi 9% gke-xlou-cdm-ds-0f601751-j488 6187m 20% 6155Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 18837m 63% 17027Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6345m 21% 6878Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 19854m 66% 15275Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 7246m 45% 5773Mi 9% 05:43:01 DEBUG --- stderr --- 05:43:01 DEBUG 05:43:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713501837" 05:43:57 INFO [http_cmd]: http status code OK 05:43:57 DEBUG --- status code --- 05:43:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:43:57 DEBUG --- http response --- 05:43:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:43:57 DEBUG ----- output ----- 05:43:57 DEBUG { 05:43:57 DEBUG "status": "success", 05:43:57 DEBUG "data": { 05:43:57 DEBUG "resultType": "vector", 05:43:57 DEBUG "result": [ 05:43:57 DEBUG { 05:43:57 DEBUG "metric": { 05:43:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:43:57 DEBUG }, 05:43:57 DEBUG "value": [ 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG [...] 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:43:57 DEBUG }, 05:43:57 DEBUG "value": [ 05:43:57 DEBUG 1713501837, 05:43:57 DEBUG "55.175" 05:43:57 DEBUG ] 05:43:57 DEBUG } 05:43:57 DEBUG ] 05:43:57 DEBUG } 05:43:57 DEBUG } 05:43:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501837" 05:43:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501837" 05:43:57 INFO [http_cmd]: http status code OK 05:43:57 DEBUG --- status code --- 05:43:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:43:57 DEBUG --- http response --- 05:43:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:43:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713501837" 05:43:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501837" 05:43:57 INFO [http_cmd]: http status code OK 05:43:57 DEBUG --- status code --- 05:43:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:43:57 DEBUG --- http response --- 05:43:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:43:57 DEBUG ----- output ----- 05:43:57 DEBUG { 05:43:57 DEBUG "status": "success", 05:43:57 DEBUG "data": { 05:43:57 DEBUG "resultType": "vector", 05:43:57 DEBUG "result": [ 05:43:57 DEBUG { 05:43:57 DEBUG "metric": { 05:43:57 DEBUG "instance": "10.142.0.107:9100" 05:43:57 DEBUG }, 05:43:57 DEBUG "value": [ 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG [...] 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG "instance": "10.142.0.43:9100" 05:43:57 DEBUG }, 05:43:57 DEBUG "value": [ 05:43:57 DEBUG 1713501837, 05:43:57 DEBUG "560895566848" 05:43:57 DEBUG ] 05:43:57 DEBUG } 05:43:57 DEBUG ] 05:43:57 DEBUG } 05:43:57 DEBUG } 05:43:57 INFO [http_cmd]: http status code OK 05:43:57 INFO [http_cmd]: http status code OK 05:43:57 DEBUG --- status code --- 05:43:57 DEBUG --- status code --- 05:43:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:43:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:43:57 DEBUG --- http response --- 05:43:57 DEBUG --- http response --- 05:43:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:43:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:43:57 DEBUG ----- output ----- 05:43:57 DEBUG ----- output ----- 05:43:57 DEBUG { 05:43:57 DEBUG { 05:43:57 DEBUG "status": "success", 05:43:57 DEBUG "status": "success", 05:43:57 DEBUG "data": { 05:43:57 DEBUG "data": { 05:43:57 DEBUG "resultType": "vector", 05:43:57 DEBUG "resultType": "vector", 05:43:57 DEBUG "result": [ 05:43:57 DEBUG "result": [ 05:43:57 DEBUG { 05:43:57 DEBUG { 05:43:57 DEBUG "metric": { 05:43:57 DEBUG "metric": { 05:43:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:43:57 DEBUG "pod": "ds-idrepo-0" 05:43:57 DEBUG }, 05:43:57 DEBUG }, 05:43:57 DEBUG "value": [ 05:43:57 DEBUG "value": [ 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG [...] 05:43:57 DEBUG [...] 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:43:57 DEBUG "pod": "ds-cts-2" 05:43:57 DEBUG }, 05:43:57 DEBUG }, 05:43:57 DEBUG "value": [ 05:43:57 DEBUG "value": [ 05:43:57 DEBUG 1713501837, 05:43:57 DEBUG 1713501837, 05:43:57 DEBUG "0" 05:43:57 DEBUG "0" 05:43:57 DEBUG ] 05:43:57 DEBUG ] 05:43:57 DEBUG } 05:43:57 DEBUG } 05:43:57 DEBUG ] 05:43:57 DEBUG ] 05:43:57 DEBUG } 05:43:57 DEBUG } 05:43:57 DEBUG } 05:43:57 DEBUG } 05:43:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501837" 05:43:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501837" 05:43:57 INFO [http_cmd]: http status code OK 05:43:57 DEBUG --- status code --- 05:43:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:43:57 DEBUG --- http response --- 05:43:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 05:43:57 DEBUG ----- output ----- 05:43:57 DEBUG { 05:43:57 DEBUG "status": "success", 05:43:57 DEBUG "data": { 05:43:57 DEBUG "resultType": "vector", 05:43:57 DEBUG "result": [ 05:43:57 DEBUG { 05:43:57 DEBUG "metric": { 05:43:57 DEBUG "pod": "ds-idrepo-1" 05:43:57 DEBUG }, 05:43:57 DEBUG "value": [ 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG [...] 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG "pod": "ds-cts-0" 05:43:57 DEBUG }, 05:43:57 DEBUG "value": [ 05:43:57 DEBUG 1713501837, 05:43:57 DEBUG "432829910.11307126" 05:43:57 DEBUG ] 05:43:57 DEBUG } 05:43:57 DEBUG ] 05:43:57 DEBUG } 05:43:57 DEBUG } 05:43:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501837" 05:43:57 INFO [http_cmd]: http status code OK 05:43:57 DEBUG --- status code --- 05:43:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:43:57 DEBUG --- http response --- 05:43:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:43:57 DEBUG ----- output ----- 05:43:57 DEBUG { 05:43:57 DEBUG "status": "success", 05:43:57 DEBUG "data": { 05:43:57 DEBUG "resultType": "vector", 05:43:57 DEBUG "result": [ 05:43:57 DEBUG { 05:43:57 DEBUG "metric": { 05:43:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:43:57 DEBUG }, 05:43:57 DEBUG "value": [ 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG [...] 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:43:57 DEBUG }, 05:43:57 DEBUG "value": [ 05:43:57 DEBUG 1713501837, 05:43:57 DEBUG "2160.9" 05:43:57 DEBUG ] 05:43:57 DEBUG } 05:43:57 DEBUG ] 05:43:57 DEBUG } 05:43:57 DEBUG } 05:43:57 INFO [http_cmd]: http status code OK 05:43:57 DEBUG --- status code --- 05:43:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:43:57 DEBUG --- http response --- 05:43:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:43:57 DEBUG ----- output ----- 05:43:57 DEBUG { 05:43:57 DEBUG "status": "success", 05:43:57 DEBUG "data": { 05:43:57 DEBUG "resultType": "vector", 05:43:57 DEBUG "result": [ 05:43:57 DEBUG { 05:43:57 DEBUG "metric": { 05:43:57 DEBUG "pod": "ds-cts-0" 05:43:57 DEBUG }, 05:43:57 DEBUG "value": [ 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG [...] 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG "pod": "ds-cts-2" 05:43:57 DEBUG }, 05:43:57 DEBUG "value": [ 05:43:57 DEBUG 1713501837, 05:43:57 DEBUG "182" 05:43:57 DEBUG ] 05:43:57 DEBUG } 05:43:57 DEBUG ] 05:43:57 DEBUG } 05:43:57 DEBUG } 05:43:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713501837" 05:43:57 ERROR [http_cmd]: ERROR 05:43:57 ERROR --- status code --- 05:43:57 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:43:57 ERROR --- http response --- 05:43:57 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:43:57 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:43:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501837" 05:43:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501837" 05:43:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501837" 05:43:57 INFO [http_cmd]: http status code OK 05:43:57 INFO [http_cmd]: http status code OK 05:43:57 DEBUG --- status code --- 05:43:57 DEBUG --- status code --- 05:43:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:43:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:43:57 DEBUG --- http response --- 05:43:57 DEBUG --- http response --- 05:43:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:43:57 DEBUG ----- output ----- 05:43:57 DEBUG { 05:43:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:43:57 DEBUG "status": "success", 05:43:57 DEBUG ----- output ----- 05:43:57 DEBUG { 05:43:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501837" 05:43:57 DEBUG "data": { 05:43:57 INFO [http_cmd]: http status code OK 05:43:57 DEBUG "status": "success", 05:43:57 DEBUG "resultType": "vector", 05:43:57 DEBUG "result": [ 05:43:57 DEBUG --- status code --- 05:43:57 DEBUG "data": { 05:43:57 DEBUG { 05:43:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:43:57 DEBUG "resultType": "vector", 05:43:57 DEBUG "metric": { 05:43:57 DEBUG --- http response --- 05:43:57 DEBUG "result": [ 05:43:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:43:57 DEBUG { 05:43:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:43:57 DEBUG }, 05:43:57 DEBUG "metric": { 05:43:57 DEBUG ----- output ----- 05:43:57 DEBUG "value": [ 05:43:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:43:57 DEBUG { 05:43:57 DEBUG 05:43:57 DEBUG }, 05:43:57 DEBUG "status": "success", 05:43:57 DEBUG 05:43:57 DEBUG "value": [ 05:43:57 DEBUG "data": { 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG "resultType": "vector", 05:43:57 DEBUG [...] 05:43:57 DEBUG 05:43:57 DEBUG "result": [ 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG { 05:43:57 DEBUG 05:43:57 DEBUG [...] 05:43:57 DEBUG "metric": { 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG "pod": "ds-idrepo-0" 05:43:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:43:57 DEBUG 05:43:57 DEBUG }, 05:43:57 DEBUG }, 05:43:57 DEBUG 05:43:57 DEBUG "value": [ 05:43:57 DEBUG "value": [ 05:43:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:43:57 DEBUG 05:43:57 DEBUG 1713501837, 05:43:57 DEBUG }, 05:43:57 DEBUG 05:43:57 DEBUG "3762021" 05:43:57 DEBUG "value": [ 05:43:57 DEBUG 05:43:57 DEBUG ] 05:43:57 DEBUG 1713501837, 05:43:57 DEBUG [...] 05:43:57 DEBUG } 05:43:57 DEBUG "2004.6666666666667" 05:43:57 DEBUG 05:43:57 DEBUG ] 05:43:57 DEBUG ] 05:43:57 DEBUG 05:43:57 DEBUG } 05:43:57 DEBUG } 05:43:57 DEBUG 05:43:57 DEBUG } 05:43:57 DEBUG ] 05:43:57 DEBUG "pod": "ds-idrepo-2" 05:43:57 DEBUG } 05:43:57 DEBUG }, 05:43:57 DEBUG } 05:43:57 DEBUG "value": [ 05:43:57 DEBUG 1713501837, 05:43:57 DEBUG "0" 05:43:57 DEBUG ] 05:43:57 DEBUG } 05:43:57 DEBUG ] 05:43:57 DEBUG } 05:43:57 DEBUG } 05:43:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501837" 05:43:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501837" 05:43:57 INFO [http_cmd]: http status code OK 05:43:57 DEBUG --- status code --- 05:43:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:43:57 DEBUG --- http response --- 05:43:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:43:57 DEBUG ----- output ----- 05:43:57 DEBUG { 05:43:57 DEBUG "status": "success", 05:43:57 DEBUG "data": { 05:43:57 DEBUG "resultType": "vector", 05:43:57 DEBUG "result": [ 05:43:57 DEBUG { 05:43:57 DEBUG "metric": { 05:43:57 DEBUG "pod": "ds-cts-0" 05:43:57 DEBUG }, 05:43:57 DEBUG "value": [ 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG [...] 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG "pod": "ds-cts-2" 05:43:57 DEBUG }, 05:43:57 DEBUG "value": [ 05:43:57 DEBUG 1713501837, 05:43:57 DEBUG "0.0004878799999999999" 05:43:57 DEBUG ] 05:43:57 DEBUG } 05:43:57 DEBUG ] 05:43:57 DEBUG } 05:43:57 DEBUG } 05:43:57 INFO [http_cmd]: http status code OK 05:43:57 DEBUG --- status code --- 05:43:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:43:57 DEBUG --- http response --- 05:43:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 05:43:57 DEBUG ----- output ----- 05:43:57 DEBUG { 05:43:57 DEBUG "status": "success", 05:43:57 DEBUG "data": { 05:43:57 DEBUG "resultType": "vector", 05:43:57 DEBUG "result": [ 05:43:57 DEBUG { 05:43:57 DEBUG "metric": { 05:43:57 DEBUG "pod": "ds-idrepo-1" 05:43:57 DEBUG }, 05:43:57 DEBUG "value": [ 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG [...] 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG "pod": "ds-cts-0" 05:43:57 DEBUG }, 05:43:57 DEBUG "value": [ 05:43:57 DEBUG 1713501837, 05:43:57 DEBUG "19.553940677161325" 05:43:57 DEBUG ] 05:43:57 DEBUG } 05:43:57 DEBUG ] 05:43:57 DEBUG } 05:43:57 DEBUG } 05:43:57 INFO [http_cmd]: http status code OK 05:43:57 DEBUG --- status code --- 05:43:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:43:57 DEBUG --- http response --- 05:43:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:43:57 DEBUG ----- output ----- 05:43:57 DEBUG { 05:43:57 DEBUG "status": "success", 05:43:57 DEBUG "data": { 05:43:57 DEBUG "resultType": "vector", 05:43:57 DEBUG "result": [ 05:43:57 DEBUG { 05:43:57 DEBUG "metric": { 05:43:57 DEBUG "pod": "ds-idrepo-0" 05:43:57 DEBUG }, 05:43:57 DEBUG "value": [ 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG [...] 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG "pod": "ds-idrepo-2" 05:43:57 DEBUG }, 05:43:57 DEBUG "value": [ 05:43:57 DEBUG 1713501837, 05:43:57 DEBUG "0" 05:43:57 DEBUG ] 05:43:57 DEBUG } 05:43:57 DEBUG ] 05:43:57 DEBUG } 05:43:57 DEBUG } 05:43:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713501837" 05:43:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713501837" 05:43:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501837" 05:43:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501837" 05:43:57 INFO [http_cmd]: http status code OK 05:43:57 DEBUG --- status code --- 05:43:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:43:57 DEBUG --- http response --- 05:43:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:43:57 DEBUG ----- output ----- 05:43:57 DEBUG { 05:43:57 DEBUG "status": "success", 05:43:57 DEBUG "data": { 05:43:57 DEBUG "resultType": "vector", 05:43:57 DEBUG "result": [ 05:43:57 DEBUG { 05:43:57 DEBUG "metric": { 05:43:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:43:57 DEBUG }, 05:43:57 DEBUG "value": [ 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG [...] 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:43:57 DEBUG }, 05:43:57 DEBUG "value": [ 05:43:57 DEBUG 1713501837, 05:43:57 DEBUG "7.093447780187242" 05:43:57 DEBUG ] 05:43:57 DEBUG } 05:43:57 DEBUG ] 05:43:57 DEBUG } 05:43:57 DEBUG } 05:43:57 INFO [http_cmd]: http status code OK 05:43:57 INFO [http_cmd]: http status code OK 05:43:57 DEBUG --- status code --- 05:43:57 DEBUG --- status code --- 05:43:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:43:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:43:57 DEBUG --- http response --- 05:43:57 DEBUG --- http response --- 05:43:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:43:57 DEBUG ----- output ----- 05:43:57 DEBUG { 05:43:57 DEBUG "status": "success", 05:43:57 DEBUG "data": { 05:43:57 DEBUG "resultType": "vector", 05:43:57 DEBUG "result": [ 05:43:57 DEBUG { 05:43:57 DEBUG "metric": { 05:43:57 DEBUG "pod": "ds-idrepo-0" 05:43:57 DEBUG }, 05:43:57 DEBUG "value": [ 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG [...] 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG "pod": "ds-idrepo-2" 05:43:57 DEBUG }, 05:43:57 DEBUG "value": [ 05:43:57 DEBUG 1713501837, 05:43:57 DEBUG "0" 05:43:57 DEBUG ] 05:43:57 DEBUG } 05:43:57 DEBUG ] 05:43:57 DEBUG } 05:43:57 DEBUG } 05:43:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:43:57 DEBUG ----- output ----- 05:43:57 DEBUG { 05:43:57 DEBUG "status": "success", 05:43:57 DEBUG "data": { 05:43:57 DEBUG "resultType": "vector", 05:43:57 DEBUG "result": [ 05:43:57 DEBUG { 05:43:57 DEBUG "metric": { 05:43:57 DEBUG "pod": "ds-cts-0" 05:43:57 DEBUG }, 05:43:57 DEBUG "value": [ 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG [...] 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG "pod": "ds-cts-2" 05:43:57 DEBUG }, 05:43:57 DEBUG "value": [ 05:43:57 DEBUG 1713501837, 05:43:57 DEBUG "31.53333333333333" 05:43:57 DEBUG ] 05:43:57 DEBUG } 05:43:57 DEBUG ] 05:43:57 DEBUG } 05:43:57 DEBUG } 05:43:57 INFO [http_cmd]: http status code OK 05:43:57 DEBUG --- status code --- 05:43:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:43:57 DEBUG --- http response --- 05:43:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:43:57 DEBUG ----- output ----- 05:43:57 DEBUG { 05:43:57 DEBUG "status": "success", 05:43:57 DEBUG "data": { 05:43:57 DEBUG "resultType": "vector", 05:43:57 DEBUG "result": [ 05:43:57 DEBUG { 05:43:57 DEBUG "metric": { 05:43:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:43:57 DEBUG }, 05:43:57 DEBUG "value": [ 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG [...] 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:43:57 DEBUG }, 05:43:57 DEBUG "value": [ 05:43:57 DEBUG 1713501837, 05:43:57 DEBUG "2282512384" 05:43:57 DEBUG ] 05:43:57 DEBUG } 05:43:57 DEBUG ] 05:43:57 DEBUG } 05:43:57 DEBUG } 05:43:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501837" 05:43:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501837" 05:43:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501837" 05:43:57 INFO [http_cmd]: http status code OK 05:43:57 DEBUG --- status code --- 05:43:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:43:57 DEBUG --- http response --- 05:43:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:43:57 DEBUG ----- output ----- 05:43:57 DEBUG { 05:43:57 DEBUG "status": "success", 05:43:57 DEBUG "data": { 05:43:57 DEBUG "resultType": "vector", 05:43:57 DEBUG "result": [ 05:43:57 DEBUG { 05:43:57 DEBUG "metric": { 05:43:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:43:57 DEBUG }, 05:43:57 DEBUG "value": [ 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG [...] 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:43:57 DEBUG }, 05:43:57 DEBUG "value": [ 05:43:57 DEBUG 1713501837, 05:43:57 DEBUG "2161.0666666666666" 05:43:57 DEBUG ] 05:43:57 DEBUG } 05:43:57 DEBUG ] 05:43:57 DEBUG } 05:43:57 DEBUG } 05:43:57 INFO [http_cmd]: http status code OK 05:43:57 DEBUG --- status code --- 05:43:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:43:57 DEBUG --- http response --- 05:43:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:43:57 DEBUG ----- output ----- 05:43:57 DEBUG { 05:43:57 DEBUG "status": "success", 05:43:57 DEBUG "data": { 05:43:57 DEBUG "resultType": "vector", 05:43:57 DEBUG "result": [ 05:43:57 DEBUG { 05:43:57 DEBUG "metric": { 05:43:57 DEBUG "instance": "10.142.0.107:9100" 05:43:57 DEBUG }, 05:43:57 DEBUG "value": [ 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG [...] 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG "instance": "10.142.0.43:9100" 05:43:57 DEBUG }, 05:43:57 DEBUG "value": [ 05:43:57 DEBUG 1713501837, 05:43:57 DEBUG "0.6062333333333498" 05:43:57 DEBUG ] 05:43:57 DEBUG } 05:43:57 DEBUG ] 05:43:57 DEBUG } 05:43:57 DEBUG } 05:43:57 INFO [http_cmd]: http status code OK 05:43:57 DEBUG --- status code --- 05:43:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:43:57 DEBUG --- http response --- 05:43:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:43:57 DEBUG ----- output ----- 05:43:57 DEBUG { 05:43:57 DEBUG "status": "success", 05:43:57 DEBUG "data": { 05:43:57 DEBUG "resultType": "vector", 05:43:57 DEBUG "result": [ 05:43:57 DEBUG { 05:43:57 DEBUG "metric": { 05:43:57 DEBUG "instance": "10.142.0.107:9100" 05:43:57 DEBUG }, 05:43:57 DEBUG "value": [ 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG [...] 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG "instance": "10.142.0.43:9100" 05:43:57 DEBUG }, 05:43:57 DEBUG "value": [ 05:43:57 DEBUG 1713501837, 05:43:57 DEBUG "0.31899999999999795" 05:43:57 DEBUG ] 05:43:57 DEBUG } 05:43:57 DEBUG ] 05:43:57 DEBUG } 05:43:57 DEBUG } 05:43:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501837" 05:43:57 INFO [http_cmd]: http status code OK 05:43:57 DEBUG --- status code --- 05:43:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:43:57 DEBUG --- http response --- 05:43:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:43:57 DEBUG ----- output ----- 05:43:57 DEBUG { 05:43:57 DEBUG "status": "success", 05:43:57 DEBUG "data": { 05:43:57 DEBUG "resultType": "vector", 05:43:57 DEBUG "result": [ 05:43:57 DEBUG { 05:43:57 DEBUG "metric": { 05:43:57 DEBUG "pod": "ds-idrepo-1" 05:43:57 DEBUG }, 05:43:57 DEBUG "value": [ 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG [...] 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG 05:43:57 DEBUG "pod": "ds-cts-0" 05:43:57 DEBUG }, 05:43:57 DEBUG "value": [ 05:43:57 DEBUG 1713501837, 05:43:57 DEBUG "72924056.37728438" 05:43:57 DEBUG ] 05:43:57 DEBUG } 05:43:57 DEBUG ] 05:43:57 DEBUG } 05:43:57 DEBUG } 05:43:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501838" 05:43:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501838" 05:43:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501838" 05:43:58 INFO [http_cmd]: http status code OK 05:43:58 DEBUG --- status code --- 05:43:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:43:58 DEBUG --- http response --- 05:43:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:43:58 DEBUG ----- output ----- 05:43:58 DEBUG { 05:43:58 DEBUG "status": "success", 05:43:58 DEBUG "data": { 05:43:58 DEBUG "resultType": "vector", 05:43:58 DEBUG "result": [ 05:43:58 DEBUG { 05:43:58 DEBUG "metric": { 05:43:58 DEBUG "instance": "10.142.0.107:9100" 05:43:58 DEBUG }, 05:43:58 DEBUG "value": [ 05:43:58 DEBUG 05:43:58 DEBUG 05:43:58 DEBUG 05:43:58 DEBUG [...] 05:43:58 DEBUG 05:43:58 DEBUG 05:43:58 DEBUG 05:43:58 DEBUG "instance": "10.142.0.43:9100" 05:43:58 DEBUG }, 05:43:58 DEBUG "value": [ 05:43:58 DEBUG 1713501838, 05:43:58 DEBUG "64077506.733333334" 05:43:58 DEBUG ] 05:43:58 DEBUG } 05:43:58 DEBUG ] 05:43:58 DEBUG } 05:43:58 DEBUG } 05:43:58 INFO [http_cmd]: http status code OK 05:43:58 DEBUG --- status code --- 05:43:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:43:58 DEBUG --- http response --- 05:43:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:43:58 DEBUG ----- output ----- 05:43:58 DEBUG { 05:43:58 DEBUG "status": "success", 05:43:58 DEBUG "data": { 05:43:58 DEBUG "resultType": "vector", 05:43:58 DEBUG "result": [ 05:43:58 DEBUG { 05:43:58 DEBUG "metric": { 05:43:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:43:58 DEBUG }, 05:43:58 DEBUG "value": [ 05:43:58 DEBUG 05:43:58 DEBUG 05:43:58 DEBUG 05:43:58 DEBUG [...] 05:43:58 DEBUG 05:43:58 DEBUG 05:43:58 DEBUG 05:43:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:43:58 DEBUG }, 05:43:58 DEBUG "value": [ 05:43:58 DEBUG 1713501838, 05:43:58 DEBUG "2166.4333333333334" 05:43:58 DEBUG ] 05:43:58 DEBUG } 05:43:58 DEBUG ] 05:43:58 DEBUG } 05:43:58 DEBUG } 05:43:58 INFO [http_cmd]: http status code OK 05:43:58 DEBUG --- status code --- 05:43:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:43:58 DEBUG --- http response --- 05:43:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:43:58 DEBUG ----- output ----- 05:43:58 DEBUG { 05:43:58 DEBUG "status": "success", 05:43:58 DEBUG "data": { 05:43:58 DEBUG "resultType": "vector", 05:43:58 DEBUG "result": [ 05:43:58 DEBUG { 05:43:58 DEBUG "metric": { 05:43:58 DEBUG "pod": "ds-idrepo-1" 05:43:58 DEBUG }, 05:43:58 DEBUG "value": [ 05:43:58 DEBUG 05:43:58 DEBUG 05:43:58 DEBUG 05:43:58 DEBUG [...] 05:43:58 DEBUG 05:43:58 DEBUG 05:43:58 DEBUG 05:43:58 DEBUG "pod": "ds-cts-0" 05:43:58 DEBUG }, 05:43:58 DEBUG "value": [ 05:43:58 DEBUG 1713501838, 05:43:58 DEBUG "44545897.317328095" 05:43:58 DEBUG ] 05:43:58 DEBUG } 05:43:58 DEBUG ] 05:43:58 DEBUG } 05:43:58 DEBUG } 05:43:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501838" 05:43:58 INFO [http_cmd]: http status code OK 05:43:58 DEBUG --- status code --- 05:43:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:43:58 DEBUG --- http response --- 05:43:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:43:58 DEBUG ----- output ----- 05:43:58 DEBUG { 05:43:58 DEBUG "status": "success", 05:43:58 DEBUG "data": { 05:43:58 DEBUG "resultType": "vector", 05:43:58 DEBUG "result": [ 05:43:58 DEBUG { 05:43:58 DEBUG "metric": { 05:43:58 DEBUG "instance": "10.142.0.107:9100" 05:43:58 DEBUG }, 05:43:58 DEBUG "value": [ 05:43:58 DEBUG 05:43:58 DEBUG 05:43:58 DEBUG 05:43:58 DEBUG [...] 05:43:58 DEBUG 05:43:58 DEBUG 05:43:58 DEBUG 05:43:58 DEBUG "instance": "10.142.0.43:9100" 05:43:58 DEBUG }, 05:43:58 DEBUG "value": [ 05:43:58 DEBUG 1713501838, 05:43:58 DEBUG "64073016.13333333" 05:43:58 DEBUG ] 05:43:58 DEBUG } 05:43:58 DEBUG ] 05:43:58 DEBUG } 05:43:58 DEBUG } 05:44:00 INFO 05:44:00 INFO [loop_until]: kubectl --namespace=xlou top pods 05:44:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:44:00 INFO [loop_until]: OK (rc = 0) 05:44:00 DEBUG --- stdout --- 05:44:00 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 14101m 9145Mi am-6fd554f8f7-hvvkw 13643m 8912Mi am-6fd554f8f7-pmj4b 14320m 8799Mi ds-cts-0 19205m 13007Mi ds-cts-1 19021m 13159Mi ds-cts-2 19911m 13019Mi ds-idrepo-0 5663m 3268Mi ds-idrepo-1 5907m 3199Mi ds-idrepo-2 5779m 3066Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 5m 1172Mi idm-b85f9d978-wnhxb 3m 1215Mi lodemon-6b89d774d5-rf7sk 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7037m 2085Mi 05:44:00 DEBUG --- stderr --- 05:44:00 DEBUG 05:44:01 INFO 05:44:01 INFO [loop_until]: kubectl --namespace=xlou top node 05:44:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:44:01 INFO [loop_until]: OK (rc = 0) 05:44:01 DEBUG --- stdout --- 05:44:01 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 69m 0% 6241Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5799m 19% 6714Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13336m 44% 11623Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 85m 0% 3248Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 19631m 65% 15720Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 137m 0% 5172Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 13640m 45% 11930Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 11271m 37% 2727Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 15619m 52% 10997Mi 9% gke-xlou-cdm-ds-0f601751-j488 6176m 20% 6172Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 19249m 64% 17291Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6409m 21% 6875Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 20093m 67% 15022Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 7263m 45% 5773Mi 9% 05:44:01 DEBUG --- stderr --- 05:44:01 DEBUG 05:44:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501897" 05:44:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713501897" 05:44:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501897" 05:44:57 INFO [http_cmd]: http status code OK 05:44:57 DEBUG --- status code --- 05:44:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:44:57 DEBUG --- http response --- 05:44:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:44:57 DEBUG ----- output ----- 05:44:57 DEBUG { 05:44:57 DEBUG "status": "success", 05:44:57 DEBUG "data": { 05:44:57 DEBUG "resultType": "vector", 05:44:57 DEBUG "result": [ 05:44:57 DEBUG { 05:44:57 DEBUG "metric": { 05:44:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:44:57 DEBUG }, 05:44:57 DEBUG "value": [ 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG [...] 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:44:57 DEBUG }, 05:44:57 DEBUG "value": [ 05:44:57 DEBUG 1713501897, 05:44:57 DEBUG "0" 05:44:57 DEBUG ] 05:44:57 DEBUG } 05:44:57 DEBUG ] 05:44:57 DEBUG } 05:44:57 DEBUG } 05:44:57 INFO [http_cmd]: http status code OK 05:44:57 DEBUG --- status code --- 05:44:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:44:57 DEBUG --- http response --- 05:44:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:44:57 DEBUG ----- output ----- 05:44:57 DEBUG { 05:44:57 DEBUG "status": "success", 05:44:57 DEBUG "data": { 05:44:57 DEBUG "resultType": "vector", 05:44:57 DEBUG "result": [ 05:44:57 DEBUG { 05:44:57 DEBUG "metric": { 05:44:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:44:57 DEBUG }, 05:44:57 DEBUG "value": [ 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG [...] 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:44:57 DEBUG }, 05:44:57 DEBUG "value": [ 05:44:57 DEBUG 1713501897, 05:44:57 DEBUG "55.175" 05:44:57 DEBUG ] 05:44:57 DEBUG } 05:44:57 DEBUG ] 05:44:57 DEBUG } 05:44:57 DEBUG } 05:44:57 INFO [http_cmd]: http status code OK 05:44:57 DEBUG --- status code --- 05:44:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:44:57 DEBUG --- http response --- 05:44:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:44:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501897" 05:44:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501897" 05:44:57 INFO [http_cmd]: http status code OK 05:44:57 DEBUG --- status code --- 05:44:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:44:57 DEBUG --- http response --- 05:44:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:44:57 DEBUG ----- output ----- 05:44:57 DEBUG { 05:44:57 DEBUG "status": "success", 05:44:57 DEBUG "data": { 05:44:57 DEBUG "resultType": "vector", 05:44:57 DEBUG "result": [ 05:44:57 DEBUG { 05:44:57 DEBUG "metric": { 05:44:57 DEBUG "instance": "10.142.0.107:9100" 05:44:57 DEBUG }, 05:44:57 DEBUG "value": [ 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG [...] 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG "instance": "10.142.0.43:9100" 05:44:57 DEBUG }, 05:44:57 DEBUG "value": [ 05:44:57 DEBUG 1713501897, 05:44:57 DEBUG "565115155456" 05:44:57 DEBUG ] 05:44:57 DEBUG } 05:44:57 DEBUG ] 05:44:57 DEBUG } 05:44:57 DEBUG } 05:44:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713501897" 05:44:57 INFO [http_cmd]: http status code OK 05:44:57 DEBUG --- status code --- 05:44:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:44:57 DEBUG --- http response --- 05:44:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:44:57 DEBUG ----- output ----- 05:44:57 DEBUG { 05:44:57 DEBUG "status": "success", 05:44:57 DEBUG "data": { 05:44:57 DEBUG "resultType": "vector", 05:44:57 DEBUG "result": [ 05:44:57 DEBUG { 05:44:57 DEBUG "metric": { 05:44:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:44:57 DEBUG }, 05:44:57 DEBUG "value": [ 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG [...] 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:44:57 DEBUG }, 05:44:57 DEBUG "value": [ 05:44:57 DEBUG 1713501897, 05:44:57 DEBUG "2154.7666666666664" 05:44:57 DEBUG ] 05:44:57 DEBUG } 05:44:57 DEBUG ] 05:44:57 DEBUG } 05:44:57 DEBUG } 05:44:57 INFO [http_cmd]: http status code OK 05:44:57 DEBUG --- status code --- 05:44:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:44:57 DEBUG --- http response --- 05:44:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:44:57 DEBUG ----- output ----- 05:44:57 DEBUG { 05:44:57 DEBUG "status": "success", 05:44:57 DEBUG "data": { 05:44:57 DEBUG "resultType": "vector", 05:44:57 DEBUG "result": [ 05:44:57 DEBUG { 05:44:57 DEBUG "metric": { 05:44:57 DEBUG "pod": "ds-idrepo-0" 05:44:57 DEBUG }, 05:44:57 DEBUG "value": [ 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG [...] 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG "pod": "ds-cts-2" 05:44:57 DEBUG }, 05:44:57 DEBUG "value": [ 05:44:57 DEBUG 1713501897, 05:44:57 DEBUG "0" 05:44:57 DEBUG ] 05:44:57 DEBUG } 05:44:57 DEBUG ] 05:44:57 DEBUG } 05:44:57 DEBUG } 05:44:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501897" 05:44:57 INFO [http_cmd]: http status code OK 05:44:57 DEBUG --- status code --- 05:44:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:44:57 DEBUG --- http response --- 05:44:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 05:44:57 DEBUG ----- output ----- 05:44:57 DEBUG { 05:44:57 DEBUG "status": "success", 05:44:57 DEBUG "data": { 05:44:57 DEBUG "resultType": "vector", 05:44:57 DEBUG "result": [ 05:44:57 DEBUG { 05:44:57 DEBUG "metric": { 05:44:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:44:57 DEBUG }, 05:44:57 DEBUG "value": [ 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG [...] 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:44:57 DEBUG }, 05:44:57 DEBUG "value": [ 05:44:57 DEBUG 1713501897, 05:44:57 DEBUG "0" 05:44:57 DEBUG ] 05:44:57 DEBUG } 05:44:57 DEBUG ] 05:44:57 DEBUG } 05:44:57 DEBUG } 05:44:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501897" 05:44:57 INFO [http_cmd]: http status code OK 05:44:57 DEBUG --- status code --- 05:44:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:44:57 DEBUG --- http response --- 05:44:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:44:57 DEBUG ----- output ----- 05:44:57 DEBUG { 05:44:57 DEBUG "status": "success", 05:44:57 DEBUG "data": { 05:44:57 DEBUG "resultType": "vector", 05:44:57 DEBUG "result": [ 05:44:57 DEBUG { 05:44:57 DEBUG "metric": { 05:44:57 DEBUG "pod": "ds-cts-0" 05:44:57 DEBUG }, 05:44:57 DEBUG "value": [ 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG [...] 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG "pod": "ds-cts-2" 05:44:57 DEBUG }, 05:44:57 DEBUG "value": [ 05:44:57 DEBUG 1713501897, 05:44:57 DEBUG "251.46666666666667" 05:44:57 DEBUG ] 05:44:57 DEBUG } 05:44:57 DEBUG ] 05:44:57 DEBUG } 05:44:57 DEBUG } 05:44:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713501897" 05:44:57 ERROR [http_cmd]: ERROR 05:44:57 ERROR --- status code --- 05:44:57 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:44:57 ERROR --- http response --- 05:44:57 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:44:57 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:44:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501897" 05:44:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501897" 05:44:57 INFO [http_cmd]: http status code OK 05:44:57 DEBUG --- status code --- 05:44:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:44:57 DEBUG --- http response --- 05:44:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:44:57 DEBUG ----- output ----- 05:44:57 DEBUG { 05:44:57 DEBUG "status": "success", 05:44:57 DEBUG "data": { 05:44:57 DEBUG "resultType": "vector", 05:44:57 DEBUG "result": [ 05:44:57 DEBUG { 05:44:57 DEBUG "metric": { 05:44:57 DEBUG "pod": "ds-idrepo-0" 05:44:57 DEBUG }, 05:44:57 DEBUG "value": [ 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG [...] 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG "pod": "ds-idrepo-2" 05:44:57 DEBUG }, 05:44:57 DEBUG "value": [ 05:44:57 DEBUG 1713501897, 05:44:57 DEBUG "0" 05:44:57 DEBUG ] 05:44:57 DEBUG } 05:44:57 DEBUG ] 05:44:57 DEBUG } 05:44:57 DEBUG } 05:44:57 INFO [http_cmd]: http status code OK 05:44:57 DEBUG --- status code --- 05:44:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:44:57 DEBUG --- http response --- 05:44:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:44:57 DEBUG ----- output ----- 05:44:57 DEBUG { 05:44:57 DEBUG "status": "success", 05:44:57 DEBUG "data": { 05:44:57 DEBUG "resultType": "vector", 05:44:57 DEBUG "result": [ 05:44:57 DEBUG { 05:44:57 DEBUG "metric": { 05:44:57 DEBUG "pod": "ds-idrepo-0" 05:44:57 DEBUG }, 05:44:57 DEBUG "value": [ 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG [...] 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG "pod": "ds-idrepo-2" 05:44:57 DEBUG }, 05:44:57 DEBUG "value": [ 05:44:57 DEBUG 1713501897, 05:44:57 DEBUG "0" 05:44:57 DEBUG ] 05:44:57 DEBUG } 05:44:57 DEBUG ] 05:44:57 DEBUG } 05:44:57 DEBUG } 05:44:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501897" 05:44:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713501897" 05:44:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501897" 05:44:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501897" 05:44:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501897" 05:44:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501897" 05:44:57 INFO [http_cmd]: http status code OK 05:44:57 DEBUG --- status code --- 05:44:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:44:57 DEBUG --- http response --- 05:44:57 INFO [http_cmd]: http status code OK 05:44:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:44:57 DEBUG --- status code --- 05:44:57 DEBUG ----- output ----- 05:44:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:44:57 DEBUG { 05:44:57 DEBUG --- http response --- 05:44:57 DEBUG "status": "success", 05:44:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 05:44:57 DEBUG "data": { 05:44:57 INFO [http_cmd]: http status code OK 05:44:57 DEBUG ----- output ----- 05:44:57 DEBUG "resultType": "vector", 05:44:57 DEBUG --- status code --- 05:44:57 DEBUG { 05:44:57 DEBUG "result": [ 05:44:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:44:57 DEBUG "status": "success", 05:44:57 DEBUG { 05:44:57 DEBUG --- http response --- 05:44:57 DEBUG "data": { 05:44:57 DEBUG "metric": { 05:44:57 DEBUG "resultType": "vector", 05:44:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:44:57 DEBUG "pod": "ds-cts-0" 05:44:57 DEBUG "result": [ 05:44:57 DEBUG ----- output ----- 05:44:57 DEBUG }, 05:44:57 DEBUG { 05:44:57 DEBUG { 05:44:57 DEBUG "value": [ 05:44:57 DEBUG "metric": { 05:44:57 DEBUG "status": "success", 05:44:57 DEBUG 05:44:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:44:57 DEBUG "data": { 05:44:57 DEBUG 05:44:57 DEBUG }, 05:44:57 DEBUG "resultType": "vector", 05:44:57 DEBUG 05:44:57 DEBUG "value": [ 05:44:57 DEBUG "result": [ 05:44:57 DEBUG [...] 05:44:57 DEBUG 05:44:57 DEBUG { 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 INFO [http_cmd]: http status code OK 05:44:57 INFO [http_cmd]: http status code OK 05:44:57 DEBUG "metric": { 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG --- status code --- 05:44:57 DEBUG --- status code --- 05:44:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:44:57 DEBUG 05:44:57 DEBUG [...] 05:44:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:44:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:44:57 DEBUG }, 05:44:57 DEBUG "pod": "ds-cts-2" 05:44:57 DEBUG 05:44:57 DEBUG --- http response --- 05:44:57 DEBUG --- http response --- 05:44:57 DEBUG "value": [ 05:44:57 DEBUG }, 05:44:57 DEBUG 05:44:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:44:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:44:57 DEBUG 05:44:57 INFO [http_cmd]: http status code OK 05:44:57 DEBUG "value": [ 05:44:57 DEBUG 05:44:57 DEBUG ----- output ----- 05:44:57 DEBUG ----- output ----- 05:44:57 DEBUG 05:44:57 DEBUG --- status code --- 05:44:57 DEBUG 1713501897, 05:44:57 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 05:44:57 DEBUG { 05:44:57 DEBUG { 05:44:57 DEBUG 05:44:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:44:57 DEBUG "0.00003333333333333336" 05:44:57 DEBUG }, 05:44:57 DEBUG "status": "success", 05:44:57 DEBUG "status": "success", 05:44:57 DEBUG [...] 05:44:57 DEBUG --- http response --- 05:44:57 DEBUG ] 05:44:57 DEBUG "value": [ 05:44:57 DEBUG "data": { 05:44:57 DEBUG "data": { 05:44:57 DEBUG 05:44:57 DEBUG } 05:44:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:44:57 DEBUG 1713501897, 05:44:57 DEBUG "resultType": "vector", 05:44:57 DEBUG "resultType": "vector", 05:44:57 DEBUG 05:44:57 DEBUG ] 05:44:57 DEBUG ----- output ----- 05:44:57 DEBUG "0" 05:44:57 DEBUG "result": [ 05:44:57 DEBUG "result": [ 05:44:57 DEBUG 05:44:57 DEBUG } 05:44:57 DEBUG { 05:44:57 DEBUG ] 05:44:57 DEBUG { 05:44:57 DEBUG { 05:44:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:44:57 DEBUG } 05:44:57 DEBUG "status": "success", 05:44:57 DEBUG } 05:44:57 DEBUG "metric": { 05:44:57 DEBUG "metric": { 05:44:57 DEBUG }, 05:44:57 DEBUG "data": { 05:44:57 DEBUG ] 05:44:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:44:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:44:57 DEBUG "value": [ 05:44:57 DEBUG "resultType": "vector", 05:44:57 DEBUG } 05:44:57 DEBUG }, 05:44:57 DEBUG }, 05:44:57 DEBUG 1713501897, 05:44:57 DEBUG "result": [ 05:44:57 DEBUG } 05:44:57 DEBUG "value": [ 05:44:57 DEBUG "value": [ 05:44:57 DEBUG "7.057746339450375" 05:44:57 DEBUG { 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG ] 05:44:57 DEBUG "metric": { 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG } 05:44:57 DEBUG "pod": "ds-idrepo-0" 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG ] 05:44:57 DEBUG }, 05:44:57 DEBUG [...] 05:44:57 DEBUG [...] 05:44:57 DEBUG } 05:44:57 DEBUG "value": [ 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG } 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:44:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:44:57 DEBUG [...] 05:44:57 DEBUG }, 05:44:57 DEBUG }, 05:44:57 DEBUG 05:44:57 DEBUG "value": [ 05:44:57 DEBUG "value": [ 05:44:57 DEBUG 05:44:57 DEBUG 1713501897, 05:44:57 DEBUG 1713501897, 05:44:57 DEBUG 05:44:57 DEBUG "1994.7" 05:44:57 DEBUG "3772070" 05:44:57 DEBUG "pod": "ds-idrepo-2" 05:44:57 DEBUG ] 05:44:57 DEBUG ] 05:44:57 DEBUG }, 05:44:57 DEBUG } 05:44:57 DEBUG } 05:44:57 DEBUG "value": [ 05:44:57 DEBUG ] 05:44:57 DEBUG ] 05:44:57 DEBUG 1713501897, 05:44:57 DEBUG } 05:44:57 DEBUG } 05:44:57 DEBUG "0" 05:44:57 DEBUG } 05:44:57 DEBUG } 05:44:57 DEBUG ] 05:44:57 DEBUG } 05:44:57 DEBUG ] 05:44:57 DEBUG } 05:44:57 DEBUG } 05:44:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713501897" 05:44:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501897" 05:44:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501897" 05:44:57 INFO [http_cmd]: http status code OK 05:44:57 DEBUG --- status code --- 05:44:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:44:57 DEBUG --- http response --- 05:44:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:44:57 DEBUG ----- output ----- 05:44:57 DEBUG { 05:44:57 DEBUG "status": "success", 05:44:57 DEBUG "data": { 05:44:57 DEBUG "resultType": "vector", 05:44:57 DEBUG "result": [ 05:44:57 DEBUG { 05:44:57 DEBUG "metric": { 05:44:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:44:57 DEBUG }, 05:44:57 DEBUG "value": [ 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG [...] 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:44:57 DEBUG }, 05:44:57 DEBUG "value": [ 05:44:57 DEBUG 1713501897, 05:44:57 DEBUG "2285092864" 05:44:57 DEBUG ] 05:44:57 DEBUG } 05:44:57 DEBUG ] 05:44:57 DEBUG } 05:44:57 DEBUG } 05:44:57 INFO [http_cmd]: http status code OK 05:44:57 DEBUG --- status code --- 05:44:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:44:57 DEBUG --- http response --- 05:44:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501897" 05:44:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:44:57 DEBUG ----- output ----- 05:44:57 DEBUG { 05:44:57 DEBUG "status": "success", 05:44:57 DEBUG "data": { 05:44:57 DEBUG "resultType": "vector", 05:44:57 DEBUG "result": [ 05:44:57 DEBUG { 05:44:57 DEBUG "metric": { 05:44:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:44:57 DEBUG }, 05:44:57 DEBUG "value": [ 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG [...] 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:44:57 DEBUG }, 05:44:57 DEBUG "value": [ 05:44:57 DEBUG 1713501897, 05:44:57 DEBUG "2155.2666666666664" 05:44:57 DEBUG ] 05:44:57 DEBUG } 05:44:57 DEBUG ] 05:44:57 DEBUG } 05:44:57 DEBUG } 05:44:57 INFO [http_cmd]: http status code OK 05:44:57 DEBUG --- status code --- 05:44:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:44:57 DEBUG --- http response --- 05:44:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:44:57 DEBUG ----- output ----- 05:44:57 DEBUG { 05:44:57 DEBUG "status": "success", 05:44:57 DEBUG "data": { 05:44:57 DEBUG "resultType": "vector", 05:44:57 DEBUG "result": [ 05:44:57 DEBUG { 05:44:57 DEBUG "metric": { 05:44:57 DEBUG "instance": "10.142.0.107:9100" 05:44:57 DEBUG }, 05:44:57 DEBUG "value": [ 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG [...] 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG "instance": "10.142.0.43:9100" 05:44:57 DEBUG }, 05:44:57 DEBUG "value": [ 05:44:57 DEBUG 1713501897, 05:44:57 DEBUG "0.3436666666666668" 05:44:57 DEBUG ] 05:44:57 DEBUG } 05:44:57 DEBUG ] 05:44:57 DEBUG } 05:44:57 DEBUG } 05:44:57 INFO [http_cmd]: http status code OK 05:44:57 DEBUG --- status code --- 05:44:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:44:57 DEBUG --- http response --- 05:44:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:44:57 DEBUG ----- output ----- 05:44:57 DEBUG { 05:44:57 DEBUG "status": "success", 05:44:57 DEBUG "data": { 05:44:57 DEBUG "resultType": "vector", 05:44:57 DEBUG "result": [ 05:44:57 DEBUG { 05:44:57 DEBUG "metric": { 05:44:57 DEBUG "pod": "ds-cts-0" 05:44:57 DEBUG }, 05:44:57 DEBUG "value": [ 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG [...] 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG 05:44:57 DEBUG "pod": "ds-cts-2" 05:44:57 DEBUG }, 05:44:57 DEBUG "value": [ 05:44:57 DEBUG 1713501897, 05:44:57 DEBUG "30" 05:44:57 DEBUG ] 05:44:57 DEBUG } 05:44:57 DEBUG ] 05:44:57 DEBUG } 05:44:57 DEBUG } 05:44:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501898" 05:44:58 INFO [http_cmd]: http status code OK 05:44:58 DEBUG --- status code --- 05:44:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:44:58 DEBUG --- http response --- 05:44:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:44:58 DEBUG ----- output ----- 05:44:58 DEBUG { 05:44:58 DEBUG "status": "success", 05:44:58 DEBUG "data": { 05:44:58 DEBUG "resultType": "vector", 05:44:58 DEBUG "result": [ 05:44:58 DEBUG { 05:44:58 DEBUG "metric": { 05:44:58 DEBUG "instance": "10.142.0.107:9100" 05:44:58 DEBUG }, 05:44:58 DEBUG "value": [ 05:44:58 DEBUG 05:44:58 DEBUG 05:44:58 DEBUG 05:44:58 DEBUG [...] 05:44:58 DEBUG 05:44:58 DEBUG 05:44:58 DEBUG 05:44:58 DEBUG "instance": "10.142.0.43:9100" 05:44:58 DEBUG }, 05:44:58 DEBUG "value": [ 05:44:58 DEBUG 1713501898, 05:44:58 DEBUG "0.642933333333345" 05:44:58 DEBUG ] 05:44:58 DEBUG } 05:44:58 DEBUG ] 05:44:58 DEBUG } 05:44:58 DEBUG } 05:44:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501898" 05:44:58 INFO [http_cmd]: http status code OK 05:44:58 DEBUG --- status code --- 05:44:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:44:58 DEBUG --- http response --- 05:44:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:44:58 DEBUG ----- output ----- 05:44:58 DEBUG { 05:44:58 DEBUG "status": "success", 05:44:58 DEBUG "data": { 05:44:58 DEBUG "resultType": "vector", 05:44:58 DEBUG "result": [ 05:44:58 DEBUG { 05:44:58 DEBUG "metric": { 05:44:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:44:58 DEBUG }, 05:44:58 DEBUG "value": [ 05:44:58 DEBUG 05:44:58 DEBUG 05:44:58 DEBUG 05:44:58 DEBUG [...] 05:44:58 DEBUG 05:44:58 DEBUG 05:44:58 DEBUG 05:44:58 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 05:44:58 DEBUG }, 05:44:58 DEBUG "value": [ 05:44:58 DEBUG 1713501898, 05:44:58 DEBUG "0" 05:44:58 DEBUG ] 05:44:58 DEBUG } 05:44:58 DEBUG ] 05:44:58 DEBUG } 05:44:58 DEBUG } 05:44:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501898" 05:44:58 INFO [http_cmd]: http status code OK 05:44:58 DEBUG --- status code --- 05:44:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:44:58 DEBUG --- http response --- 05:44:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:44:58 DEBUG ----- output ----- 05:44:58 DEBUG { 05:44:58 DEBUG "status": "success", 05:44:58 DEBUG "data": { 05:44:58 DEBUG "resultType": "vector", 05:44:58 DEBUG "result": [ 05:44:58 DEBUG { 05:44:58 DEBUG "metric": { 05:44:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:44:58 DEBUG }, 05:44:58 DEBUG "value": [ 05:44:58 DEBUG 05:44:58 DEBUG 05:44:58 DEBUG 05:44:58 DEBUG [...] 05:44:58 DEBUG 05:44:58 DEBUG 05:44:58 DEBUG 05:44:58 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 05:44:58 DEBUG }, 05:44:58 DEBUG "value": [ 05:44:58 DEBUG 1713501898, 05:44:58 DEBUG "0" 05:44:58 DEBUG ] 05:44:58 DEBUG } 05:44:58 DEBUG ] 05:44:58 DEBUG } 05:44:58 DEBUG } 05:44:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501898" 05:44:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501898" 05:44:58 INFO [http_cmd]: http status code OK 05:44:58 DEBUG --- status code --- 05:44:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:44:58 DEBUG --- http response --- 05:44:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:44:58 DEBUG ----- output ----- 05:44:58 DEBUG { 05:44:58 DEBUG "status": "success", 05:44:58 DEBUG "data": { 05:44:58 DEBUG "resultType": "vector", 05:44:58 DEBUG "result": [ 05:44:58 DEBUG { 05:44:58 DEBUG "metric": { 05:44:58 DEBUG "instance": "10.142.0.107:9100" 05:44:58 DEBUG }, 05:44:58 DEBUG "value": [ 05:44:58 DEBUG 05:44:58 DEBUG 05:44:58 DEBUG 05:44:58 DEBUG [...] 05:44:58 DEBUG 05:44:58 DEBUG 05:44:58 DEBUG 05:44:58 DEBUG "instance": "10.142.0.43:9100" 05:44:58 DEBUG }, 05:44:58 DEBUG "value": [ 05:44:58 DEBUG 1713501898, 05:44:58 DEBUG "63576933.1" 05:44:58 DEBUG ] 05:44:58 DEBUG } 05:44:58 DEBUG ] 05:44:58 DEBUG } 05:44:58 DEBUG } 05:44:58 INFO [http_cmd]: http status code OK 05:44:58 DEBUG --- status code --- 05:44:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:44:58 DEBUG --- http response --- 05:44:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:44:58 DEBUG ----- output ----- 05:44:58 DEBUG { 05:44:58 DEBUG "status": "success", 05:44:58 DEBUG "data": { 05:44:58 DEBUG "resultType": "vector", 05:44:58 DEBUG "result": [ 05:44:58 DEBUG { 05:44:58 DEBUG "metric": { 05:44:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:44:58 DEBUG }, 05:44:58 DEBUG "value": [ 05:44:58 DEBUG 05:44:58 DEBUG 05:44:58 DEBUG 05:44:58 DEBUG [...] 05:44:58 DEBUG 05:44:58 DEBUG 05:44:58 DEBUG 05:44:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:44:58 DEBUG }, 05:44:58 DEBUG "value": [ 05:44:58 DEBUG 1713501898, 05:44:58 DEBUG "2156.4666666666667" 05:44:58 DEBUG ] 05:44:58 DEBUG } 05:44:58 DEBUG ] 05:44:58 DEBUG } 05:44:58 DEBUG } 05:44:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501898" 05:44:58 INFO [http_cmd]: http status code OK 05:44:58 DEBUG --- status code --- 05:44:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:44:58 DEBUG --- http response --- 05:44:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:44:58 DEBUG ----- output ----- 05:44:58 DEBUG { 05:44:58 DEBUG "status": "success", 05:44:58 DEBUG "data": { 05:44:58 DEBUG "resultType": "vector", 05:44:58 DEBUG "result": [ 05:44:58 DEBUG { 05:44:58 DEBUG "metric": { 05:44:58 DEBUG "instance": "10.142.0.107:9100" 05:44:58 DEBUG }, 05:44:58 DEBUG "value": [ 05:44:58 DEBUG 05:44:58 DEBUG 05:44:58 DEBUG 05:44:58 DEBUG [...] 05:44:58 DEBUG 05:44:58 DEBUG 05:44:58 DEBUG 05:44:58 DEBUG "instance": "10.142.0.43:9100" 05:44:58 DEBUG }, 05:44:58 DEBUG "value": [ 05:44:58 DEBUG 1713501898, 05:44:58 DEBUG "63573365.2" 05:44:58 DEBUG ] 05:44:58 DEBUG } 05:44:58 DEBUG ] 05:44:58 DEBUG } 05:44:58 DEBUG } 05:45:00 INFO 05:45:00 INFO [loop_until]: kubectl --namespace=xlou top pods 05:45:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:45:00 INFO [loop_until]: OK (rc = 0) 05:45:00 DEBUG --- stdout --- 05:45:00 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 13257m 9145Mi am-6fd554f8f7-hvvkw 12519m 8911Mi am-6fd554f8f7-pmj4b 14495m 8798Mi ds-cts-0 20401m 13349Mi ds-cts-1 16570m 13245Mi ds-cts-2 20077m 13215Mi ds-idrepo-0 5741m 3268Mi ds-idrepo-1 5724m 3198Mi ds-idrepo-2 5540m 3066Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1172Mi idm-b85f9d978-wnhxb 4m 1215Mi lodemon-6b89d774d5-rf7sk 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7240m 2086Mi 05:45:00 DEBUG --- stderr --- 05:45:00 DEBUG 05:45:01 INFO 05:45:01 INFO [loop_until]: kubectl --namespace=xlou top node 05:45:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:45:01 INFO [loop_until]: OK (rc = 0) 05:45:01 DEBUG --- stdout --- 05:45:01 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 68m 0% 6240Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5559m 18% 6706Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 12891m 43% 11626Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 84m 0% 3241Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 19187m 64% 15974Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 147m 0% 5164Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 13899m 46% 11930Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 11254m 37% 2742Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 14693m 49% 10997Mi 9% gke-xlou-cdm-ds-0f601751-j488 5949m 19% 6163Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 16316m 54% 17294Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6189m 20% 6876Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 18255m 61% 15290Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 7246m 45% 5773Mi 9% 05:45:01 DEBUG --- stderr --- 05:45:01 DEBUG 05:45:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713501957" 05:45:57 INFO [http_cmd]: http status code OK 05:45:57 DEBUG --- status code --- 05:45:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:45:57 DEBUG --- http response --- 05:45:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:45:57 DEBUG ----- output ----- 05:45:57 DEBUG { 05:45:57 DEBUG "status": "success", 05:45:57 DEBUG "data": { 05:45:57 DEBUG "resultType": "vector", 05:45:57 DEBUG "result": [ 05:45:57 DEBUG { 05:45:57 DEBUG "metric": { 05:45:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:45:57 DEBUG }, 05:45:57 DEBUG "value": [ 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG [...] 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:45:57 DEBUG }, 05:45:57 DEBUG "value": [ 05:45:57 DEBUG 1713501957, 05:45:57 DEBUG "55.1755" 05:45:57 DEBUG ] 05:45:57 DEBUG } 05:45:57 DEBUG ] 05:45:57 DEBUG } 05:45:57 DEBUG } 05:45:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501957" 05:45:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501957" 05:45:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501957" 05:45:57 INFO [http_cmd]: http status code OK 05:45:57 INFO [http_cmd]: http status code OK 05:45:57 INFO [http_cmd]: http status code OK 05:45:57 DEBUG --- status code --- 05:45:57 DEBUG --- status code --- 05:45:57 DEBUG --- status code --- 05:45:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:45:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:45:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:45:57 DEBUG --- http response --- 05:45:57 DEBUG --- http response --- 05:45:57 DEBUG --- http response --- 05:45:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:45:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:45:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:45:57 DEBUG ----- output ----- 05:45:57 DEBUG ----- output ----- 05:45:57 DEBUG { 05:45:57 DEBUG { 05:45:57 DEBUG "status": "success", 05:45:57 DEBUG "status": "success", 05:45:57 DEBUG "data": { 05:45:57 DEBUG "data": { 05:45:57 DEBUG "resultType": "vector", 05:45:57 DEBUG "resultType": "vector", 05:45:57 DEBUG "result": [ 05:45:57 DEBUG "result": [ 05:45:57 DEBUG { 05:45:57 DEBUG { 05:45:57 DEBUG "metric": { 05:45:57 DEBUG "metric": { 05:45:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:45:57 DEBUG "instance": "10.142.0.107:9100" 05:45:57 DEBUG }, 05:45:57 DEBUG }, 05:45:57 DEBUG "value": [ 05:45:57 DEBUG "value": [ 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG [...] 05:45:57 DEBUG [...] 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:45:57 DEBUG "instance": "10.142.0.43:9100" 05:45:57 DEBUG }, 05:45:57 DEBUG }, 05:45:57 DEBUG "value": [ 05:45:57 DEBUG "value": [ 05:45:57 DEBUG 1713501957, 05:45:57 DEBUG 1713501957, 05:45:57 DEBUG "0" 05:45:57 DEBUG "568429241344" 05:45:57 DEBUG ] 05:45:57 DEBUG ] 05:45:57 DEBUG } 05:45:57 DEBUG } 05:45:57 DEBUG ] 05:45:57 DEBUG ] 05:45:57 DEBUG } 05:45:57 DEBUG } 05:45:57 DEBUG } 05:45:57 DEBUG } 05:45:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501957" 05:45:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713501957" 05:45:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501957" 05:45:57 INFO [http_cmd]: http status code OK 05:45:57 DEBUG --- status code --- 05:45:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:45:57 DEBUG --- http response --- 05:45:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:45:57 DEBUG ----- output ----- 05:45:57 DEBUG { 05:45:57 DEBUG "status": "success", 05:45:57 DEBUG "data": { 05:45:57 DEBUG "resultType": "vector", 05:45:57 DEBUG "result": [ 05:45:57 DEBUG { 05:45:57 DEBUG "metric": { 05:45:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:45:57 DEBUG }, 05:45:57 DEBUG "value": [ 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG [...] 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:45:57 DEBUG }, 05:45:57 DEBUG "value": [ 05:45:57 DEBUG 1713501957, 05:45:57 DEBUG "2164.233333333333" 05:45:57 DEBUG ] 05:45:57 DEBUG } 05:45:57 DEBUG ] 05:45:57 DEBUG } 05:45:57 DEBUG } 05:45:57 INFO [http_cmd]: http status code OK 05:45:57 DEBUG --- status code --- 05:45:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:45:57 DEBUG --- http response --- 05:45:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:45:57 DEBUG ----- output ----- 05:45:57 DEBUG { 05:45:57 DEBUG "status": "success", 05:45:57 DEBUG "data": { 05:45:57 DEBUG "resultType": "vector", 05:45:57 DEBUG "result": [ 05:45:57 DEBUG { 05:45:57 DEBUG "metric": { 05:45:57 DEBUG "pod": "ds-idrepo-0" 05:45:57 DEBUG }, 05:45:57 DEBUG "value": [ 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG [...] 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG "pod": "ds-cts-2" 05:45:57 DEBUG }, 05:45:57 DEBUG "value": [ 05:45:57 DEBUG 1713501957, 05:45:57 DEBUG "0" 05:45:57 DEBUG ] 05:45:57 DEBUG } 05:45:57 DEBUG ] 05:45:57 DEBUG } 05:45:57 DEBUG } 05:45:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713501957" 05:45:57 INFO [http_cmd]: http status code OK 05:45:57 DEBUG --- status code --- 05:45:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:45:57 DEBUG --- http response --- 05:45:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:45:57 DEBUG ----- output ----- 05:45:57 DEBUG { 05:45:57 DEBUG "status": "success", 05:45:57 DEBUG "data": { 05:45:57 DEBUG "resultType": "vector", 05:45:57 DEBUG "result": [ 05:45:57 DEBUG { 05:45:57 DEBUG "metric": { 05:45:57 DEBUG "pod": "ds-cts-0" 05:45:57 DEBUG }, 05:45:57 DEBUG "value": [ 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG [...] 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG "pod": "ds-cts-2" 05:45:57 DEBUG }, 05:45:57 DEBUG "value": [ 05:45:57 DEBUG 1713501957, 05:45:57 DEBUG "120.5" 05:45:57 DEBUG ] 05:45:57 DEBUG } 05:45:57 DEBUG ] 05:45:57 DEBUG } 05:45:57 DEBUG } 05:45:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501957" 05:45:57 ERROR [http_cmd]: ERROR 05:45:57 ERROR --- status code --- 05:45:57 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:45:57 ERROR --- http response --- 05:45:57 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:45:57 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:45:57 INFO [http_cmd]: http status code OK 05:45:57 DEBUG --- status code --- 05:45:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:45:57 DEBUG --- http response --- 05:45:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 05:45:57 DEBUG ----- output ----- 05:45:57 DEBUG { 05:45:57 DEBUG "status": "success", 05:45:57 DEBUG "data": { 05:45:57 DEBUG "resultType": "vector", 05:45:57 DEBUG "result": [ 05:45:57 DEBUG { 05:45:57 DEBUG "metric": { 05:45:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:45:57 DEBUG }, 05:45:57 DEBUG "value": [ 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG [...] 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 05:45:57 DEBUG }, 05:45:57 DEBUG "value": [ 05:45:57 DEBUG 1713501957, 05:45:57 DEBUG "0" 05:45:57 DEBUG ] 05:45:57 DEBUG } 05:45:57 DEBUG ] 05:45:57 DEBUG } 05:45:57 DEBUG } 05:45:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501957" 05:45:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713501957" 05:45:57 INFO [http_cmd]: http status code OK 05:45:57 DEBUG --- status code --- 05:45:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:45:57 DEBUG --- http response --- 05:45:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:45:57 DEBUG ----- output ----- 05:45:57 DEBUG { 05:45:57 DEBUG "status": "success", 05:45:57 DEBUG "data": { 05:45:57 DEBUG "resultType": "vector", 05:45:57 DEBUG "result": [ 05:45:57 DEBUG { 05:45:57 DEBUG "metric": { 05:45:57 DEBUG "pod": "ds-idrepo-0" 05:45:57 DEBUG }, 05:45:57 DEBUG "value": [ 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG [...] 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG "pod": "ds-idrepo-2" 05:45:57 DEBUG }, 05:45:57 DEBUG "value": [ 05:45:57 DEBUG 1713501957, 05:45:57 DEBUG "0" 05:45:57 DEBUG ] 05:45:57 DEBUG } 05:45:57 DEBUG ] 05:45:57 DEBUG } 05:45:57 DEBUG } 05:45:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501957" 05:45:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501957" 05:45:57 INFO [http_cmd]: http status code OK 05:45:57 DEBUG --- status code --- 05:45:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:45:57 DEBUG --- http response --- 05:45:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:45:57 DEBUG ----- output ----- 05:45:57 DEBUG { 05:45:57 DEBUG "status": "success", 05:45:57 DEBUG "data": { 05:45:57 DEBUG "resultType": "vector", 05:45:57 DEBUG "result": [ 05:45:57 DEBUG { 05:45:57 DEBUG "metric": { 05:45:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:45:57 DEBUG }, 05:45:57 DEBUG "value": [ 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG [...] 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:45:57 DEBUG }, 05:45:57 DEBUG "value": [ 05:45:57 DEBUG 1713501957, 05:45:57 DEBUG "7.082609405977485" 05:45:57 DEBUG ] 05:45:57 DEBUG } 05:45:57 DEBUG ] 05:45:57 DEBUG } 05:45:57 DEBUG } 05:45:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501957" 05:45:57 INFO [http_cmd]: http status code OK 05:45:57 INFO [http_cmd]: http status code OK 05:45:57 DEBUG --- status code --- 05:45:57 DEBUG --- status code --- 05:45:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:45:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:45:57 DEBUG --- http response --- 05:45:57 DEBUG --- http response --- 05:45:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:45:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:45:57 DEBUG ----- output ----- 05:45:57 DEBUG ----- output ----- 05:45:57 DEBUG { 05:45:57 DEBUG { 05:45:57 DEBUG "status": "success", 05:45:57 DEBUG "status": "success", 05:45:57 DEBUG "data": { 05:45:57 DEBUG "data": { 05:45:57 DEBUG "resultType": "vector", 05:45:57 DEBUG "resultType": "vector", 05:45:57 DEBUG "result": [ 05:45:57 DEBUG "result": [ 05:45:57 DEBUG { 05:45:57 DEBUG { 05:45:57 DEBUG "metric": { 05:45:57 DEBUG "metric": { 05:45:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:45:57 DEBUG "pod": "ds-idrepo-0" 05:45:57 DEBUG }, 05:45:57 DEBUG }, 05:45:57 DEBUG "value": [ 05:45:57 DEBUG "value": [ 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG [...] 05:45:57 DEBUG [...] 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG 05:45:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:45:57 DEBUG "pod": "ds-idrepo-2" 05:45:57 DEBUG }, 05:45:57 DEBUG }, 05:45:57 DEBUG "value": [ 05:45:57 DEBUG "value": [ 05:45:57 DEBUG 1713501957, 05:45:57 DEBUG 1713501957, 05:45:57 DEBUG "3851548" 05:45:57 DEBUG "0" 05:45:57 DEBUG ] 05:45:57 DEBUG ] 05:45:57 DEBUG } 05:45:57 DEBUG } 05:45:57 DEBUG ] 05:45:57 DEBUG ] 05:45:57 DEBUG } 05:45:57 DEBUG } 05:45:57 DEBUG } 05:45:57 DEBUG } 05:45:58 INFO [http_cmd]: http status code OK 05:45:58 DEBUG --- status code --- 05:45:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:45:58 DEBUG --- http response --- 05:45:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:45:58 DEBUG ----- output ----- 05:45:58 DEBUG { 05:45:58 DEBUG "status": "success", 05:45:58 DEBUG "data": { 05:45:58 DEBUG "resultType": "vector", 05:45:58 DEBUG "result": [ 05:45:58 DEBUG { 05:45:58 DEBUG "metric": { 05:45:58 DEBUG "instance": "10.142.0.107:9100" 05:45:58 DEBUG }, 05:45:58 DEBUG "value": [ 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG [...] 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG "instance": "10.142.0.43:9100" 05:45:58 DEBUG }, 05:45:58 DEBUG "value": [ 05:45:58 DEBUG 1713501957, 05:45:58 DEBUG "0.09900000000000045" 05:45:58 DEBUG ] 05:45:58 DEBUG } 05:45:58 DEBUG ] 05:45:58 DEBUG } 05:45:58 DEBUG } 05:45:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501958" 05:45:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501958" 05:45:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501958" 05:45:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501958" 05:45:58 INFO [http_cmd]: http status code OK 05:45:58 DEBUG --- status code --- 05:45:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:45:58 DEBUG --- http response --- 05:45:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:45:58 DEBUG ----- output ----- 05:45:58 DEBUG { 05:45:58 DEBUG "status": "success", 05:45:58 DEBUG "data": { 05:45:58 DEBUG "resultType": "vector", 05:45:58 DEBUG "result": [ 05:45:58 DEBUG { 05:45:58 DEBUG "metric": { 05:45:58 DEBUG "pod": "ds-idrepo-0" 05:45:58 DEBUG }, 05:45:58 DEBUG "value": [ 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG [...] 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG "pod": "ds-idrepo-2" 05:45:58 DEBUG }, 05:45:58 DEBUG "value": [ 05:45:58 DEBUG 1713501958, 05:45:58 DEBUG "0" 05:45:58 DEBUG ] 05:45:58 DEBUG } 05:45:58 DEBUG ] 05:45:58 DEBUG } 05:45:58 DEBUG } 05:45:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501958" 05:45:58 INFO [http_cmd]: http status code OK 05:45:58 INFO [http_cmd]: http status code OK 05:45:58 DEBUG --- status code --- 05:45:58 DEBUG --- status code --- 05:45:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:45:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:45:58 DEBUG --- http response --- 05:45:58 DEBUG --- http response --- 05:45:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:45:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 05:45:58 DEBUG ----- output ----- 05:45:58 DEBUG ----- output ----- 05:45:58 INFO [http_cmd]: http status code OK 05:45:58 DEBUG { 05:45:58 DEBUG { 05:45:58 DEBUG --- status code --- 05:45:58 DEBUG "status": "success", 05:45:58 DEBUG "status": "success", 05:45:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:45:58 DEBUG "data": { 05:45:58 DEBUG "data": { 05:45:58 DEBUG --- http response --- 05:45:58 DEBUG "resultType": "vector", 05:45:58 DEBUG "resultType": "vector", 05:45:58 DEBUG "result": [ 05:45:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:45:58 DEBUG "result": [ 05:45:58 DEBUG { 05:45:58 DEBUG ----- output ----- 05:45:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713501958" 05:45:58 DEBUG { 05:45:58 DEBUG "metric": { 05:45:58 DEBUG { 05:45:58 DEBUG "metric": { 05:45:58 DEBUG "pod": "ds-cts-0" 05:45:58 DEBUG "status": "success", 05:45:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:45:58 DEBUG }, 05:45:58 DEBUG "data": { 05:45:58 DEBUG }, 05:45:58 DEBUG "value": [ 05:45:58 DEBUG "resultType": "vector", 05:45:58 DEBUG "value": [ 05:45:58 DEBUG 05:45:58 DEBUG "result": [ 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG { 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG "metric": { 05:45:58 DEBUG 05:45:58 DEBUG [...] 05:45:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:45:58 DEBUG [...] 05:45:58 DEBUG 05:45:58 DEBUG }, 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG "value": [ 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG "pod": "ds-cts-2" 05:45:58 DEBUG 05:45:58 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 05:45:58 DEBUG }, 05:45:58 DEBUG 05:45:58 DEBUG }, 05:45:58 DEBUG "value": [ 05:45:58 DEBUG [...] 05:45:58 DEBUG "value": [ 05:45:58 DEBUG 1713501958, 05:45:58 DEBUG 05:45:58 DEBUG 1713501958, 05:45:58 DEBUG "0.00036720666666666666" 05:45:58 DEBUG 05:45:58 DEBUG "0.000015115075661514048" 05:45:58 DEBUG ] 05:45:58 DEBUG 05:45:58 DEBUG ] 05:45:58 DEBUG } 05:45:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:45:58 DEBUG } 05:45:58 DEBUG ] 05:45:58 DEBUG }, 05:45:58 DEBUG ] 05:45:58 DEBUG } 05:45:58 DEBUG "value": [ 05:45:58 DEBUG } 05:45:58 DEBUG } 05:45:58 DEBUG 1713501958, 05:45:58 DEBUG } 05:45:58 DEBUG "80.96666666666667" 05:45:58 DEBUG ] 05:45:58 DEBUG } 05:45:58 DEBUG ] 05:45:58 DEBUG } 05:45:58 DEBUG } 05:45:58 INFO [http_cmd]: http status code OK 05:45:58 DEBUG --- status code --- 05:45:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:45:58 DEBUG --- http response --- 05:45:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:45:58 DEBUG ----- output ----- 05:45:58 DEBUG { 05:45:58 DEBUG "status": "success", 05:45:58 DEBUG "data": { 05:45:58 DEBUG "resultType": "vector", 05:45:58 DEBUG "result": [ 05:45:58 DEBUG { 05:45:58 DEBUG "metric": { 05:45:58 DEBUG "pod": "ds-cts-0" 05:45:58 DEBUG }, 05:45:58 DEBUG "value": [ 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG [...] 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG "pod": "ds-cts-2" 05:45:58 DEBUG }, 05:45:58 DEBUG "value": [ 05:45:58 DEBUG 1713501958, 05:45:58 DEBUG "30" 05:45:58 DEBUG ] 05:45:58 DEBUG } 05:45:58 DEBUG ] 05:45:58 DEBUG } 05:45:58 DEBUG } 05:45:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501958" 05:45:58 INFO [http_cmd]: http status code OK 05:45:58 DEBUG --- status code --- 05:45:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:45:58 DEBUG --- http response --- 05:45:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:45:58 DEBUG ----- output ----- 05:45:58 DEBUG { 05:45:58 DEBUG "status": "success", 05:45:58 DEBUG "data": { 05:45:58 DEBUG "resultType": "vector", 05:45:58 DEBUG "result": [ 05:45:58 DEBUG { 05:45:58 DEBUG "metric": { 05:45:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:45:58 DEBUG }, 05:45:58 DEBUG "value": [ 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG [...] 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:45:58 DEBUG }, 05:45:58 DEBUG "value": [ 05:45:58 DEBUG 1713501958, 05:45:58 DEBUG "2288394240" 05:45:58 DEBUG ] 05:45:58 DEBUG } 05:45:58 DEBUG ] 05:45:58 DEBUG } 05:45:58 DEBUG } 05:45:58 INFO [http_cmd]: http status code OK 05:45:58 DEBUG --- status code --- 05:45:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:45:58 DEBUG --- http response --- 05:45:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501958" 05:45:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501958" 05:45:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:45:58 DEBUG ----- output ----- 05:45:58 DEBUG { 05:45:58 DEBUG "status": "success", 05:45:58 DEBUG "data": { 05:45:58 DEBUG "resultType": "vector", 05:45:58 DEBUG "result": [ 05:45:58 DEBUG { 05:45:58 DEBUG "metric": { 05:45:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:45:58 DEBUG }, 05:45:58 DEBUG "value": [ 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG [...] 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:45:58 DEBUG }, 05:45:58 DEBUG "value": [ 05:45:58 DEBUG 1713501958, 05:45:58 DEBUG "2164.2" 05:45:58 DEBUG ] 05:45:58 DEBUG } 05:45:58 DEBUG ] 05:45:58 DEBUG } 05:45:58 DEBUG } 05:45:58 INFO [http_cmd]: http status code OK 05:45:58 INFO [http_cmd]: http status code OK 05:45:58 DEBUG --- status code --- 05:45:58 DEBUG --- status code --- 05:45:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:45:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:45:58 DEBUG --- http response --- 05:45:58 DEBUG --- http response --- 05:45:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:45:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 05:45:58 DEBUG ----- output ----- 05:45:58 DEBUG ----- output ----- 05:45:58 DEBUG { 05:45:58 DEBUG { 05:45:58 DEBUG "status": "success", 05:45:58 DEBUG "status": "success", 05:45:58 DEBUG "data": { 05:45:58 DEBUG "data": { 05:45:58 DEBUG "resultType": "vector", 05:45:58 DEBUG "resultType": "vector", 05:45:58 DEBUG "result": [ 05:45:58 DEBUG "result": [ 05:45:58 DEBUG { 05:45:58 DEBUG { 05:45:58 DEBUG "metric": { 05:45:58 DEBUG "metric": { 05:45:58 DEBUG "instance": "10.142.0.107:9100" 05:45:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:45:58 DEBUG }, 05:45:58 DEBUG }, 05:45:58 DEBUG "value": [ 05:45:58 DEBUG "value": [ 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG [...] 05:45:58 DEBUG [...] 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG "instance": "10.142.0.43:9100" 05:45:58 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 05:45:58 DEBUG }, 05:45:58 DEBUG }, 05:45:58 DEBUG "value": [ 05:45:58 DEBUG "value": [ 05:45:58 DEBUG 1713501958, 05:45:58 DEBUG 1713501958, 05:45:58 DEBUG "0.17126666666667065" 05:45:58 DEBUG "0" 05:45:58 DEBUG ] 05:45:58 DEBUG ] 05:45:58 DEBUG } 05:45:58 DEBUG } 05:45:58 DEBUG ] 05:45:58 DEBUG ] 05:45:58 DEBUG } 05:45:58 DEBUG } 05:45:58 DEBUG } 05:45:58 DEBUG } 05:45:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501958" 05:45:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713501958" 05:45:58 INFO [http_cmd]: http status code OK 05:45:58 DEBUG --- status code --- 05:45:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:45:58 DEBUG --- http response --- 05:45:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 05:45:58 DEBUG ----- output ----- 05:45:58 DEBUG { 05:45:58 DEBUG "status": "success", 05:45:58 DEBUG "data": { 05:45:58 DEBUG "resultType": "vector", 05:45:58 DEBUG "result": [ 05:45:58 DEBUG { 05:45:58 DEBUG "metric": { 05:45:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:45:58 DEBUG }, 05:45:58 DEBUG "value": [ 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG [...] 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 05:45:58 DEBUG }, 05:45:58 DEBUG "value": [ 05:45:58 DEBUG 1713501958, 05:45:58 DEBUG "0" 05:45:58 DEBUG ] 05:45:58 DEBUG } 05:45:58 DEBUG ] 05:45:58 DEBUG } 05:45:58 DEBUG } 05:45:58 INFO [http_cmd]: http status code OK 05:45:58 DEBUG --- status code --- 05:45:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:45:58 DEBUG --- http response --- 05:45:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:45:58 DEBUG ----- output ----- 05:45:58 DEBUG { 05:45:58 DEBUG "status": "success", 05:45:58 DEBUG "data": { 05:45:58 DEBUG "resultType": "vector", 05:45:58 DEBUG "result": [ 05:45:58 DEBUG { 05:45:58 DEBUG "metric": { 05:45:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:45:58 DEBUG }, 05:45:58 DEBUG "value": [ 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG [...] 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:45:58 DEBUG }, 05:45:58 DEBUG "value": [ 05:45:58 DEBUG 1713501958, 05:45:58 DEBUG "2164.233333333333" 05:45:58 DEBUG ] 05:45:58 DEBUG } 05:45:58 DEBUG ] 05:45:58 DEBUG } 05:45:58 DEBUG } 05:45:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501958" 05:45:58 INFO [http_cmd]: http status code OK 05:45:58 DEBUG --- status code --- 05:45:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:45:58 DEBUG --- http response --- 05:45:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:45:58 DEBUG ----- output ----- 05:45:58 DEBUG { 05:45:58 DEBUG "status": "success", 05:45:58 DEBUG "data": { 05:45:58 DEBUG "resultType": "vector", 05:45:58 DEBUG "result": [ 05:45:58 DEBUG { 05:45:58 DEBUG "metric": { 05:45:58 DEBUG "instance": "10.142.0.107:9100" 05:45:58 DEBUG }, 05:45:58 DEBUG "value": [ 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG [...] 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG "instance": "10.142.0.43:9100" 05:45:58 DEBUG }, 05:45:58 DEBUG "value": [ 05:45:58 DEBUG 1713501958, 05:45:58 DEBUG "52367043.36666667" 05:45:58 DEBUG ] 05:45:58 DEBUG } 05:45:58 DEBUG ] 05:45:58 DEBUG } 05:45:58 DEBUG } 05:45:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713501958" 05:45:58 INFO [http_cmd]: http status code OK 05:45:58 DEBUG --- status code --- 05:45:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:45:58 DEBUG --- http response --- 05:45:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:45:58 DEBUG ----- output ----- 05:45:58 DEBUG { 05:45:58 DEBUG "status": "success", 05:45:58 DEBUG "data": { 05:45:58 DEBUG "resultType": "vector", 05:45:58 DEBUG "result": [ 05:45:58 DEBUG { 05:45:58 DEBUG "metric": { 05:45:58 DEBUG "instance": "10.142.0.107:9100" 05:45:58 DEBUG }, 05:45:58 DEBUG "value": [ 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG [...] 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG 05:45:58 DEBUG "instance": "10.142.0.43:9100" 05:45:58 DEBUG }, 05:45:58 DEBUG "value": [ 05:45:58 DEBUG 1713501958, 05:45:58 DEBUG "52359959.06666666" 05:45:58 DEBUG ] 05:45:58 DEBUG } 05:45:58 DEBUG ] 05:45:58 DEBUG } 05:45:58 DEBUG } 05:46:00 INFO 05:46:00 INFO [loop_until]: kubectl --namespace=xlou top pods 05:46:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:46:00 INFO [loop_until]: OK (rc = 0) 05:46:00 DEBUG --- stdout --- 05:46:00 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 11644m 9138Mi am-6fd554f8f7-hvvkw 11638m 8911Mi am-6fd554f8f7-pmj4b 11990m 8791Mi ds-cts-0 10835m 13023Mi ds-cts-1 11636m 13015Mi ds-cts-2 11628m 13058Mi ds-idrepo-0 5225m 3268Mi ds-idrepo-1 5337m 3199Mi ds-idrepo-2 5191m 3066Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 4m 1215Mi lodemon-6b89d774d5-rf7sk 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 6740m 2088Mi 05:46:00 DEBUG --- stderr --- 05:46:00 DEBUG 05:46:01 INFO 05:46:01 INFO [loop_until]: kubectl --namespace=xlou top node 05:46:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:46:01 INFO [loop_until]: OK (rc = 0) 05:46:01 DEBUG --- stdout --- 05:46:01 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 71m 0% 6244Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5442m 18% 6725Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 10912m 36% 11630Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 84m 0% 3245Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 11814m 39% 15615Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 154m 0% 5172Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 11574m 38% 11922Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 11252m 37% 2735Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 13288m 44% 11009Mi 9% gke-xlou-cdm-ds-0f601751-j488 5728m 19% 6177Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 12805m 42% 17087Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 5958m 19% 6874Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 12171m 40% 15069Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 7032m 44% 5777Mi 9% 05:46:01 DEBUG --- stderr --- 05:46:01 DEBUG 05:46:21 WARNING File /tmp/lodemon_data-20240419_044251/node_disk_read_bytes_total.json does not exist. Skipping average value computation... 127.0.0.1 - - [19/Apr/2024 05:46:21] "GET /monitoring/average?start_time=20240419_041709&stop_time=20240419_044421 HTTP/1.1" 200 - 05:46:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713502017" 05:46:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713502017" 05:46:57 INFO [http_cmd]: http status code OK 05:46:57 INFO [http_cmd]: http status code OK 05:46:57 DEBUG --- status code --- 05:46:57 DEBUG --- status code --- 05:46:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:46:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:46:57 DEBUG --- http response --- 05:46:57 DEBUG --- http response --- 05:46:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:46:57 DEBUG ----- output ----- 05:46:57 DEBUG { 05:46:57 DEBUG "status": "success", 05:46:57 DEBUG "data": { 05:46:57 DEBUG "resultType": "vector", 05:46:57 DEBUG "result": [ 05:46:57 DEBUG { 05:46:57 DEBUG "metric": { 05:46:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:46:57 DEBUG }, 05:46:57 DEBUG "value": [ 05:46:57 DEBUG 05:46:57 DEBUG 05:46:57 DEBUG 05:46:57 DEBUG [...] 05:46:57 DEBUG 05:46:57 DEBUG 05:46:57 DEBUG 05:46:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:46:57 DEBUG }, 05:46:57 DEBUG "value": [ 05:46:57 DEBUG 1713502017, 05:46:57 DEBUG "55.176" 05:46:57 DEBUG ] 05:46:57 DEBUG } 05:46:57 DEBUG ] 05:46:57 DEBUG } 05:46:57 DEBUG } 05:46:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:46:57 DEBUG ----- output ----- 05:46:57 DEBUG { 05:46:57 DEBUG "status": "success", 05:46:57 DEBUG "data": { 05:46:57 DEBUG "resultType": "vector", 05:46:57 DEBUG "result": [ 05:46:57 DEBUG { 05:46:57 DEBUG "metric": { 05:46:57 DEBUG "instance": "10.142.0.107:9100" 05:46:57 DEBUG }, 05:46:57 DEBUG "value": [ 05:46:57 DEBUG 05:46:57 DEBUG 05:46:57 DEBUG 05:46:57 DEBUG [...] 05:46:57 DEBUG 05:46:57 DEBUG 05:46:57 DEBUG 05:46:57 DEBUG "instance": "10.142.0.43:9100" 05:46:57 DEBUG }, 05:46:57 DEBUG "value": [ 05:46:57 DEBUG 1713502017, 05:46:57 DEBUG "570717633536" 05:46:57 DEBUG ] 05:46:57 DEBUG } 05:46:57 DEBUG ] 05:46:57 DEBUG } 05:46:57 DEBUG } 05:46:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502017" 05:46:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502017" 05:46:57 INFO [http_cmd]: http status code OK 05:46:57 DEBUG --- status code --- 05:46:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:46:57 DEBUG --- http response --- 05:46:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:46:57 DEBUG ----- output ----- 05:46:57 DEBUG { 05:46:57 DEBUG "status": "success", 05:46:57 DEBUG "data": { 05:46:57 DEBUG "resultType": "vector", 05:46:57 DEBUG "result": [ 05:46:57 DEBUG { 05:46:57 DEBUG "metric": { 05:46:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:46:57 DEBUG }, 05:46:57 DEBUG "value": [ 05:46:57 DEBUG 05:46:57 DEBUG 05:46:57 DEBUG 05:46:57 DEBUG [...] 05:46:57 DEBUG 05:46:57 DEBUG 05:46:57 DEBUG 05:46:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:46:57 DEBUG }, 05:46:57 DEBUG "value": [ 05:46:57 DEBUG 1713502017, 05:46:57 DEBUG "0" 05:46:57 DEBUG ] 05:46:57 DEBUG } 05:46:57 DEBUG ] 05:46:57 DEBUG } 05:46:57 DEBUG } 05:46:57 INFO [http_cmd]: http status code OK 05:46:57 DEBUG --- status code --- 05:46:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:46:57 DEBUG --- http response --- 05:46:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:46:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502017" 05:46:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713502017" 05:46:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502017" 05:46:57 INFO [http_cmd]: http status code OK 05:46:57 INFO [http_cmd]: http status code OK 05:46:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713502017" 05:46:57 DEBUG --- status code --- 05:46:57 DEBUG --- status code --- 05:46:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:46:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:46:57 DEBUG --- http response --- 05:46:57 DEBUG --- http response --- 05:46:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:46:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:46:57 DEBUG ----- output ----- 05:46:57 DEBUG ----- output ----- 05:46:57 DEBUG { 05:46:57 DEBUG { 05:46:57 DEBUG "status": "success", 05:46:57 DEBUG "status": "success", 05:46:57 DEBUG "data": { 05:46:57 DEBUG "data": { 05:46:57 DEBUG "resultType": "vector", 05:46:57 DEBUG "resultType": "vector", 05:46:57 DEBUG "result": [ 05:46:57 DEBUG "result": [ 05:46:57 DEBUG { 05:46:57 DEBUG { 05:46:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502017" 05:46:57 DEBUG "metric": { 05:46:57 DEBUG "metric": { 05:46:57 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:46:57 DEBUG "pod": "ds-idrepo-0" 05:46:57 DEBUG }, 05:46:57 DEBUG }, 05:46:57 DEBUG "value": [ 05:46:57 DEBUG "value": [ 05:46:57 DEBUG 05:46:57 DEBUG 05:46:57 DEBUG 05:46:57 DEBUG 05:46:57 DEBUG 05:46:57 DEBUG 05:46:57 DEBUG [...] 05:46:57 DEBUG [...] 05:46:57 DEBUG 05:46:57 DEBUG 05:46:57 DEBUG 05:46:57 INFO [http_cmd]: http status code OK 05:46:57 DEBUG 05:46:57 DEBUG 05:46:57 DEBUG --- status code --- 05:46:57 DEBUG 05:46:57 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:46:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:46:57 DEBUG "pod": "ds-cts-2" 05:46:57 DEBUG }, 05:46:57 DEBUG --- http response --- 05:46:57 DEBUG }, 05:46:57 DEBUG "value": [ 05:46:57 DEBUG "value": [ 05:46:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:46:57 DEBUG 1713502017, 05:46:57 DEBUG 1713502017, 05:46:57 DEBUG ----- output ----- 05:46:57 DEBUG "0" 05:46:57 DEBUG "0" 05:46:57 DEBUG { 05:46:57 DEBUG ] 05:46:57 DEBUG ] 05:46:57 DEBUG "status": "success", 05:46:57 DEBUG } 05:46:57 DEBUG } 05:46:57 DEBUG "data": { 05:46:57 DEBUG ] 05:46:57 DEBUG ] 05:46:57 DEBUG "resultType": "vector", 05:46:57 DEBUG } 05:46:57 DEBUG } 05:46:57 DEBUG "result": [ 05:46:57 DEBUG } 05:46:57 DEBUG } 05:46:57 DEBUG { 05:46:57 DEBUG "metric": { 05:46:57 DEBUG "pod": "ds-cts-0" 05:46:57 DEBUG }, 05:46:57 DEBUG "value": [ 05:46:57 DEBUG 05:46:57 DEBUG 05:46:57 DEBUG 05:46:57 DEBUG [...] 05:46:57 DEBUG 05:46:57 DEBUG 05:46:57 DEBUG 05:46:57 DEBUG "pod": "ds-cts-2" 05:46:57 DEBUG }, 05:46:57 DEBUG "value": [ 05:46:57 DEBUG 1713502017, 05:46:57 DEBUG "310.2" 05:46:57 DEBUG ] 05:46:57 DEBUG } 05:46:57 DEBUG ] 05:46:57 DEBUG } 05:46:57 DEBUG } 05:46:57 ERROR [http_cmd]: ERROR 05:46:57 ERROR --- status code --- 05:46:57 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:46:57 ERROR --- http response --- 05:46:57 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:46:57 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:46:58 INFO [http_cmd]: http status code OK 05:46:58 DEBUG --- status code --- 05:46:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:46:58 DEBUG --- http response --- 05:46:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:46:58 DEBUG ----- output ----- 05:46:58 DEBUG { 05:46:58 DEBUG "status": "success", 05:46:58 DEBUG "data": { 05:46:58 DEBUG "resultType": "vector", 05:46:58 DEBUG "result": [ 05:46:58 DEBUG { 05:46:58 DEBUG "metric": { 05:46:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:46:58 DEBUG }, 05:46:58 DEBUG "value": [ 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG [...] 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG "pod": "ds-idrepo-2" 05:46:58 DEBUG }, 05:46:58 DEBUG "value": [ 05:46:58 DEBUG 1713502017, 05:46:58 DEBUG "149722458.4632726" 05:46:58 DEBUG ] 05:46:58 DEBUG } 05:46:58 DEBUG ] 05:46:58 DEBUG } 05:46:58 DEBUG } 05:46:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502018" 05:46:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502018" 05:46:58 INFO [http_cmd]: http status code OK 05:46:58 DEBUG --- status code --- 05:46:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:46:58 DEBUG --- http response --- 05:46:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:46:58 DEBUG ----- output ----- 05:46:58 DEBUG { 05:46:58 DEBUG "status": "success", 05:46:58 DEBUG "data": { 05:46:58 DEBUG "resultType": "vector", 05:46:58 DEBUG "result": [ 05:46:58 DEBUG { 05:46:58 DEBUG "metric": { 05:46:58 DEBUG "pod": "ds-idrepo-0" 05:46:58 DEBUG }, 05:46:58 DEBUG "value": [ 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG [...] 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG "pod": "ds-idrepo-2" 05:46:58 DEBUG }, 05:46:58 DEBUG "value": [ 05:46:58 DEBUG 1713502018, 05:46:58 DEBUG "0" 05:46:58 DEBUG ] 05:46:58 DEBUG } 05:46:58 DEBUG ] 05:46:58 DEBUG } 05:46:58 DEBUG } 05:46:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502018" 05:46:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713502018" 05:46:58 INFO [http_cmd]: http status code OK 05:46:58 DEBUG --- status code --- 05:46:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:46:58 DEBUG --- http response --- 05:46:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:46:58 DEBUG ----- output ----- 05:46:58 DEBUG { 05:46:58 DEBUG "status": "success", 05:46:58 DEBUG "data": { 05:46:58 DEBUG "resultType": "vector", 05:46:58 DEBUG "result": [ 05:46:58 DEBUG { 05:46:58 DEBUG "metric": { 05:46:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:46:58 DEBUG }, 05:46:58 DEBUG "value": [ 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG [...] 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:46:58 DEBUG }, 05:46:58 DEBUG "value": [ 05:46:58 DEBUG 1713502018, 05:46:58 DEBUG "3849947" 05:46:58 DEBUG ] 05:46:58 DEBUG } 05:46:58 DEBUG ] 05:46:58 DEBUG } 05:46:58 DEBUG } 05:46:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502018" 05:46:58 INFO [http_cmd]: http status code OK 05:46:58 DEBUG --- status code --- 05:46:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:46:58 DEBUG --- http response --- 05:46:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:46:58 DEBUG ----- output ----- 05:46:58 DEBUG { 05:46:58 DEBUG "status": "success", 05:46:58 DEBUG "data": { 05:46:58 DEBUG "resultType": "vector", 05:46:58 DEBUG "result": [ 05:46:58 DEBUG { 05:46:58 DEBUG "metric": { 05:46:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:46:58 DEBUG }, 05:46:58 DEBUG "value": [ 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG [...] 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:46:58 DEBUG }, 05:46:58 DEBUG "value": [ 05:46:58 DEBUG 1713502018, 05:46:58 DEBUG "2.8753441860590385" 05:46:58 DEBUG ] 05:46:58 DEBUG } 05:46:58 DEBUG ] 05:46:58 DEBUG } 05:46:58 DEBUG } 05:46:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502018" 05:46:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713502018" 05:46:58 INFO [http_cmd]: http status code OK 05:46:58 DEBUG --- status code --- 05:46:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:46:58 DEBUG --- http response --- 05:46:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:46:58 DEBUG ----- output ----- 05:46:58 DEBUG { 05:46:58 DEBUG "status": "success", 05:46:58 DEBUG "data": { 05:46:58 DEBUG "resultType": "vector", 05:46:58 DEBUG "result": [ 05:46:58 DEBUG { 05:46:58 DEBUG "metric": { 05:46:58 DEBUG "pod": "ds-cts-0" 05:46:58 DEBUG }, 05:46:58 DEBUG "value": [ 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG [...] 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG "pod": "ds-cts-2" 05:46:58 DEBUG }, 05:46:58 DEBUG "value": [ 05:46:58 DEBUG 1713502018, 05:46:58 DEBUG "29.966666666666665" 05:46:58 DEBUG ] 05:46:58 DEBUG } 05:46:58 DEBUG ] 05:46:58 DEBUG } 05:46:58 DEBUG } 05:46:58 INFO [http_cmd]: http status code OK 05:46:58 DEBUG --- status code --- 05:46:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:46:58 DEBUG --- http response --- 05:46:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:46:58 DEBUG ----- output ----- 05:46:58 DEBUG { 05:46:58 DEBUG "status": "success", 05:46:58 DEBUG "data": { 05:46:58 DEBUG "resultType": "vector", 05:46:58 DEBUG "result": [ 05:46:58 DEBUG { 05:46:58 DEBUG "metric": { 05:46:58 DEBUG "pod": "ds-idrepo-0" 05:46:58 DEBUG }, 05:46:58 DEBUG "value": [ 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG [...] 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG "pod": "ds-idrepo-2" 05:46:58 DEBUG }, 05:46:58 DEBUG "value": [ 05:46:58 DEBUG 1713502018, 05:46:58 DEBUG "0" 05:46:58 DEBUG ] 05:46:58 DEBUG } 05:46:58 DEBUG ] 05:46:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502018" 05:46:58 DEBUG } 05:46:58 INFO [http_cmd]: http status code OK 05:46:58 DEBUG } 05:46:58 DEBUG --- status code --- 05:46:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:46:58 DEBUG --- http response --- 05:46:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:46:58 DEBUG ----- output ----- 05:46:58 DEBUG { 05:46:58 DEBUG "status": "success", 05:46:58 DEBUG "data": { 05:46:58 DEBUG "resultType": "vector", 05:46:58 DEBUG "result": [ 05:46:58 DEBUG { 05:46:58 DEBUG "metric": { 05:46:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:46:58 DEBUG }, 05:46:58 DEBUG "value": [ 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG [...] 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG "pod": "ds-idrepo-2" 05:46:58 DEBUG }, 05:46:58 DEBUG "value": [ 05:46:58 DEBUG 1713502018, 05:46:58 DEBUG "0.5967246289687909" 05:46:58 DEBUG ] 05:46:58 DEBUG } 05:46:58 DEBUG ] 05:46:58 DEBUG } 05:46:58 DEBUG } 05:46:58 INFO [http_cmd]: http status code OK 05:46:58 DEBUG --- status code --- 05:46:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:46:58 DEBUG --- http response --- 05:46:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:46:58 DEBUG ----- output ----- 05:46:58 DEBUG { 05:46:58 DEBUG "status": "success", 05:46:58 DEBUG "data": { 05:46:58 DEBUG "resultType": "vector", 05:46:58 DEBUG "result": [ 05:46:58 DEBUG { 05:46:58 DEBUG "metric": { 05:46:58 DEBUG "instance": "10.142.0.107:9100" 05:46:58 DEBUG }, 05:46:58 DEBUG "value": [ 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG [...] 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG "instance": "10.142.0.43:9100" 05:46:58 DEBUG }, 05:46:58 DEBUG "value": [ 05:46:58 DEBUG 1713502018, 05:46:58 DEBUG "0.6549999999999998" 05:46:58 DEBUG ] 05:46:58 DEBUG } 05:46:58 DEBUG ] 05:46:58 DEBUG } 05:46:58 DEBUG } 05:46:58 INFO [http_cmd]: http status code OK 05:46:58 DEBUG --- status code --- 05:46:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:46:58 DEBUG --- http response --- 05:46:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:46:58 DEBUG ----- output ----- 05:46:58 DEBUG { 05:46:58 DEBUG "status": "success", 05:46:58 DEBUG "data": { 05:46:58 DEBUG "resultType": "vector", 05:46:58 DEBUG "result": [ 05:46:58 DEBUG { 05:46:58 DEBUG "metric": { 05:46:58 DEBUG "pod": "ds-idrepo-0" 05:46:58 DEBUG }, 05:46:58 DEBUG "value": [ 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG [...] 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG "pod": "ds-idrepo-2" 05:46:58 DEBUG }, 05:46:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502018" 05:46:58 DEBUG "value": [ 05:46:58 DEBUG 1713502018, 05:46:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502018" 05:46:58 DEBUG "0" 05:46:58 DEBUG ] 05:46:58 DEBUG } 05:46:58 DEBUG ] 05:46:58 DEBUG } 05:46:58 DEBUG } 05:46:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713502018" 05:46:58 INFO [http_cmd]: http status code OK 05:46:58 INFO [http_cmd]: http status code OK 05:46:58 DEBUG --- status code --- 05:46:58 DEBUG --- status code --- 05:46:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:46:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:46:58 DEBUG --- http response --- 05:46:58 DEBUG --- http response --- 05:46:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:46:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:46:58 DEBUG ----- output ----- 05:46:58 DEBUG ----- output ----- 05:46:58 DEBUG { 05:46:58 DEBUG { 05:46:58 DEBUG "status": "success", 05:46:58 DEBUG "status": "success", 05:46:58 DEBUG "data": { 05:46:58 DEBUG "data": { 05:46:58 DEBUG "resultType": "vector", 05:46:58 DEBUG "resultType": "vector", 05:46:58 DEBUG "result": [ 05:46:58 DEBUG "result": [ 05:46:58 DEBUG { 05:46:58 DEBUG { 05:46:58 DEBUG "metric": { 05:46:58 DEBUG "metric": { 05:46:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:46:58 DEBUG "pod": "ds-cts-0" 05:46:58 DEBUG }, 05:46:58 DEBUG }, 05:46:58 DEBUG "value": [ 05:46:58 DEBUG "value": [ 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG [...] 05:46:58 DEBUG [...] 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:46:58 DEBUG "pod": "ds-cts-2" 05:46:58 DEBUG }, 05:46:58 DEBUG }, 05:46:58 DEBUG "value": [ 05:46:58 DEBUG "value": [ 05:46:58 DEBUG 1713502018, 05:46:58 DEBUG 1713502018, 05:46:58 DEBUG "2169.366666666667" 05:46:58 DEBUG "0.00011596" 05:46:58 DEBUG ] 05:46:58 DEBUG ] 05:46:58 DEBUG } 05:46:58 DEBUG } 05:46:58 DEBUG ] 05:46:58 DEBUG ] 05:46:58 DEBUG } 05:46:58 DEBUG } 05:46:58 DEBUG } 05:46:58 DEBUG } 05:46:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502018" 05:46:58 INFO [http_cmd]: http status code OK 05:46:58 DEBUG --- status code --- 05:46:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:46:58 DEBUG --- http response --- 05:46:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:46:58 DEBUG ----- output ----- 05:46:58 DEBUG { 05:46:58 DEBUG "status": "success", 05:46:58 DEBUG "data": { 05:46:58 DEBUG "resultType": "vector", 05:46:58 DEBUG "result": [ 05:46:58 DEBUG { 05:46:58 DEBUG "metric": { 05:46:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:46:58 DEBUG }, 05:46:58 DEBUG "value": [ 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG [...] 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:46:58 DEBUG }, 05:46:58 DEBUG "value": [ 05:46:58 DEBUG 1713502018, 05:46:58 DEBUG "1742118912" 05:46:58 DEBUG ] 05:46:58 DEBUG } 05:46:58 DEBUG ] 05:46:58 DEBUG } 05:46:58 DEBUG } 05:46:58 INFO [http_cmd]: http status code OK 05:46:58 DEBUG --- status code --- 05:46:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:46:58 DEBUG --- http response --- 05:46:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:46:58 DEBUG ----- output ----- 05:46:58 DEBUG { 05:46:58 DEBUG "status": "success", 05:46:58 DEBUG "data": { 05:46:58 DEBUG "resultType": "vector", 05:46:58 DEBUG "result": [ 05:46:58 DEBUG { 05:46:58 DEBUG "metric": { 05:46:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:46:58 DEBUG }, 05:46:58 DEBUG "value": [ 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG [...] 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:46:58 DEBUG }, 05:46:58 DEBUG "value": [ 05:46:58 DEBUG 1713502018, 05:46:58 DEBUG "0" 05:46:58 DEBUG ] 05:46:58 DEBUG } 05:46:58 DEBUG ] 05:46:58 DEBUG } 05:46:58 DEBUG } 05:46:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502018" 05:46:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713502018" 05:46:58 INFO [http_cmd]: http status code OK 05:46:58 INFO [http_cmd]: http status code OK 05:46:58 DEBUG --- status code --- 05:46:58 DEBUG --- status code --- 05:46:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:46:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:46:58 DEBUG --- http response --- 05:46:58 DEBUG --- http response --- 05:46:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:46:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:46:58 DEBUG ----- output ----- 05:46:58 DEBUG ----- output ----- 05:46:58 DEBUG { 05:46:58 DEBUG { 05:46:58 DEBUG "status": "success", 05:46:58 DEBUG "status": "success", 05:46:58 DEBUG "data": { 05:46:58 DEBUG "data": { 05:46:58 DEBUG "resultType": "vector", 05:46:58 DEBUG "resultType": "vector", 05:46:58 DEBUG "result": [ 05:46:58 DEBUG "result": [ 05:46:58 DEBUG { 05:46:58 DEBUG { 05:46:58 DEBUG "metric": { 05:46:58 DEBUG "metric": { 05:46:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:46:58 DEBUG "instance": "10.142.0.107:9100" 05:46:58 DEBUG }, 05:46:58 DEBUG }, 05:46:58 DEBUG "value": [ 05:46:58 DEBUG "value": [ 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG [...] 05:46:58 DEBUG [...] 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG "pod": "ds-idrepo-2" 05:46:58 DEBUG "instance": "10.142.0.43:9100" 05:46:58 DEBUG }, 05:46:58 DEBUG }, 05:46:58 DEBUG "value": [ 05:46:58 DEBUG "value": [ 05:46:58 DEBUG 1713502018, 05:46:58 DEBUG 1713502018, 05:46:58 DEBUG "0" 05:46:58 DEBUG "0.5835999999999966" 05:46:58 DEBUG ] 05:46:58 DEBUG ] 05:46:58 DEBUG } 05:46:58 DEBUG } 05:46:58 DEBUG ] 05:46:58 DEBUG ] 05:46:58 DEBUG } 05:46:58 DEBUG } 05:46:58 DEBUG } 05:46:58 DEBUG } 05:46:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502018" 05:46:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502018" 05:46:58 INFO [http_cmd]: http status code OK 05:46:58 DEBUG --- status code --- 05:46:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:46:58 DEBUG --- http response --- 05:46:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:46:58 DEBUG ----- output ----- 05:46:58 DEBUG { 05:46:58 DEBUG "status": "success", 05:46:58 DEBUG "data": { 05:46:58 DEBUG "resultType": "vector", 05:46:58 DEBUG "result": [ 05:46:58 DEBUG { 05:46:58 DEBUG "metric": { 05:46:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:46:58 DEBUG }, 05:46:58 DEBUG "value": [ 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG [...] 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:46:58 DEBUG }, 05:46:58 DEBUG "value": [ 05:46:58 DEBUG 1713502018, 05:46:58 DEBUG "0" 05:46:58 DEBUG ] 05:46:58 DEBUG } 05:46:58 DEBUG ] 05:46:58 DEBUG } 05:46:58 DEBUG } 05:46:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713502018" 05:46:58 INFO [http_cmd]: http status code OK 05:46:58 DEBUG --- status code --- 05:46:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:46:58 DEBUG --- http response --- 05:46:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:46:58 DEBUG ----- output ----- 05:46:58 DEBUG { 05:46:58 DEBUG "status": "success", 05:46:58 DEBUG "data": { 05:46:58 DEBUG "resultType": "vector", 05:46:58 DEBUG "result": [ 05:46:58 DEBUG { 05:46:58 DEBUG "metric": { 05:46:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:46:58 DEBUG }, 05:46:58 DEBUG "value": [ 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG [...] 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG "pod": "ds-idrepo-2" 05:46:58 DEBUG }, 05:46:58 DEBUG "value": [ 05:46:58 DEBUG 1713502018, 05:46:58 DEBUG "4155.250798421943" 05:46:58 DEBUG ] 05:46:58 DEBUG } 05:46:58 DEBUG ] 05:46:58 DEBUG } 05:46:58 DEBUG } 05:46:58 INFO [http_cmd]: http status code OK 05:46:58 DEBUG --- status code --- 05:46:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:46:58 DEBUG --- http response --- 05:46:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:46:58 DEBUG ----- output ----- 05:46:58 DEBUG { 05:46:58 DEBUG "status": "success", 05:46:58 DEBUG "data": { 05:46:58 DEBUG "resultType": "vector", 05:46:58 DEBUG "result": [ 05:46:58 DEBUG { 05:46:58 DEBUG "metric": { 05:46:58 DEBUG "instance": "10.142.0.107:9100" 05:46:58 DEBUG }, 05:46:58 DEBUG "value": [ 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG [...] 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG "instance": "10.142.0.43:9100" 05:46:58 DEBUG }, 05:46:58 DEBUG "value": [ 05:46:58 DEBUG 1713502018, 05:46:58 DEBUG "14386516.833333332" 05:46:58 DEBUG ] 05:46:58 DEBUG } 05:46:58 DEBUG ] 05:46:58 DEBUG } 05:46:58 DEBUG } 05:46:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713502018" 05:46:58 INFO [http_cmd]: http status code OK 05:46:58 DEBUG --- status code --- 05:46:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:46:58 DEBUG --- http response --- 05:46:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:46:58 DEBUG ----- output ----- 05:46:58 DEBUG { 05:46:58 DEBUG "status": "success", 05:46:58 DEBUG "data": { 05:46:58 DEBUG "resultType": "vector", 05:46:58 DEBUG "result": [ 05:46:58 DEBUG { 05:46:58 DEBUG "metric": { 05:46:58 DEBUG "instance": "10.142.0.107:9100" 05:46:58 DEBUG }, 05:46:58 DEBUG "value": [ 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG [...] 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG 05:46:58 DEBUG "instance": "10.142.0.43:9100" 05:46:58 DEBUG }, 05:46:58 DEBUG "value": [ 05:46:58 DEBUG 1713502018, 05:46:58 DEBUG "14201206.533333333" 05:46:58 DEBUG ] 05:46:58 DEBUG } 05:46:58 DEBUG ] 05:46:58 DEBUG } 05:46:58 DEBUG } 05:47:00 INFO 05:47:00 INFO [loop_until]: kubectl --namespace=xlou top pods 05:47:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:47:00 INFO [loop_until]: OK (rc = 0) 05:47:00 DEBUG --- stdout --- 05:47:00 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 2116m 9145Mi am-6fd554f8f7-hvvkw 2024m 8913Mi am-6fd554f8f7-pmj4b 2100m 8797Mi ds-cts-0 9355m 13507Mi ds-cts-1 8024m 13718Mi ds-cts-2 9068m 13073Mi ds-idrepo-0 612m 3268Mi ds-idrepo-1 606m 3198Mi ds-idrepo-2 586m 3066Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 4m 1215Mi lodemon-6b89d774d5-rf7sk 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 1m 1566Mi 05:47:00 DEBUG --- stderr --- 05:47:00 DEBUG 05:47:01 INFO 05:47:01 INFO [loop_until]: kubectl --namespace=xlou top node 05:47:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:47:01 INFO [loop_until]: OK (rc = 0) 05:47:01 DEBUG --- stdout --- 05:47:01 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 74m 0% 6241Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 819m 2% 6733Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 2065m 6% 11672Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 3265Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 9301m 31% 16052Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 165m 0% 5173Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 2254m 7% 11956Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 101m 0% 2715Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 2314m 7% 11041Mi 9% gke-xlou-cdm-ds-0f601751-j488 680m 2% 6165Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 8043m 26% 17899Mi 16% gke-xlou-cdm-ds-0f601751-nj6q 691m 2% 6892Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 9234m 30% 15202Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 71m 0% 5261Mi 8% 05:47:01 DEBUG --- stderr --- 05:47:01 DEBUG 05:47:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713502077" 05:47:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713502077" 05:47:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502078" 05:47:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502078" 05:47:58 INFO [http_cmd]: http status code OK 05:47:58 DEBUG --- status code --- 05:47:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:47:58 DEBUG --- http response --- 05:47:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:47:58 DEBUG ----- output ----- 05:47:58 DEBUG { 05:47:58 DEBUG "status": "success", 05:47:58 DEBUG "data": { 05:47:58 DEBUG "resultType": "vector", 05:47:58 DEBUG "result": [ 05:47:58 DEBUG { 05:47:58 DEBUG "metric": { 05:47:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:47:58 DEBUG }, 05:47:58 DEBUG "value": [ 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG [...] 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:47:58 DEBUG }, 05:47:58 DEBUG "value": [ 05:47:58 DEBUG 1713502077, 05:47:58 DEBUG "55.176" 05:47:58 DEBUG ] 05:47:58 DEBUG } 05:47:58 DEBUG ] 05:47:58 DEBUG } 05:47:58 DEBUG } 05:47:58 INFO [http_cmd]: http status code OK 05:47:58 DEBUG --- status code --- 05:47:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:47:58 DEBUG --- http response --- 05:47:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:47:58 DEBUG ----- output ----- 05:47:58 DEBUG { 05:47:58 DEBUG "status": "success", 05:47:58 DEBUG "data": { 05:47:58 DEBUG "resultType": "vector", 05:47:58 DEBUG "result": [ 05:47:58 DEBUG { 05:47:58 DEBUG "metric": { 05:47:58 DEBUG "instance": "10.142.0.107:9100" 05:47:58 DEBUG }, 05:47:58 DEBUG "value": [ 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG [...] 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG "instance": "10.142.0.43:9100" 05:47:58 DEBUG }, 05:47:58 DEBUG "value": [ 05:47:58 DEBUG 1713502077, 05:47:58 DEBUG "571647048704" 05:47:58 DEBUG ] 05:47:58 DEBUG } 05:47:58 DEBUG ] 05:47:58 DEBUG } 05:47:58 DEBUG } 05:47:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713502078" 05:47:58 INFO [http_cmd]: http status code OK 05:47:58 INFO [http_cmd]: http status code OK 05:47:58 DEBUG --- status code --- 05:47:58 DEBUG --- status code --- 05:47:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:47:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:47:58 DEBUG --- http response --- 05:47:58 DEBUG --- http response --- 05:47:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:47:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:47:58 DEBUG ----- output ----- 05:47:58 DEBUG ----- output ----- 05:47:58 DEBUG { 05:47:58 DEBUG { 05:47:58 DEBUG "status": "success", 05:47:58 DEBUG "status": "success", 05:47:58 DEBUG "data": { 05:47:58 DEBUG "data": { 05:47:58 DEBUG "resultType": "vector", 05:47:58 DEBUG "resultType": "vector", 05:47:58 DEBUG "result": [ 05:47:58 DEBUG "result": [ 05:47:58 DEBUG { 05:47:58 DEBUG { 05:47:58 DEBUG "metric": { 05:47:58 DEBUG "metric": { 05:47:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:47:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:47:58 DEBUG }, 05:47:58 DEBUG }, 05:47:58 DEBUG "value": [ 05:47:58 DEBUG "value": [ 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG [...] 05:47:58 DEBUG [...] 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:47:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:47:58 DEBUG }, 05:47:58 DEBUG }, 05:47:58 DEBUG "value": [ 05:47:58 DEBUG "value": [ 05:47:58 DEBUG 1713502078, 05:47:58 DEBUG 1713502078, 05:47:58 DEBUG "0" 05:47:58 DEBUG "0" 05:47:58 DEBUG ] 05:47:58 DEBUG ] 05:47:58 DEBUG } 05:47:58 DEBUG } 05:47:58 DEBUG ] 05:47:58 DEBUG ] 05:47:58 DEBUG } 05:47:58 DEBUG } 05:47:58 DEBUG } 05:47:58 DEBUG } 05:47:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502078" 05:47:58 ERROR [http_cmd]: ERROR 05:47:58 ERROR --- status code --- 05:47:58 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:47:58 ERROR --- http response --- 05:47:58 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:47:58 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:47:58 INFO [http_cmd]: http status code OK 05:47:58 DEBUG --- status code --- 05:47:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:47:58 DEBUG --- http response --- 05:47:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:47:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502078" 05:47:58 INFO [http_cmd]: http status code OK 05:47:58 DEBUG --- status code --- 05:47:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:47:58 DEBUG --- http response --- 05:47:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:47:58 DEBUG ----- output ----- 05:47:58 DEBUG { 05:47:58 DEBUG "status": "success", 05:47:58 DEBUG "data": { 05:47:58 DEBUG "resultType": "vector", 05:47:58 DEBUG "result": [ 05:47:58 DEBUG { 05:47:58 DEBUG "metric": { 05:47:58 DEBUG "pod": "ds-cts-0" 05:47:58 DEBUG }, 05:47:58 DEBUG "value": [ 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG [...] 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG "pod": "ds-cts-2" 05:47:58 DEBUG }, 05:47:58 DEBUG "value": [ 05:47:58 DEBUG 1713502078, 05:47:58 DEBUG "278.1333333333333" 05:47:58 DEBUG ] 05:47:58 DEBUG } 05:47:58 DEBUG ] 05:47:58 DEBUG } 05:47:58 DEBUG } 05:47:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713502078" 05:47:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502078" 05:47:58 INFO [http_cmd]: http status code OK 05:47:58 DEBUG --- status code --- 05:47:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:47:58 DEBUG --- http response --- 05:47:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:47:58 DEBUG ----- output ----- 05:47:58 DEBUG { 05:47:58 DEBUG "status": "success", 05:47:58 DEBUG "data": { 05:47:58 DEBUG "resultType": "vector", 05:47:58 DEBUG "result": [ 05:47:58 DEBUG { 05:47:58 DEBUG "metric": { 05:47:58 DEBUG "pod": "ds-idrepo-0" 05:47:58 DEBUG }, 05:47:58 DEBUG "value": [ 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG [...] 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG "pod": "ds-cts-2" 05:47:58 DEBUG }, 05:47:58 DEBUG "value": [ 05:47:58 DEBUG 1713502078, 05:47:58 DEBUG "0" 05:47:58 DEBUG ] 05:47:58 DEBUG } 05:47:58 DEBUG ] 05:47:58 DEBUG } 05:47:58 DEBUG } 05:47:58 INFO [http_cmd]: http status code OK 05:47:58 DEBUG --- status code --- 05:47:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:47:58 DEBUG --- http response --- 05:47:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:47:58 DEBUG ----- output ----- 05:47:58 DEBUG { 05:47:58 DEBUG "status": "success", 05:47:58 DEBUG "data": { 05:47:58 DEBUG "resultType": "vector", 05:47:58 DEBUG "result": [ 05:47:58 DEBUG { 05:47:58 DEBUG "metric": { 05:47:58 DEBUG "pod": "ds-idrepo-1" 05:47:58 DEBUG }, 05:47:58 DEBUG "value": [ 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG [...] 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG "pod": "idm-b85f9d978-txpwv" 05:47:58 DEBUG }, 05:47:58 DEBUG "value": [ 05:47:58 DEBUG 1713502078, 05:47:58 DEBUG "0" 05:47:58 DEBUG ] 05:47:58 DEBUG } 05:47:58 DEBUG ] 05:47:58 DEBUG } 05:47:58 DEBUG } 05:47:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502078" 05:47:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502078" 05:47:58 INFO [http_cmd]: http status code OK 05:47:58 DEBUG --- status code --- 05:47:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:47:58 DEBUG --- http response --- 05:47:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:47:58 DEBUG ----- output ----- 05:47:58 DEBUG { 05:47:58 DEBUG "status": "success", 05:47:58 DEBUG "data": { 05:47:58 DEBUG "resultType": "vector", 05:47:58 DEBUG "result": [ 05:47:58 DEBUG { 05:47:58 DEBUG "metric": { 05:47:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:47:58 DEBUG }, 05:47:58 DEBUG "value": [ 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG [...] 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:47:58 DEBUG }, 05:47:58 DEBUG "value": [ 05:47:58 DEBUG 1713502078, 05:47:58 DEBUG "3719667.5" 05:47:58 DEBUG ] 05:47:58 DEBUG } 05:47:58 DEBUG ] 05:47:58 DEBUG } 05:47:58 DEBUG } 05:47:58 INFO [http_cmd]: http status code OK 05:47:58 DEBUG --- status code --- 05:47:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:47:58 DEBUG --- http response --- 05:47:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:47:58 DEBUG ----- output ----- 05:47:58 DEBUG { 05:47:58 DEBUG "status": "success", 05:47:58 DEBUG "data": { 05:47:58 DEBUG "resultType": "vector", 05:47:58 DEBUG "result": [ 05:47:58 DEBUG { 05:47:58 DEBUG "metric": { 05:47:58 DEBUG "pod": "ds-idrepo-0" 05:47:58 DEBUG }, 05:47:58 DEBUG "value": [ 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG [...] 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG "pod": "ds-idrepo-2" 05:47:58 DEBUG }, 05:47:58 DEBUG "value": [ 05:47:58 DEBUG 1713502078, 05:47:58 DEBUG "0.06666666666666667" 05:47:58 DEBUG ] 05:47:58 DEBUG } 05:47:58 DEBUG ] 05:47:58 DEBUG } 05:47:58 DEBUG } 05:47:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713502078" 05:47:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502078" 05:47:58 INFO [http_cmd]: http status code OK 05:47:58 DEBUG --- status code --- 05:47:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:47:58 DEBUG --- http response --- 05:47:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:47:58 DEBUG ----- output ----- 05:47:58 DEBUG { 05:47:58 DEBUG "status": "success", 05:47:58 DEBUG "data": { 05:47:58 DEBUG "resultType": "vector", 05:47:58 DEBUG "result": [ 05:47:58 DEBUG { 05:47:58 DEBUG "metric": { 05:47:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502078" 05:47:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:47:58 DEBUG }, 05:47:58 DEBUG "value": [ 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG [...] 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:47:58 DEBUG }, 05:47:58 DEBUG "value": [ 05:47:58 DEBUG 1713502078, 05:47:58 DEBUG "1.2830882089770594" 05:47:58 DEBUG ] 05:47:58 DEBUG } 05:47:58 DEBUG ] 05:47:58 DEBUG } 05:47:58 DEBUG } 05:47:58 INFO [http_cmd]: http status code OK 05:47:58 DEBUG --- status code --- 05:47:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:47:58 DEBUG --- http response --- 05:47:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:47:58 DEBUG ----- output ----- 05:47:58 DEBUG { 05:47:58 DEBUG "status": "success", 05:47:58 DEBUG "data": { 05:47:58 DEBUG "resultType": "vector", 05:47:58 DEBUG "result": [ 05:47:58 DEBUG { 05:47:58 DEBUG "metric": { 05:47:58 DEBUG "pod": "ds-idrepo-0" 05:47:58 DEBUG }, 05:47:58 DEBUG "value": [ 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG [...] 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG "pod": "ds-idrepo-2" 05:47:58 DEBUG }, 05:47:58 DEBUG "value": [ 05:47:58 DEBUG 1713502078, 05:47:58 DEBUG "0.06666666666666667" 05:47:58 DEBUG ] 05:47:58 DEBUG } 05:47:58 DEBUG ] 05:47:58 DEBUG } 05:47:58 DEBUG } 05:47:58 INFO [http_cmd]: http status code OK 05:47:58 DEBUG --- status code --- 05:47:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:47:58 DEBUG --- http response --- 05:47:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:47:58 DEBUG ----- output ----- 05:47:58 DEBUG { 05:47:58 DEBUG "status": "success", 05:47:58 DEBUG "data": { 05:47:58 DEBUG "resultType": "vector", 05:47:58 DEBUG "result": [ 05:47:58 DEBUG { 05:47:58 DEBUG "metric": { 05:47:58 DEBUG "pod": "ds-cts-0" 05:47:58 DEBUG }, 05:47:58 DEBUG "value": [ 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG [...] 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG "pod": "ds-cts-2" 05:47:58 DEBUG }, 05:47:58 DEBUG "value": [ 05:47:58 DEBUG 1713502078, 05:47:58 DEBUG "29.966666666666665" 05:47:58 DEBUG ] 05:47:58 DEBUG } 05:47:58 DEBUG ] 05:47:58 DEBUG } 05:47:58 DEBUG } 05:47:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502078" 05:47:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502078" 05:47:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502078" 05:47:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713502078" 05:47:58 INFO [http_cmd]: http status code OK 05:47:58 DEBUG --- status code --- 05:47:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:47:58 DEBUG --- http response --- 05:47:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:47:58 DEBUG ----- output ----- 05:47:58 DEBUG { 05:47:58 DEBUG "status": "success", 05:47:58 DEBUG "data": { 05:47:58 DEBUG "resultType": "vector", 05:47:58 DEBUG "result": [ 05:47:58 DEBUG { 05:47:58 DEBUG "metric": { 05:47:58 DEBUG "pod": "ds-idrepo-1" 05:47:58 DEBUG }, 05:47:58 DEBUG "value": [ 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG [...] 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG "pod": "idm-b85f9d978-txpwv" 05:47:58 DEBUG }, 05:47:58 DEBUG "value": [ 05:47:58 DEBUG 1713502078, 05:47:58 DEBUG "0" 05:47:58 DEBUG ] 05:47:58 DEBUG } 05:47:58 DEBUG ] 05:47:58 DEBUG } 05:47:58 DEBUG } 05:47:58 INFO [http_cmd]: http status code OK 05:47:58 DEBUG --- status code --- 05:47:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:47:58 DEBUG --- http response --- 05:47:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:47:58 DEBUG ----- output ----- 05:47:58 DEBUG { 05:47:58 DEBUG "status": "success", 05:47:58 DEBUG "data": { 05:47:58 DEBUG "resultType": "vector", 05:47:58 DEBUG "result": [ 05:47:58 DEBUG { 05:47:58 DEBUG "metric": { 05:47:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:47:58 DEBUG }, 05:47:58 DEBUG "value": [ 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG [...] 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:47:58 DEBUG }, 05:47:58 DEBUG "value": [ 05:47:58 DEBUG 1713502078, 05:47:58 DEBUG "2167.4666666666667" 05:47:58 DEBUG ] 05:47:58 DEBUG } 05:47:58 DEBUG ] 05:47:58 DEBUG } 05:47:58 DEBUG } 05:47:58 INFO [http_cmd]: http status code OK 05:47:58 DEBUG --- status code --- 05:47:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:47:58 DEBUG --- http response --- 05:47:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:47:58 DEBUG ----- output ----- 05:47:58 DEBUG { 05:47:58 DEBUG "status": "success", 05:47:58 DEBUG "data": { 05:47:58 DEBUG "resultType": "vector", 05:47:58 DEBUG "result": [ 05:47:58 DEBUG { 05:47:58 DEBUG "metric": { 05:47:58 DEBUG "pod": "ds-idrepo-0" 05:47:58 DEBUG }, 05:47:58 DEBUG "value": [ 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG [...] 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG "pod": "ds-idrepo-2" 05:47:58 DEBUG }, 05:47:58 DEBUG "value": [ 05:47:58 DEBUG 1713502078, 05:47:58 DEBUG "0.06666666666666667" 05:47:58 DEBUG ] 05:47:58 DEBUG } 05:47:58 DEBUG ] 05:47:58 DEBUG } 05:47:58 DEBUG } 05:47:58 INFO [http_cmd]: http status code OK 05:47:58 DEBUG --- status code --- 05:47:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:47:58 DEBUG --- http response --- 05:47:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:47:58 DEBUG ----- output ----- 05:47:58 DEBUG { 05:47:58 DEBUG "status": "success", 05:47:58 DEBUG "data": { 05:47:58 DEBUG "resultType": "vector", 05:47:58 DEBUG "result": [ 05:47:58 DEBUG { 05:47:58 DEBUG "metric": { 05:47:58 DEBUG "instance": "10.142.0.107:9100" 05:47:58 DEBUG }, 05:47:58 DEBUG "value": [ 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG [...] 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG "instance": "10.142.0.43:9100" 05:47:58 DEBUG }, 05:47:58 DEBUG "value": [ 05:47:58 DEBUG 1713502078, 05:47:58 DEBUG "0.6676666666666662" 05:47:58 DEBUG ] 05:47:58 DEBUG } 05:47:58 DEBUG ] 05:47:58 DEBUG } 05:47:58 DEBUG } 05:47:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502078" 05:47:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502078" 05:47:58 INFO [http_cmd]: http status code OK 05:47:58 INFO [http_cmd]: http status code OK 05:47:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713502078" 05:47:58 DEBUG --- status code --- 05:47:58 DEBUG --- status code --- 05:47:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:47:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:47:58 DEBUG --- http response --- 05:47:58 DEBUG --- http response --- 05:47:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:47:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:47:58 DEBUG ----- output ----- 05:47:58 DEBUG ----- output ----- 05:47:58 DEBUG { 05:47:58 DEBUG { 05:47:58 DEBUG "status": "success", 05:47:58 DEBUG "status": "success", 05:47:58 DEBUG "data": { 05:47:58 DEBUG "data": { 05:47:58 DEBUG "resultType": "vector", 05:47:58 DEBUG "resultType": "vector", 05:47:58 DEBUG "result": [ 05:47:58 DEBUG "result": [ 05:47:58 DEBUG { 05:47:58 DEBUG { 05:47:58 DEBUG "metric": { 05:47:58 DEBUG "metric": { 05:47:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:47:58 DEBUG "pod": "ds-cts-0" 05:47:58 DEBUG }, 05:47:58 DEBUG }, 05:47:58 DEBUG "value": [ 05:47:58 DEBUG "value": [ 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG [...] 05:47:58 DEBUG [...] 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:47:58 DEBUG "pod": "ds-cts-2" 05:47:58 DEBUG }, 05:47:58 DEBUG }, 05:47:58 DEBUG "value": [ 05:47:58 DEBUG "value": [ 05:47:58 DEBUG 1713502078, 05:47:58 DEBUG 1713502078, 05:47:58 DEBUG "0" 05:47:58 DEBUG "0.0002546133333333333" 05:47:58 DEBUG ] 05:47:58 DEBUG ] 05:47:58 DEBUG } 05:47:58 DEBUG } 05:47:58 DEBUG ] 05:47:58 DEBUG ] 05:47:58 DEBUG } 05:47:58 DEBUG } 05:47:58 DEBUG } 05:47:58 DEBUG } 05:47:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502078" 05:47:58 INFO [http_cmd]: http status code OK 05:47:58 DEBUG --- status code --- 05:47:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:47:58 DEBUG --- http response --- 05:47:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:47:58 DEBUG ----- output ----- 05:47:58 DEBUG { 05:47:58 DEBUG "status": "success", 05:47:58 DEBUG "data": { 05:47:58 DEBUG "resultType": "vector", 05:47:58 DEBUG "result": [ 05:47:58 DEBUG { 05:47:58 DEBUG "metric": { 05:47:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:47:58 DEBUG }, 05:47:58 DEBUG "value": [ 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG [...] 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:47:58 DEBUG }, 05:47:58 DEBUG "value": [ 05:47:58 DEBUG 1713502078, 05:47:58 DEBUG "2104377344" 05:47:58 DEBUG ] 05:47:58 DEBUG } 05:47:58 DEBUG ] 05:47:58 DEBUG } 05:47:58 DEBUG } 05:47:58 INFO [http_cmd]: http status code OK 05:47:58 DEBUG --- status code --- 05:47:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:47:58 DEBUG --- http response --- 05:47:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 05:47:58 DEBUG ----- output ----- 05:47:58 DEBUG { 05:47:58 DEBUG "status": "success", 05:47:58 DEBUG "data": { 05:47:58 DEBUG "resultType": "vector", 05:47:58 DEBUG "result": [ 05:47:58 DEBUG { 05:47:58 DEBUG "metric": { 05:47:58 DEBUG "pod": "ds-idrepo-1" 05:47:58 DEBUG }, 05:47:58 DEBUG "value": [ 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG [...] 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:47:58 DEBUG }, 05:47:58 DEBUG "value": [ 05:47:58 DEBUG 1713502078, 05:47:58 DEBUG "0" 05:47:58 DEBUG ] 05:47:58 DEBUG } 05:47:58 DEBUG ] 05:47:58 DEBUG } 05:47:58 DEBUG } 05:47:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502078" 05:47:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713502078" 05:47:58 INFO [http_cmd]: http status code OK 05:47:58 INFO [http_cmd]: http status code OK 05:47:58 DEBUG --- status code --- 05:47:58 DEBUG --- status code --- 05:47:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:47:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:47:58 DEBUG --- http response --- 05:47:58 DEBUG --- http response --- 05:47:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 05:47:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:47:58 DEBUG ----- output ----- 05:47:58 DEBUG ----- output ----- 05:47:58 DEBUG { 05:47:58 DEBUG { 05:47:58 DEBUG "status": "success", 05:47:58 DEBUG "status": "success", 05:47:58 DEBUG "data": { 05:47:58 DEBUG "data": { 05:47:58 DEBUG "resultType": "vector", 05:47:58 DEBUG "resultType": "vector", 05:47:58 DEBUG "result": [ 05:47:58 DEBUG "result": [ 05:47:58 DEBUG { 05:47:58 DEBUG { 05:47:58 DEBUG "metric": { 05:47:58 DEBUG "metric": { 05:47:58 DEBUG "pod": "ds-idrepo-1" 05:47:58 DEBUG "instance": "10.142.0.107:9100" 05:47:58 DEBUG }, 05:47:58 DEBUG }, 05:47:58 DEBUG "value": [ 05:47:58 DEBUG "value": [ 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG [...] 05:47:58 DEBUG [...] 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:47:58 DEBUG "instance": "10.142.0.43:9100" 05:47:58 DEBUG }, 05:47:58 DEBUG }, 05:47:58 DEBUG "value": [ 05:47:58 DEBUG "value": [ 05:47:58 DEBUG 1713502078, 05:47:58 DEBUG 1713502078, 05:47:58 DEBUG "0" 05:47:58 DEBUG "0.6856000000000203" 05:47:58 DEBUG ] 05:47:58 DEBUG ] 05:47:58 DEBUG } 05:47:58 DEBUG } 05:47:58 DEBUG ] 05:47:58 DEBUG ] 05:47:58 DEBUG } 05:47:58 DEBUG } 05:47:58 DEBUG } 05:47:58 DEBUG } 05:47:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713502078" 05:47:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502078" 05:47:58 INFO [http_cmd]: http status code OK 05:47:58 DEBUG --- status code --- 05:47:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:47:58 DEBUG --- http response --- 05:47:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:47:58 DEBUG ----- output ----- 05:47:58 DEBUG { 05:47:58 DEBUG "status": "success", 05:47:58 DEBUG "data": { 05:47:58 DEBUG "resultType": "vector", 05:47:58 DEBUG "result": [ 05:47:58 DEBUG { 05:47:58 DEBUG "metric": { 05:47:58 DEBUG "instance": "10.142.0.107:9100" 05:47:58 DEBUG }, 05:47:58 DEBUG "value": [ 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG [...] 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG "instance": "10.142.0.43:9100" 05:47:58 DEBUG }, 05:47:58 DEBUG "value": [ 05:47:58 DEBUG 1713502078, 05:47:58 DEBUG "14182274.266666666" 05:47:58 DEBUG ] 05:47:58 DEBUG } 05:47:58 DEBUG ] 05:47:58 DEBUG } 05:47:58 DEBUG } 05:47:58 INFO [http_cmd]: http status code OK 05:47:58 DEBUG --- status code --- 05:47:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:47:58 DEBUG --- http response --- 05:47:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:47:58 DEBUG ----- output ----- 05:47:58 DEBUG { 05:47:58 DEBUG "status": "success", 05:47:58 DEBUG "data": { 05:47:58 DEBUG "resultType": "vector", 05:47:58 DEBUG "result": [ 05:47:58 DEBUG { 05:47:58 DEBUG "metric": { 05:47:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:47:58 DEBUG }, 05:47:58 DEBUG "value": [ 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG [...] 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:47:58 DEBUG }, 05:47:58 DEBUG "value": [ 05:47:58 DEBUG 1713502078, 05:47:58 DEBUG "0" 05:47:58 DEBUG ] 05:47:58 DEBUG } 05:47:58 DEBUG ] 05:47:58 DEBUG } 05:47:58 DEBUG } 05:47:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713502078" 05:47:58 INFO [http_cmd]: http status code OK 05:47:58 DEBUG --- status code --- 05:47:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:47:58 DEBUG --- http response --- 05:47:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:47:58 DEBUG ----- output ----- 05:47:58 DEBUG { 05:47:58 DEBUG "status": "success", 05:47:58 DEBUG "data": { 05:47:58 DEBUG "resultType": "vector", 05:47:58 DEBUG "result": [ 05:47:58 DEBUG { 05:47:58 DEBUG "metric": { 05:47:58 DEBUG "instance": "10.142.0.107:9100" 05:47:58 DEBUG }, 05:47:58 DEBUG "value": [ 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG [...] 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG 05:47:58 DEBUG "instance": "10.142.0.43:9100" 05:47:58 DEBUG }, 05:47:58 DEBUG "value": [ 05:47:58 DEBUG 1713502078, 05:47:58 DEBUG "14172955.4" 05:47:58 DEBUG ] 05:47:58 DEBUG } 05:47:58 DEBUG ] 05:47:58 DEBUG } 05:47:58 DEBUG } 05:48:00 INFO 05:48:00 INFO [loop_until]: kubectl --namespace=xlou top pods 05:48:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:48:00 INFO [loop_until]: OK (rc = 0) 05:48:00 DEBUG --- stdout --- 05:48:00 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 1982m 9113Mi am-6fd554f8f7-hvvkw 2070m 8881Mi am-6fd554f8f7-pmj4b 2175m 8771Mi ds-cts-0 8750m 13523Mi ds-cts-1 8273m 12819Mi ds-cts-2 9171m 13217Mi ds-idrepo-0 1056m 3297Mi ds-idrepo-1 715m 3213Mi ds-idrepo-2 926m 3092Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 4m 1216Mi lodemon-6b89d774d5-rf7sk 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 1015m 2080Mi 05:48:00 DEBUG --- stderr --- 05:48:00 DEBUG 05:48:01 INFO 05:48:01 INFO [loop_until]: kubectl --namespace=xlou top node 05:48:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:48:01 INFO [loop_until]: OK (rc = 0) 05:48:01 DEBUG --- stdout --- 05:48:01 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 72m 0% 6235Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 1088m 3% 6723Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 2072m 6% 11646Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 3264Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 8844m 29% 16113Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 154m 0% 5155Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 2216m 7% 11929Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 76m 0% 2715Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 2324m 7% 11010Mi 9% gke-xlou-cdm-ds-0f601751-j488 814m 2% 6183Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 8297m 27% 16950Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 809m 2% 6883Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 9045m 30% 15330Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 1075m 6% 5806Mi 9% 05:48:01 DEBUG --- stderr --- 05:48:01 DEBUG 05:48:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502138" 05:48:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713502138" 05:48:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713502138" 05:48:58 INFO [http_cmd]: http status code OK 05:48:58 DEBUG --- status code --- 05:48:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:48:58 DEBUG --- http response --- 05:48:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:48:58 DEBUG ----- output ----- 05:48:58 DEBUG { 05:48:58 DEBUG "status": "success", 05:48:58 DEBUG "data": { 05:48:58 DEBUG "resultType": "vector", 05:48:58 DEBUG "result": [ 05:48:58 DEBUG { 05:48:58 DEBUG "metric": { 05:48:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:48:58 DEBUG }, 05:48:58 DEBUG "value": [ 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG [...] 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:48:58 DEBUG }, 05:48:58 DEBUG "value": [ 05:48:58 DEBUG 1713502138, 05:48:58 DEBUG "0" 05:48:58 DEBUG ] 05:48:58 DEBUG } 05:48:58 DEBUG ] 05:48:58 DEBUG } 05:48:58 DEBUG } 05:48:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502138" 05:48:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502138" 05:48:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713502138" 05:48:58 INFO [http_cmd]: http status code OK 05:48:58 INFO [http_cmd]: http status code OK 05:48:58 DEBUG --- status code --- 05:48:58 DEBUG --- status code --- 05:48:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:48:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:48:58 DEBUG --- http response --- 05:48:58 DEBUG --- http response --- 05:48:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:48:58 DEBUG ----- output ----- 05:48:58 DEBUG { 05:48:58 DEBUG "status": "success", 05:48:58 DEBUG "data": { 05:48:58 DEBUG "resultType": "vector", 05:48:58 DEBUG "result": [ 05:48:58 DEBUG { 05:48:58 DEBUG "metric": { 05:48:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:48:58 DEBUG }, 05:48:58 DEBUG "value": [ 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG [...] 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:48:58 DEBUG }, 05:48:58 DEBUG "value": [ 05:48:58 DEBUG 1713502138, 05:48:58 DEBUG "55.176" 05:48:58 DEBUG ] 05:48:58 DEBUG } 05:48:58 DEBUG ] 05:48:58 DEBUG } 05:48:58 DEBUG } 05:48:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:48:58 DEBUG ----- output ----- 05:48:58 DEBUG { 05:48:58 DEBUG "status": "success", 05:48:58 DEBUG "data": { 05:48:58 DEBUG "resultType": "vector", 05:48:58 DEBUG "result": [ 05:48:58 DEBUG { 05:48:58 DEBUG "metric": { 05:48:58 DEBUG "instance": "10.142.0.107:9100" 05:48:58 DEBUG }, 05:48:58 DEBUG "value": [ 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG [...] 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG "instance": "10.142.0.43:9100" 05:48:58 DEBUG }, 05:48:58 DEBUG "value": [ 05:48:58 INFO [http_cmd]: http status code OK 05:48:58 DEBUG 1713502138, 05:48:58 DEBUG --- status code --- 05:48:58 DEBUG "572693101568" 05:48:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:48:58 DEBUG ] 05:48:58 DEBUG --- http response --- 05:48:58 DEBUG } 05:48:58 DEBUG ] 05:48:58 DEBUG } 05:48:58 DEBUG } 05:48:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:48:58 DEBUG ----- output ----- 05:48:58 DEBUG { 05:48:58 DEBUG "status": "success", 05:48:58 DEBUG "data": { 05:48:58 DEBUG "resultType": "vector", 05:48:58 DEBUG "result": [ 05:48:58 DEBUG { 05:48:58 DEBUG "metric": { 05:48:58 DEBUG "pod": "ds-idrepo-0" 05:48:58 DEBUG }, 05:48:58 DEBUG "value": [ 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG [...] 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:48:58 DEBUG }, 05:48:58 DEBUG "value": [ 05:48:58 DEBUG 1713502138, 05:48:58 DEBUG "134.38328981723237" 05:48:58 DEBUG ] 05:48:58 DEBUG } 05:48:58 DEBUG ] 05:48:58 DEBUG } 05:48:58 DEBUG } 05:48:58 INFO [http_cmd]: http status code OK 05:48:58 DEBUG --- status code --- 05:48:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:48:58 DEBUG --- http response --- 05:48:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:48:58 DEBUG ----- output ----- 05:48:58 DEBUG { 05:48:58 DEBUG "status": "success", 05:48:58 DEBUG "data": { 05:48:58 DEBUG "resultType": "vector", 05:48:58 DEBUG "result": [ 05:48:58 DEBUG { 05:48:58 DEBUG "metric": { 05:48:58 DEBUG "pod": "ds-idrepo-0" 05:48:58 DEBUG }, 05:48:58 DEBUG "value": [ 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG [...] 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG "pod": "ds-cts-2" 05:48:58 DEBUG }, 05:48:58 DEBUG "value": [ 05:48:58 DEBUG 1713502138, 05:48:58 DEBUG "0" 05:48:58 DEBUG ] 05:48:58 DEBUG } 05:48:58 DEBUG ] 05:48:58 DEBUG } 05:48:58 DEBUG } 05:48:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713502138" 05:48:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502138" 05:48:58 INFO [http_cmd]: http status code OK 05:48:58 DEBUG --- status code --- 05:48:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:48:58 DEBUG --- http response --- 05:48:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:48:58 DEBUG ----- output ----- 05:48:58 DEBUG { 05:48:58 DEBUG "status": "success", 05:48:58 DEBUG "data": { 05:48:58 DEBUG "resultType": "vector", 05:48:58 DEBUG "result": [ 05:48:58 DEBUG { 05:48:58 DEBUG "metric": { 05:48:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:48:58 DEBUG }, 05:48:58 DEBUG "value": [ 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG [...] 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:48:58 DEBUG }, 05:48:58 DEBUG "value": [ 05:48:58 DEBUG 1713502138, 05:48:58 DEBUG "0" 05:48:58 DEBUG ] 05:48:58 DEBUG } 05:48:58 DEBUG ] 05:48:58 DEBUG } 05:48:58 DEBUG } 05:48:58 ERROR [http_cmd]: ERROR 05:48:58 ERROR --- status code --- 05:48:58 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:48:58 ERROR --- http response --- 05:48:58 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:48:58 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:48:58 INFO [http_cmd]: http status code OK 05:48:58 DEBUG --- status code --- 05:48:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:48:58 DEBUG --- http response --- 05:48:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:48:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502138" 05:48:58 INFO [http_cmd]: http status code OK 05:48:58 DEBUG --- status code --- 05:48:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:48:58 DEBUG --- http response --- 05:48:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:48:58 DEBUG ----- output ----- 05:48:58 DEBUG { 05:48:58 DEBUG "status": "success", 05:48:58 DEBUG "data": { 05:48:58 DEBUG "resultType": "vector", 05:48:58 DEBUG "result": [ 05:48:58 DEBUG { 05:48:58 DEBUG "metric": { 05:48:58 DEBUG "pod": "ds-cts-0" 05:48:58 DEBUG }, 05:48:58 DEBUG "value": [ 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG [...] 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG "pod": "ds-cts-2" 05:48:58 DEBUG }, 05:48:58 DEBUG "value": [ 05:48:58 DEBUG 1713502138, 05:48:58 DEBUG "201" 05:48:58 DEBUG ] 05:48:58 DEBUG } 05:48:58 DEBUG ] 05:48:58 DEBUG } 05:48:58 DEBUG } 05:48:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502138" 05:48:58 INFO [http_cmd]: http status code OK 05:48:58 DEBUG --- status code --- 05:48:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:48:58 DEBUG --- http response --- 05:48:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:48:58 DEBUG ----- output ----- 05:48:58 DEBUG { 05:48:58 DEBUG "status": "success", 05:48:58 DEBUG "data": { 05:48:58 DEBUG "resultType": "vector", 05:48:58 DEBUG "result": [ 05:48:58 DEBUG { 05:48:58 DEBUG "metric": { 05:48:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:48:58 DEBUG }, 05:48:58 DEBUG "value": [ 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG [...] 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:48:58 DEBUG }, 05:48:58 DEBUG "value": [ 05:48:58 DEBUG 1713502138, 05:48:58 DEBUG "3589531.5" 05:48:58 DEBUG ] 05:48:58 DEBUG } 05:48:58 DEBUG ] 05:48:58 DEBUG } 05:48:58 DEBUG } 05:48:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502138" 05:48:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502138" 05:48:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502138" 05:48:58 INFO [http_cmd]: http status code OK 05:48:58 DEBUG --- status code --- 05:48:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:48:58 DEBUG --- http response --- 05:48:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:48:58 DEBUG ----- output ----- 05:48:58 DEBUG { 05:48:58 DEBUG "status": "success", 05:48:58 DEBUG "data": { 05:48:58 DEBUG "resultType": "vector", 05:48:58 DEBUG "result": [ 05:48:58 DEBUG { 05:48:58 DEBUG "metric": { 05:48:58 DEBUG "pod": "ds-idrepo-0" 05:48:58 DEBUG }, 05:48:58 DEBUG "value": [ 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG [...] 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG "pod": "ds-idrepo-2" 05:48:58 DEBUG }, 05:48:58 DEBUG "value": [ 05:48:58 DEBUG 1713502138, 05:48:58 DEBUG "0" 05:48:58 DEBUG ] 05:48:58 DEBUG } 05:48:58 DEBUG ] 05:48:58 DEBUG } 05:48:58 DEBUG } 05:48:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713502138" 05:48:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502138" 05:48:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713502138" 05:48:58 INFO [http_cmd]: http status code OK 05:48:58 DEBUG --- status code --- 05:48:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:48:58 DEBUG --- http response --- 05:48:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:48:58 INFO [http_cmd]: http status code OK 05:48:58 DEBUG ----- output ----- 05:48:58 DEBUG --- status code --- 05:48:58 DEBUG { 05:48:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:48:58 DEBUG "status": "success", 05:48:58 DEBUG --- http response --- 05:48:58 DEBUG "data": { 05:48:58 DEBUG "resultType": "vector", 05:48:58 DEBUG "result": [ 05:48:58 DEBUG { 05:48:58 DEBUG "metric": { 05:48:58 DEBUG "pod": "ds-idrepo-0" 05:48:58 DEBUG }, 05:48:58 DEBUG "value": [ 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG [...] 05:48:58 DEBUG 05:48:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:48:58 DEBUG 05:48:58 DEBUG ----- output ----- 05:48:58 DEBUG 05:48:58 DEBUG { 05:48:58 DEBUG "pod": "ds-idrepo-2" 05:48:58 DEBUG "status": "success", 05:48:58 DEBUG }, 05:48:58 DEBUG "data": { 05:48:58 DEBUG "value": [ 05:48:58 DEBUG "resultType": "vector", 05:48:58 DEBUG 1713502138, 05:48:58 DEBUG "result": [ 05:48:58 DEBUG "0" 05:48:58 DEBUG { 05:48:58 DEBUG ] 05:48:58 DEBUG "metric": { 05:48:58 DEBUG } 05:48:58 DEBUG "pod": "ds-idrepo-0" 05:48:58 DEBUG ] 05:48:58 DEBUG }, 05:48:58 DEBUG } 05:48:58 DEBUG "value": [ 05:48:58 DEBUG } 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG [...] 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:48:58 DEBUG }, 05:48:58 DEBUG "value": [ 05:48:58 DEBUG 1713502138, 05:48:58 DEBUG "0.000007545936684049693" 05:48:58 DEBUG ] 05:48:58 DEBUG } 05:48:58 DEBUG ] 05:48:58 DEBUG } 05:48:58 DEBUG } 05:48:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502138" 05:48:58 INFO [http_cmd]: http status code OK 05:48:58 INFO [http_cmd]: http status code OK 05:48:58 INFO [http_cmd]: http status code OK 05:48:58 DEBUG --- status code --- 05:48:58 DEBUG --- status code --- 05:48:58 DEBUG --- status code --- 05:48:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:48:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:48:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:48:58 DEBUG --- http response --- 05:48:58 DEBUG --- http response --- 05:48:58 DEBUG --- http response --- 05:48:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:48:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:48:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:48:58 DEBUG ----- output ----- 05:48:58 DEBUG ----- output ----- 05:48:58 DEBUG ----- output ----- 05:48:58 DEBUG { 05:48:58 DEBUG { 05:48:58 DEBUG { 05:48:58 DEBUG "status": "success", 05:48:58 DEBUG "status": "success", 05:48:58 DEBUG "status": "success", 05:48:58 DEBUG "data": { 05:48:58 DEBUG "data": { 05:48:58 DEBUG "data": { 05:48:58 DEBUG "resultType": "vector", 05:48:58 DEBUG "resultType": "vector", 05:48:58 DEBUG "resultType": "vector", 05:48:58 DEBUG "result": [ 05:48:58 DEBUG "result": [ 05:48:58 DEBUG "result": [ 05:48:58 DEBUG { 05:48:58 DEBUG { 05:48:58 DEBUG { 05:48:58 DEBUG "metric": { 05:48:58 DEBUG "metric": { 05:48:58 DEBUG "metric": { 05:48:58 DEBUG "pod": "ds-idrepo-0" 05:48:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:48:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:48:58 DEBUG }, 05:48:58 DEBUG }, 05:48:58 DEBUG }, 05:48:58 DEBUG "value": [ 05:48:58 DEBUG "value": [ 05:48:58 DEBUG "value": [ 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG [...] 05:48:58 DEBUG [...] 05:48:58 DEBUG [...] 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG "pod": "ds-idrepo-2" 05:48:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:48:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:48:58 DEBUG }, 05:48:58 DEBUG }, 05:48:58 DEBUG }, 05:48:58 DEBUG "value": [ 05:48:58 DEBUG "value": [ 05:48:58 DEBUG "value": [ 05:48:58 DEBUG 1713502138, 05:48:58 DEBUG 1713502138, 05:48:58 DEBUG 1713502138, 05:48:58 DEBUG "0" 05:48:58 DEBUG "1.0332324741863368" 05:48:58 DEBUG "2500775936" 05:48:58 DEBUG ] 05:48:58 DEBUG ] 05:48:58 DEBUG ] 05:48:58 DEBUG } 05:48:58 DEBUG } 05:48:58 DEBUG } 05:48:58 DEBUG ] 05:48:58 DEBUG ] 05:48:58 DEBUG ] 05:48:58 DEBUG } 05:48:58 DEBUG } 05:48:58 DEBUG } 05:48:58 DEBUG } 05:48:58 DEBUG } 05:48:58 DEBUG } 05:48:58 INFO [http_cmd]: http status code OK 05:48:58 DEBUG --- status code --- 05:48:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:48:58 DEBUG --- http response --- 05:48:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:48:58 DEBUG ----- output ----- 05:48:58 DEBUG { 05:48:58 DEBUG "status": "success", 05:48:58 DEBUG "data": { 05:48:58 DEBUG "resultType": "vector", 05:48:58 DEBUG "result": [ 05:48:58 DEBUG { 05:48:58 DEBUG "metric": { 05:48:58 DEBUG "pod": "ds-cts-0" 05:48:58 DEBUG }, 05:48:58 DEBUG "value": [ 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG [...] 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG "pod": "ds-cts-2" 05:48:58 DEBUG }, 05:48:58 DEBUG "value": [ 05:48:58 DEBUG 1713502138, 05:48:58 DEBUG "30" 05:48:58 DEBUG ] 05:48:58 DEBUG } 05:48:58 DEBUG ] 05:48:58 DEBUG } 05:48:58 DEBUG } 05:48:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502138" 05:48:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713502138" 05:48:58 INFO [http_cmd]: http status code OK 05:48:58 DEBUG --- status code --- 05:48:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:48:58 DEBUG --- http response --- 05:48:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:48:58 DEBUG ----- output ----- 05:48:58 DEBUG { 05:48:58 DEBUG "status": "success", 05:48:58 DEBUG "data": { 05:48:58 DEBUG "resultType": "vector", 05:48:58 DEBUG "result": [ 05:48:58 DEBUG { 05:48:58 DEBUG "metric": { 05:48:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:48:58 DEBUG }, 05:48:58 DEBUG "value": [ 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG [...] 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:48:58 DEBUG }, 05:48:58 DEBUG "value": [ 05:48:58 DEBUG 1713502138, 05:48:58 DEBUG "2170.9" 05:48:58 DEBUG ] 05:48:58 DEBUG } 05:48:58 DEBUG ] 05:48:58 DEBUG } 05:48:58 DEBUG } 05:48:58 INFO [http_cmd]: http status code OK 05:48:58 DEBUG --- status code --- 05:48:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:48:58 DEBUG --- http response --- 05:48:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:48:58 DEBUG ----- output ----- 05:48:58 DEBUG { 05:48:58 DEBUG "status": "success", 05:48:58 DEBUG "data": { 05:48:58 DEBUG "resultType": "vector", 05:48:58 DEBUG "result": [ 05:48:58 DEBUG { 05:48:58 DEBUG "metric": { 05:48:58 DEBUG "instance": "10.142.0.107:9100" 05:48:58 DEBUG }, 05:48:58 DEBUG "value": [ 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG [...] 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG "instance": "10.142.0.43:9100" 05:48:58 DEBUG }, 05:48:58 DEBUG "value": [ 05:48:58 DEBUG 1713502138, 05:48:58 DEBUG "0.5293333333333309" 05:48:58 DEBUG ] 05:48:58 DEBUG } 05:48:58 DEBUG ] 05:48:58 DEBUG } 05:48:58 DEBUG } 05:48:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502138" 05:48:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502138" 05:48:58 INFO [http_cmd]: http status code OK 05:48:58 INFO [http_cmd]: http status code OK 05:48:58 DEBUG --- status code --- 05:48:58 DEBUG --- status code --- 05:48:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:48:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:48:58 DEBUG --- http response --- 05:48:58 DEBUG --- http response --- 05:48:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:48:58 DEBUG ----- output ----- 05:48:58 DEBUG { 05:48:58 DEBUG "status": "success", 05:48:58 DEBUG "data": { 05:48:58 DEBUG "resultType": "vector", 05:48:58 DEBUG "result": [ 05:48:58 DEBUG { 05:48:58 DEBUG "metric": { 05:48:58 DEBUG "pod": "ds-cts-0" 05:48:58 DEBUG }, 05:48:58 DEBUG "value": [ 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG [...] 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG "pod": "ds-cts-2" 05:48:58 DEBUG }, 05:48:58 DEBUG "value": [ 05:48:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:48:58 DEBUG 1713502138, 05:48:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502138" 05:48:58 DEBUG ----- output ----- 05:48:58 DEBUG "0.0001333333333333333" 05:48:58 DEBUG { 05:48:58 DEBUG ] 05:48:58 DEBUG "status": "success", 05:48:58 DEBUG } 05:48:58 DEBUG "data": { 05:48:58 DEBUG ] 05:48:58 DEBUG "resultType": "vector", 05:48:58 DEBUG } 05:48:58 DEBUG "result": [ 05:48:58 DEBUG } 05:48:58 DEBUG { 05:48:58 DEBUG "metric": { 05:48:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:48:58 DEBUG }, 05:48:58 DEBUG "value": [ 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG [...] 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:48:58 DEBUG }, 05:48:58 DEBUG "value": [ 05:48:58 DEBUG 1713502138, 05:48:58 DEBUG "0" 05:48:58 DEBUG ] 05:48:58 DEBUG } 05:48:58 DEBUG ] 05:48:58 DEBUG } 05:48:58 DEBUG } 05:48:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502138" 05:48:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713502138" 05:48:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502138" 05:48:58 INFO [http_cmd]: http status code OK 05:48:58 DEBUG --- status code --- 05:48:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:48:58 DEBUG --- http response --- 05:48:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 05:48:58 DEBUG ----- output ----- 05:48:58 DEBUG { 05:48:58 DEBUG "status": "success", 05:48:58 DEBUG "data": { 05:48:58 DEBUG "resultType": "vector", 05:48:58 DEBUG "result": [ 05:48:58 DEBUG { 05:48:58 DEBUG "metric": { 05:48:58 DEBUG "pod": "ds-idrepo-0" 05:48:58 DEBUG }, 05:48:58 DEBUG "value": [ 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG [...] 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:48:58 DEBUG }, 05:48:58 DEBUG "value": [ 05:48:58 DEBUG 1713502138, 05:48:58 DEBUG "0" 05:48:58 DEBUG ] 05:48:58 DEBUG } 05:48:58 DEBUG ] 05:48:58 DEBUG } 05:48:58 DEBUG } 05:48:58 INFO [http_cmd]: http status code OK 05:48:58 DEBUG --- status code --- 05:48:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:48:58 DEBUG --- http response --- 05:48:58 INFO [http_cmd]: http status code OK 05:48:58 INFO [http_cmd]: http status code OK 05:48:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:48:58 DEBUG --- status code --- 05:48:58 DEBUG --- status code --- 05:48:58 DEBUG ----- output ----- 05:48:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:48:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:48:58 DEBUG { 05:48:58 DEBUG --- http response --- 05:48:58 DEBUG --- http response --- 05:48:58 DEBUG "status": "success", 05:48:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 05:48:58 DEBUG "data": { 05:48:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:48:58 DEBUG ----- output ----- 05:48:58 DEBUG "resultType": "vector", 05:48:58 DEBUG ----- output ----- 05:48:58 DEBUG { 05:48:58 DEBUG "result": [ 05:48:58 DEBUG { 05:48:58 DEBUG "status": "success", 05:48:58 DEBUG { 05:48:58 DEBUG "status": "success", 05:48:58 DEBUG "data": { 05:48:58 DEBUG "metric": { 05:48:58 DEBUG "data": { 05:48:58 DEBUG "resultType": "vector", 05:48:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:48:58 DEBUG "resultType": "vector", 05:48:58 DEBUG "result": [ 05:48:58 DEBUG }, 05:48:58 DEBUG "result": [ 05:48:58 DEBUG { 05:48:58 DEBUG "value": [ 05:48:58 DEBUG { 05:48:58 DEBUG "metric": { 05:48:58 DEBUG 05:48:58 DEBUG "metric": { 05:48:58 DEBUG "pod": "ds-idrepo-0" 05:48:58 DEBUG 05:48:58 DEBUG "instance": "10.142.0.107:9100" 05:48:58 DEBUG }, 05:48:58 DEBUG 05:48:58 DEBUG }, 05:48:58 DEBUG "value": [ 05:48:58 DEBUG [...] 05:48:58 DEBUG "value": [ 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG [...] 05:48:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:48:58 DEBUG [...] 05:48:58 DEBUG 05:48:58 DEBUG }, 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG "value": [ 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 1713502138, 05:48:58 DEBUG 05:48:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:48:58 DEBUG "0" 05:48:58 DEBUG "instance": "10.142.0.43:9100" 05:48:58 DEBUG }, 05:48:58 DEBUG ] 05:48:58 DEBUG }, 05:48:58 DEBUG "value": [ 05:48:58 DEBUG } 05:48:58 DEBUG "value": [ 05:48:58 DEBUG 1713502138, 05:48:58 DEBUG ] 05:48:58 DEBUG 1713502138, 05:48:58 DEBUG "0" 05:48:58 DEBUG } 05:48:58 DEBUG "0.47420000000000184" 05:48:58 DEBUG ] 05:48:58 DEBUG } 05:48:58 DEBUG ] 05:48:58 DEBUG } 05:48:58 DEBUG } 05:48:58 DEBUG ] 05:48:58 DEBUG ] 05:48:58 DEBUG } 05:48:58 DEBUG } 05:48:58 DEBUG } 05:48:58 DEBUG } 05:48:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713502138" 05:48:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713502138" 05:48:58 INFO [http_cmd]: http status code OK 05:48:58 DEBUG --- status code --- 05:48:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:48:58 DEBUG --- http response --- 05:48:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:48:58 DEBUG ----- output ----- 05:48:58 DEBUG { 05:48:58 DEBUG "status": "success", 05:48:58 DEBUG "data": { 05:48:58 DEBUG "resultType": "vector", 05:48:58 DEBUG "result": [ 05:48:58 DEBUG { 05:48:58 DEBUG "metric": { 05:48:58 DEBUG "instance": "10.142.0.107:9100" 05:48:58 DEBUG }, 05:48:58 DEBUG "value": [ 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG [...] 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG "instance": "10.142.0.43:9100" 05:48:58 DEBUG }, 05:48:58 DEBUG "value": [ 05:48:58 DEBUG 1713502138, 05:48:58 DEBUG "14270425.766666666" 05:48:58 DEBUG ] 05:48:58 DEBUG } 05:48:58 DEBUG ] 05:48:58 DEBUG } 05:48:58 DEBUG } 05:48:58 INFO [http_cmd]: http status code OK 05:48:58 DEBUG --- status code --- 05:48:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:48:58 DEBUG --- http response --- 05:48:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:48:58 DEBUG ----- output ----- 05:48:58 DEBUG { 05:48:58 DEBUG "status": "success", 05:48:58 DEBUG "data": { 05:48:58 DEBUG "resultType": "vector", 05:48:58 DEBUG "result": [ 05:48:58 DEBUG { 05:48:58 DEBUG "metric": { 05:48:58 DEBUG "instance": "10.142.0.107:9100" 05:48:58 DEBUG }, 05:48:58 DEBUG "value": [ 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG [...] 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG 05:48:58 DEBUG "instance": "10.142.0.43:9100" 05:48:58 DEBUG }, 05:48:58 DEBUG "value": [ 05:48:58 DEBUG 1713502138, 05:48:58 DEBUG "14261432.066666666" 05:48:58 DEBUG ] 05:48:58 DEBUG } 05:48:58 DEBUG ] 05:48:58 DEBUG } 05:48:58 DEBUG } 05:49:00 INFO 05:49:00 INFO [loop_until]: kubectl --namespace=xlou top pods 05:49:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:49:00 INFO [loop_until]: OK (rc = 0) 05:49:00 DEBUG --- stdout --- 05:49:00 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 2145m 9115Mi am-6fd554f8f7-hvvkw 2133m 8882Mi am-6fd554f8f7-pmj4b 2220m 8763Mi ds-cts-0 7841m 13552Mi ds-cts-1 8083m 12845Mi ds-cts-2 8746m 12601Mi ds-idrepo-0 623m 3297Mi ds-idrepo-1 624m 3213Mi ds-idrepo-2 596m 3092Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 3m 1216Mi lodemon-6b89d774d5-rf7sk 1m 64Mi login-ui-5fdf49fc66-dlhjd 1m 5Mi overseer-0-5c67bf94db-77mh9 1003m 2450Mi 05:49:00 DEBUG --- stderr --- 05:49:00 DEBUG 05:49:01 INFO 05:49:01 INFO [loop_until]: kubectl --namespace=xlou top node 05:49:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:49:01 INFO [loop_until]: OK (rc = 0) 05:49:01 DEBUG --- stdout --- 05:49:01 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 73m 0% 6249Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 685m 2% 6741Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 2108m 7% 11640Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 3255Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 8574m 28% 16342Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 155m 0% 5165Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 2239m 7% 11935Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 77m 0% 2701Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 2322m 7% 10996Mi 9% gke-xlou-cdm-ds-0f601751-j488 663m 2% 6186Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 7856m 26% 16885Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 706m 2% 6896Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 8911m 29% 15060Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 1077m 6% 6189Mi 10% 05:49:01 DEBUG --- stderr --- 05:49:01 DEBUG 05:49:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502198" 05:49:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502198" 05:49:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713502198" 05:49:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713502198" 05:49:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502198" 05:49:58 INFO [http_cmd]: http status code OK 05:49:58 DEBUG --- status code --- 05:49:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:49:58 DEBUG --- http response --- 05:49:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:49:58 DEBUG ----- output ----- 05:49:58 DEBUG { 05:49:58 DEBUG "status": "success", 05:49:58 DEBUG "data": { 05:49:58 DEBUG "resultType": "vector", 05:49:58 DEBUG "result": [ 05:49:58 DEBUG { 05:49:58 DEBUG "metric": { 05:49:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:49:58 DEBUG }, 05:49:58 DEBUG "value": [ 05:49:58 DEBUG 05:49:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713502198" 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG [...] 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:49:58 DEBUG }, 05:49:58 DEBUG "value": [ 05:49:58 DEBUG 1713502198, 05:49:58 DEBUG "0" 05:49:58 DEBUG ] 05:49:58 DEBUG } 05:49:58 DEBUG ] 05:49:58 DEBUG } 05:49:58 DEBUG } 05:49:58 INFO [http_cmd]: http status code OK 05:49:58 INFO [http_cmd]: http status code OK 05:49:58 DEBUG --- status code --- 05:49:58 DEBUG --- status code --- 05:49:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:49:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:49:58 DEBUG --- http response --- 05:49:58 DEBUG --- http response --- 05:49:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:49:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:49:58 DEBUG ----- output ----- 05:49:58 DEBUG ----- output ----- 05:49:58 DEBUG { 05:49:58 DEBUG { 05:49:58 DEBUG "status": "success", 05:49:58 DEBUG "status": "success", 05:49:58 DEBUG "data": { 05:49:58 DEBUG "data": { 05:49:58 DEBUG "resultType": "vector", 05:49:58 DEBUG "resultType": "vector", 05:49:58 DEBUG "result": [ 05:49:58 DEBUG "result": [ 05:49:58 DEBUG { 05:49:58 DEBUG { 05:49:58 DEBUG "metric": { 05:49:58 DEBUG "metric": { 05:49:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:49:58 DEBUG "instance": "10.142.0.107:9100" 05:49:58 DEBUG }, 05:49:58 DEBUG }, 05:49:58 DEBUG "value": [ 05:49:58 DEBUG "value": [ 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG [...] 05:49:58 DEBUG [...] 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:49:58 INFO [http_cmd]: http status code OK 05:49:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713502198" 05:49:58 DEBUG "instance": "10.142.0.43:9100" 05:49:58 DEBUG }, 05:49:58 DEBUG --- status code --- 05:49:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:49:58 DEBUG }, 05:49:58 DEBUG "value": [ 05:49:58 INFO [http_cmd]: http status code OK 05:49:58 DEBUG --- http response --- 05:49:58 DEBUG "value": [ 05:49:58 DEBUG 1713502198, 05:49:58 DEBUG --- status code --- 05:49:58 DEBUG 1713502198, 05:49:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:49:58 DEBUG "0" 05:49:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:49:58 DEBUG "574076257280" 05:49:58 DEBUG ----- output ----- 05:49:58 DEBUG ] 05:49:58 DEBUG --- http response --- 05:49:58 DEBUG ] 05:49:58 DEBUG { 05:49:58 DEBUG } 05:49:58 DEBUG } 05:49:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 05:49:58 DEBUG "status": "success", 05:49:58 DEBUG ] 05:49:58 DEBUG ] 05:49:58 DEBUG ----- output ----- 05:49:58 DEBUG "data": { 05:49:58 DEBUG } 05:49:58 DEBUG } 05:49:58 DEBUG { 05:49:58 DEBUG "resultType": "vector", 05:49:58 DEBUG } 05:49:58 DEBUG } 05:49:58 DEBUG "status": "success", 05:49:58 DEBUG "result": [ 05:49:58 DEBUG "data": { 05:49:58 DEBUG { 05:49:58 DEBUG "resultType": "vector", 05:49:58 DEBUG "metric": { 05:49:58 DEBUG "result": [ 05:49:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:49:58 DEBUG { 05:49:58 DEBUG }, 05:49:58 DEBUG "metric": { 05:49:58 DEBUG "value": [ 05:49:58 DEBUG "pod": "ds-idrepo-1" 05:49:58 DEBUG 05:49:58 DEBUG }, 05:49:58 DEBUG 05:49:58 DEBUG "value": [ 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG [...] 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG [...] 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:49:58 DEBUG 05:49:58 DEBUG }, 05:49:58 DEBUG 05:49:58 DEBUG "value": [ 05:49:58 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 05:49:58 DEBUG 1713502198, 05:49:58 DEBUG }, 05:49:58 DEBUG "55.176" 05:49:58 DEBUG "value": [ 05:49:58 DEBUG ] 05:49:58 DEBUG 1713502198, 05:49:58 DEBUG } 05:49:58 DEBUG "0" 05:49:58 DEBUG ] 05:49:58 DEBUG ] 05:49:58 DEBUG } 05:49:58 DEBUG } 05:49:58 DEBUG } 05:49:58 DEBUG ] 05:49:58 DEBUG } 05:49:58 DEBUG } 05:49:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502198" 05:49:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502198" 05:49:58 INFO [http_cmd]: http status code OK 05:49:58 DEBUG --- status code --- 05:49:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:49:58 DEBUG --- http response --- 05:49:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:49:58 DEBUG ----- output ----- 05:49:58 DEBUG { 05:49:58 DEBUG "status": "success", 05:49:58 DEBUG "data": { 05:49:58 DEBUG "resultType": "vector", 05:49:58 DEBUG "result": [ 05:49:58 DEBUG { 05:49:58 DEBUG "metric": { 05:49:58 DEBUG "pod": "ds-idrepo-0" 05:49:58 DEBUG }, 05:49:58 DEBUG "value": [ 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG [...] 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG "pod": "ds-cts-2" 05:49:58 DEBUG }, 05:49:58 DEBUG "value": [ 05:49:58 DEBUG 1713502198, 05:49:58 DEBUG "0" 05:49:58 DEBUG ] 05:49:58 DEBUG } 05:49:58 DEBUG ] 05:49:58 DEBUG } 05:49:58 DEBUG } 05:49:58 ERROR [http_cmd]: ERROR 05:49:58 ERROR --- status code --- 05:49:58 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:49:58 ERROR --- http response --- 05:49:58 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:49:58 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:49:58 INFO [http_cmd]: http status code OK 05:49:58 INFO [http_cmd]: http status code OK 05:49:58 DEBUG --- status code --- 05:49:58 DEBUG --- status code --- 05:49:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:49:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:49:58 DEBUG --- http response --- 05:49:58 DEBUG --- http response --- 05:49:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:49:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:49:58 DEBUG ----- output ----- 05:49:58 DEBUG { 05:49:58 DEBUG "status": "success", 05:49:58 DEBUG "data": { 05:49:58 DEBUG "resultType": "vector", 05:49:58 DEBUG "result": [ 05:49:58 DEBUG { 05:49:58 DEBUG "metric": { 05:49:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:49:58 DEBUG }, 05:49:58 DEBUG "value": [ 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG [...] 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:49:58 DEBUG }, 05:49:58 DEBUG "value": [ 05:49:58 DEBUG 1713502198, 05:49:58 DEBUG "3459374" 05:49:58 DEBUG ] 05:49:58 DEBUG } 05:49:58 DEBUG ] 05:49:58 DEBUG } 05:49:58 DEBUG } 05:49:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502198" 05:49:58 INFO [http_cmd]: http status code OK 05:49:58 DEBUG --- status code --- 05:49:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:49:58 DEBUG --- http response --- 05:49:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:49:58 DEBUG ----- output ----- 05:49:58 DEBUG { 05:49:58 DEBUG "status": "success", 05:49:58 DEBUG "data": { 05:49:58 DEBUG "resultType": "vector", 05:49:58 DEBUG "result": [ 05:49:58 DEBUG { 05:49:58 DEBUG "metric": { 05:49:58 DEBUG "pod": "ds-cts-0" 05:49:58 DEBUG }, 05:49:58 DEBUG "value": [ 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG [...] 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG "pod": "ds-cts-2" 05:49:58 DEBUG }, 05:49:58 DEBUG "value": [ 05:49:58 DEBUG 1713502198, 05:49:58 DEBUG "264.1666666666667" 05:49:58 DEBUG ] 05:49:58 DEBUG } 05:49:58 DEBUG ] 05:49:58 DEBUG } 05:49:58 DEBUG } 05:49:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502198" 05:49:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502198" 05:49:58 INFO [http_cmd]: http status code OK 05:49:58 DEBUG --- status code --- 05:49:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:49:58 DEBUG --- http response --- 05:49:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:49:58 DEBUG ----- output ----- 05:49:58 DEBUG { 05:49:58 DEBUG "status": "success", 05:49:58 DEBUG "data": { 05:49:58 DEBUG "resultType": "vector", 05:49:58 DEBUG "result": [ 05:49:58 DEBUG { 05:49:58 DEBUG "metric": { 05:49:58 DEBUG "pod": "ds-idrepo-0" 05:49:58 DEBUG }, 05:49:58 DEBUG "value": [ 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG [...] 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG "pod": "ds-idrepo-2" 05:49:58 DEBUG }, 05:49:58 DEBUG "value": [ 05:49:58 DEBUG 1713502198, 05:49:58 DEBUG "0" 05:49:58 DEBUG ] 05:49:58 DEBUG } 05:49:58 DEBUG ] 05:49:58 DEBUG } 05:49:58 DEBUG } 05:49:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502198" 05:49:58 INFO [http_cmd]: http status code OK 05:49:58 DEBUG --- status code --- 05:49:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:49:58 DEBUG --- http response --- 05:49:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:49:58 DEBUG ----- output ----- 05:49:58 DEBUG { 05:49:58 DEBUG "status": "success", 05:49:58 DEBUG "data": { 05:49:58 DEBUG "resultType": "vector", 05:49:58 DEBUG "result": [ 05:49:58 DEBUG { 05:49:58 DEBUG "metric": { 05:49:58 DEBUG "pod": "ds-idrepo-0" 05:49:58 DEBUG }, 05:49:58 DEBUG "value": [ 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG [...] 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG "pod": "ds-idrepo-2" 05:49:58 DEBUG }, 05:49:58 DEBUG "value": [ 05:49:58 DEBUG 1713502198, 05:49:58 DEBUG "0" 05:49:58 DEBUG ] 05:49:58 DEBUG } 05:49:58 DEBUG ] 05:49:58 DEBUG } 05:49:58 DEBUG } 05:49:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713502198" 05:49:58 INFO [http_cmd]: http status code OK 05:49:58 DEBUG --- status code --- 05:49:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:49:58 DEBUG --- http response --- 05:49:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 05:49:58 DEBUG ----- output ----- 05:49:58 DEBUG { 05:49:58 DEBUG "status": "success", 05:49:58 DEBUG "data": { 05:49:58 DEBUG "resultType": "vector", 05:49:58 DEBUG "result": [ 05:49:58 DEBUG { 05:49:58 DEBUG "metric": { 05:49:58 DEBUG "pod": "ds-idrepo-1" 05:49:58 DEBUG }, 05:49:58 DEBUG "value": [ 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG [...] 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 05:49:58 DEBUG }, 05:49:58 DEBUG "value": [ 05:49:58 DEBUG 1713502198, 05:49:58 DEBUG "0.000021175762678436085" 05:49:58 DEBUG ] 05:49:58 DEBUG } 05:49:58 DEBUG ] 05:49:58 DEBUG } 05:49:58 DEBUG } 05:49:58 INFO [http_cmd]: http status code OK 05:49:58 DEBUG --- status code --- 05:49:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:49:58 DEBUG --- http response --- 05:49:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:49:58 DEBUG ----- output ----- 05:49:58 DEBUG { 05:49:58 DEBUG "status": "success", 05:49:58 DEBUG "data": { 05:49:58 DEBUG "resultType": "vector", 05:49:58 DEBUG "result": [ 05:49:58 DEBUG { 05:49:58 DEBUG "metric": { 05:49:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:49:58 DEBUG }, 05:49:58 DEBUG "value": [ 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG [...] 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:49:58 DEBUG }, 05:49:58 DEBUG "value": [ 05:49:58 DEBUG 1713502198, 05:49:58 DEBUG "1.029345726051057" 05:49:58 DEBUG ] 05:49:58 DEBUG } 05:49:58 DEBUG ] 05:49:58 DEBUG } 05:49:58 DEBUG } 05:49:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713502198" 05:49:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502198" 05:49:58 INFO [http_cmd]: http status code OK 05:49:58 DEBUG --- status code --- 05:49:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:49:58 DEBUG --- http response --- 05:49:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:49:58 DEBUG ----- output ----- 05:49:58 DEBUG { 05:49:58 DEBUG "status": "success", 05:49:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502198" 05:49:58 DEBUG "data": { 05:49:58 DEBUG "resultType": "vector", 05:49:58 DEBUG "result": [ 05:49:58 DEBUG { 05:49:58 DEBUG "metric": { 05:49:58 DEBUG "instance": "10.142.0.107:9100" 05:49:58 DEBUG }, 05:49:58 DEBUG "value": [ 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG [...] 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG "instance": "10.142.0.43:9100" 05:49:58 DEBUG }, 05:49:58 DEBUG "value": [ 05:49:58 DEBUG 1713502198, 05:49:58 DEBUG "0.6910000000000011" 05:49:58 DEBUG ] 05:49:58 DEBUG } 05:49:58 DEBUG ] 05:49:58 DEBUG } 05:49:58 DEBUG } 05:49:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502198" 05:49:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713502198" 05:49:58 INFO [http_cmd]: http status code OK 05:49:58 DEBUG --- status code --- 05:49:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:49:58 DEBUG --- http response --- 05:49:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:49:58 DEBUG ----- output ----- 05:49:58 DEBUG { 05:49:58 DEBUG "status": "success", 05:49:58 DEBUG "data": { 05:49:58 DEBUG "resultType": "vector", 05:49:58 DEBUG "result": [ 05:49:58 DEBUG { 05:49:58 DEBUG "metric": { 05:49:58 DEBUG "pod": "ds-idrepo-0" 05:49:58 DEBUG }, 05:49:58 DEBUG "value": [ 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG [...] 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG "pod": "ds-idrepo-2" 05:49:58 DEBUG }, 05:49:58 DEBUG "value": [ 05:49:58 DEBUG 1713502198, 05:49:58 DEBUG "0" 05:49:58 DEBUG ] 05:49:58 DEBUG } 05:49:58 DEBUG ] 05:49:58 DEBUG } 05:49:58 DEBUG } 05:49:58 INFO [http_cmd]: http status code OK 05:49:58 DEBUG --- status code --- 05:49:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:49:58 DEBUG --- http response --- 05:49:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:49:58 DEBUG ----- output ----- 05:49:58 DEBUG { 05:49:58 DEBUG "status": "success", 05:49:58 DEBUG "data": { 05:49:58 DEBUG "resultType": "vector", 05:49:58 DEBUG "result": [ 05:49:58 DEBUG { 05:49:58 DEBUG "metric": { 05:49:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:49:58 DEBUG }, 05:49:58 DEBUG "value": [ 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG [...] 05:49:58 INFO [http_cmd]: http status code OK 05:49:58 DEBUG 05:49:58 DEBUG --- status code --- 05:49:58 DEBUG 05:49:58 INFO [http_cmd]: http status code OK 05:49:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:49:58 DEBUG 05:49:58 DEBUG --- status code --- 05:49:58 DEBUG --- http response --- 05:49:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:49:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:49:58 DEBUG }, 05:49:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:49:58 DEBUG --- http response --- 05:49:58 DEBUG "value": [ 05:49:58 DEBUG ----- output ----- 05:49:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:49:58 DEBUG 1713502198, 05:49:58 DEBUG { 05:49:58 DEBUG ----- output ----- 05:49:58 DEBUG "0" 05:49:58 DEBUG "status": "success", 05:49:58 DEBUG { 05:49:58 DEBUG ] 05:49:58 DEBUG "data": { 05:49:58 DEBUG "status": "success", 05:49:58 DEBUG } 05:49:58 DEBUG "resultType": "vector", 05:49:58 DEBUG "data": { 05:49:58 DEBUG ] 05:49:58 DEBUG "result": [ 05:49:58 DEBUG "resultType": "vector", 05:49:58 DEBUG } 05:49:58 DEBUG { 05:49:58 DEBUG "result": [ 05:49:58 DEBUG } 05:49:58 DEBUG "metric": { 05:49:58 DEBUG { 05:49:58 DEBUG "pod": "ds-cts-0" 05:49:58 DEBUG "metric": { 05:49:58 DEBUG }, 05:49:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:49:58 DEBUG "value": [ 05:49:58 DEBUG }, 05:49:58 DEBUG 05:49:58 DEBUG "value": [ 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG [...] 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG [...] 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG "pod": "ds-cts-2" 05:49:58 DEBUG 05:49:58 DEBUG }, 05:49:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:49:58 DEBUG "value": [ 05:49:58 DEBUG }, 05:49:58 DEBUG 1713502198, 05:49:58 DEBUG "value": [ 05:49:58 DEBUG "30" 05:49:58 DEBUG 1713502198, 05:49:58 DEBUG ] 05:49:58 DEBUG "2885632000" 05:49:58 DEBUG } 05:49:58 DEBUG ] 05:49:58 DEBUG ] 05:49:58 DEBUG } 05:49:58 DEBUG } 05:49:58 DEBUG ] 05:49:58 DEBUG } 05:49:58 DEBUG } 05:49:58 DEBUG } 05:49:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502198" 05:49:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502198" 05:49:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502198" 05:49:58 INFO [http_cmd]: http status code OK 05:49:58 DEBUG --- status code --- 05:49:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:49:58 DEBUG --- http response --- 05:49:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:49:58 DEBUG ----- output ----- 05:49:58 DEBUG { 05:49:58 DEBUG "status": "success", 05:49:58 DEBUG "data": { 05:49:58 DEBUG "resultType": "vector", 05:49:58 DEBUG "result": [ 05:49:58 DEBUG { 05:49:58 DEBUG "metric": { 05:49:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:49:58 DEBUG }, 05:49:58 DEBUG "value": [ 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG [...] 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:49:58 DEBUG }, 05:49:58 DEBUG "value": [ 05:49:58 DEBUG 1713502198, 05:49:58 DEBUG "2175.4" 05:49:58 DEBUG ] 05:49:58 DEBUG } 05:49:58 DEBUG ] 05:49:58 DEBUG } 05:49:58 DEBUG } 05:49:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502198" 05:49:58 INFO [http_cmd]: http status code OK 05:49:58 INFO [http_cmd]: http status code OK 05:49:58 DEBUG --- status code --- 05:49:58 DEBUG --- status code --- 05:49:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:49:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:49:58 DEBUG --- http response --- 05:49:58 DEBUG --- http response --- 05:49:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:49:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:49:58 DEBUG ----- output ----- 05:49:58 DEBUG ----- output ----- 05:49:58 DEBUG { 05:49:58 DEBUG { 05:49:58 DEBUG "status": "success", 05:49:58 DEBUG "status": "success", 05:49:58 DEBUG "data": { 05:49:58 DEBUG "data": { 05:49:58 DEBUG "resultType": "vector", 05:49:58 DEBUG "resultType": "vector", 05:49:58 DEBUG "result": [ 05:49:58 DEBUG "result": [ 05:49:58 DEBUG { 05:49:58 DEBUG { 05:49:58 DEBUG "metric": { 05:49:58 DEBUG "metric": { 05:49:58 DEBUG "pod": "ds-idrepo-1" 05:49:58 DEBUG "pod": "ds-idrepo-1" 05:49:58 DEBUG }, 05:49:58 DEBUG }, 05:49:58 DEBUG "value": [ 05:49:58 DEBUG "value": [ 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG [...] 05:49:58 DEBUG [...] 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 05:49:58 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 05:49:58 DEBUG }, 05:49:58 DEBUG }, 05:49:58 DEBUG "value": [ 05:49:58 DEBUG "value": [ 05:49:58 DEBUG 1713502198, 05:49:58 DEBUG 1713502198, 05:49:58 DEBUG "0" 05:49:58 DEBUG "0" 05:49:58 DEBUG ] 05:49:58 DEBUG ] 05:49:58 DEBUG } 05:49:58 DEBUG } 05:49:58 DEBUG ] 05:49:58 DEBUG ] 05:49:58 DEBUG } 05:49:58 DEBUG } 05:49:58 DEBUG } 05:49:58 DEBUG } 05:49:58 INFO [http_cmd]: http status code OK 05:49:58 DEBUG --- status code --- 05:49:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:49:58 DEBUG --- http response --- 05:49:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:49:58 DEBUG ----- output ----- 05:49:58 DEBUG { 05:49:58 DEBUG "status": "success", 05:49:58 DEBUG "data": { 05:49:58 DEBUG "resultType": "vector", 05:49:58 DEBUG "result": [ 05:49:58 DEBUG { 05:49:58 DEBUG "metric": { 05:49:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:49:58 DEBUG }, 05:49:58 DEBUG "value": [ 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG [...] 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:49:58 DEBUG }, 05:49:58 DEBUG "value": [ 05:49:58 DEBUG 1713502198, 05:49:58 DEBUG "0" 05:49:58 DEBUG ] 05:49:58 DEBUG } 05:49:58 DEBUG ] 05:49:58 DEBUG } 05:49:58 DEBUG } 05:49:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502198" 05:49:58 INFO [http_cmd]: http status code OK 05:49:58 DEBUG --- status code --- 05:49:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:49:58 DEBUG --- http response --- 05:49:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:49:58 DEBUG ----- output ----- 05:49:58 DEBUG { 05:49:58 DEBUG "status": "success", 05:49:58 DEBUG "data": { 05:49:58 DEBUG "resultType": "vector", 05:49:58 DEBUG "result": [ 05:49:58 DEBUG { 05:49:58 DEBUG "metric": { 05:49:58 DEBUG "pod": "ds-cts-0" 05:49:58 DEBUG }, 05:49:58 DEBUG "value": [ 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG [...] 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG "pod": "ds-cts-2" 05:49:58 DEBUG }, 05:49:58 DEBUG "value": [ 05:49:58 DEBUG 1713502198, 05:49:58 DEBUG "0.00023333333333333344" 05:49:58 DEBUG ] 05:49:58 DEBUG } 05:49:58 DEBUG ] 05:49:58 DEBUG } 05:49:58 DEBUG } 05:49:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713502198" 05:49:58 INFO [http_cmd]: http status code OK 05:49:58 DEBUG --- status code --- 05:49:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:49:58 DEBUG --- http response --- 05:49:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:49:58 DEBUG ----- output ----- 05:49:58 DEBUG { 05:49:58 DEBUG "status": "success", 05:49:58 DEBUG "data": { 05:49:58 DEBUG "resultType": "vector", 05:49:58 DEBUG "result": [ 05:49:58 DEBUG { 05:49:58 DEBUG "metric": { 05:49:58 DEBUG "instance": "10.142.0.107:9100" 05:49:58 DEBUG }, 05:49:58 DEBUG "value": [ 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG [...] 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG "instance": "10.142.0.43:9100" 05:49:58 DEBUG }, 05:49:58 DEBUG "value": [ 05:49:58 DEBUG 1713502198, 05:49:58 DEBUG "0.619666666666681" 05:49:58 DEBUG ] 05:49:58 DEBUG } 05:49:58 DEBUG ] 05:49:58 DEBUG } 05:49:58 DEBUG } 05:49:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713502198" 05:49:58 INFO [http_cmd]: http status code OK 05:49:58 DEBUG --- status code --- 05:49:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:49:58 DEBUG --- http response --- 05:49:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713502198" 05:49:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:49:58 DEBUG ----- output ----- 05:49:58 DEBUG { 05:49:58 DEBUG "status": "success", 05:49:58 DEBUG "data": { 05:49:58 DEBUG "resultType": "vector", 05:49:58 DEBUG "result": [ 05:49:58 DEBUG { 05:49:58 DEBUG "metric": { 05:49:58 DEBUG "instance": "10.142.0.107:9100" 05:49:58 DEBUG }, 05:49:58 DEBUG "value": [ 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG [...] 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG "instance": "10.142.0.43:9100" 05:49:58 DEBUG }, 05:49:58 DEBUG "value": [ 05:49:58 DEBUG 1713502198, 05:49:58 DEBUG "14265326.833333332" 05:49:58 DEBUG ] 05:49:58 DEBUG } 05:49:58 DEBUG ] 05:49:58 DEBUG } 05:49:58 DEBUG } 05:49:58 INFO [http_cmd]: http status code OK 05:49:58 DEBUG --- status code --- 05:49:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:49:58 DEBUG --- http response --- 05:49:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:49:58 DEBUG ----- output ----- 05:49:58 DEBUG { 05:49:58 DEBUG "status": "success", 05:49:58 DEBUG "data": { 05:49:58 DEBUG "resultType": "vector", 05:49:58 DEBUG "result": [ 05:49:58 DEBUG { 05:49:58 DEBUG "metric": { 05:49:58 DEBUG "instance": "10.142.0.107:9100" 05:49:58 DEBUG }, 05:49:58 DEBUG "value": [ 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG [...] 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG 05:49:58 DEBUG "instance": "10.142.0.43:9100" 05:49:58 DEBUG }, 05:49:58 DEBUG "value": [ 05:49:58 DEBUG 1713502198, 05:49:58 DEBUG "14256291.7" 05:49:58 DEBUG ] 05:49:58 DEBUG } 05:49:58 DEBUG ] 05:49:58 DEBUG } 05:49:58 DEBUG } 05:50:00 INFO 05:50:00 INFO [loop_until]: kubectl --namespace=xlou top pods 05:50:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:50:00 INFO [loop_until]: OK (rc = 0) 05:50:00 DEBUG --- stdout --- 05:50:00 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 2133m 9114Mi am-6fd554f8f7-hvvkw 2093m 8885Mi am-6fd554f8f7-pmj4b 2208m 8765Mi ds-cts-0 9254m 12890Mi ds-cts-1 7794m 13430Mi ds-cts-2 8529m 13317Mi ds-idrepo-0 633m 3296Mi ds-idrepo-1 616m 3212Mi ds-idrepo-2 614m 3092Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 3m 1216Mi lodemon-6b89d774d5-rf7sk 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 1008m 2781Mi 05:50:00 DEBUG --- stderr --- 05:50:00 DEBUG 05:50:01 INFO 05:50:01 INFO [loop_until]: kubectl --namespace=xlou top node 05:50:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:50:01 INFO [loop_until]: OK (rc = 0) 05:50:01 DEBUG --- stdout --- 05:50:01 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 70m 0% 6250Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 685m 2% 6751Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 2083m 6% 11647Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 3273Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 9495m 31% 15526Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 137m 0% 5182Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 2213m 7% 11934Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 75m 0% 2700Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 2312m 7% 11004Mi 9% gke-xlou-cdm-ds-0f601751-j488 657m 2% 6189Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 7858m 26% 17555Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 705m 2% 6893Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 8308m 27% 15361Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 1073m 6% 6514Mi 11% 05:50:01 DEBUG --- stderr --- 05:50:01 DEBUG 05:50:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502258" 05:50:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713502258" 05:50:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502258" 05:50:58 INFO [http_cmd]: http status code OK 05:50:58 DEBUG --- status code --- 05:50:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:50:58 DEBUG --- http response --- 05:50:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:50:58 DEBUG ----- output ----- 05:50:58 DEBUG { 05:50:58 DEBUG "status": "success", 05:50:58 DEBUG "data": { 05:50:58 DEBUG "resultType": "vector", 05:50:58 DEBUG "result": [ 05:50:58 DEBUG { 05:50:58 DEBUG "metric": { 05:50:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:50:58 DEBUG }, 05:50:58 DEBUG "value": [ 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG [...] 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:50:58 DEBUG }, 05:50:58 DEBUG "value": [ 05:50:58 DEBUG 1713502258, 05:50:58 DEBUG "0" 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG } 05:50:58 ERROR [http_cmd]: ERROR 05:50:58 ERROR --- status code --- 05:50:58 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:50:58 ERROR --- http response --- 05:50:58 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:50:58 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:50:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713502258" 05:50:58 INFO [http_cmd]: http status code OK 05:50:58 DEBUG --- status code --- 05:50:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:50:58 DEBUG --- http response --- 05:50:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:50:58 DEBUG ----- output ----- 05:50:58 DEBUG { 05:50:58 DEBUG "status": "success", 05:50:58 DEBUG "data": { 05:50:58 DEBUG "resultType": "vector", 05:50:58 DEBUG "result": [ 05:50:58 DEBUG { 05:50:58 DEBUG "metric": { 05:50:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:50:58 DEBUG }, 05:50:58 DEBUG "value": [ 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG [...] 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:50:58 DEBUG }, 05:50:58 DEBUG "value": [ 05:50:58 DEBUG 1713502258, 05:50:58 DEBUG "3328896" 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG } 05:50:58 INFO [http_cmd]: http status code OK 05:50:58 DEBUG --- status code --- 05:50:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:50:58 DEBUG --- http response --- 05:50:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:50:58 DEBUG ----- output ----- 05:50:58 DEBUG { 05:50:58 DEBUG "status": "success", 05:50:58 DEBUG "data": { 05:50:58 DEBUG "resultType": "vector", 05:50:58 DEBUG "result": [ 05:50:58 DEBUG { 05:50:58 DEBUG "metric": { 05:50:58 DEBUG "pod": "ds-idrepo-0" 05:50:58 DEBUG }, 05:50:58 DEBUG "value": [ 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG [...] 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG "pod": "ds-cts-2" 05:50:58 DEBUG }, 05:50:58 DEBUG "value": [ 05:50:58 DEBUG 1713502258, 05:50:58 DEBUG "0" 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG } 05:50:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502258" 05:50:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713502258" 05:50:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502258" 05:50:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713502258" 05:50:58 INFO [http_cmd]: http status code OK 05:50:58 DEBUG --- status code --- 05:50:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:50:58 DEBUG --- http response --- 05:50:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:50:58 DEBUG ----- output ----- 05:50:58 DEBUG { 05:50:58 DEBUG "status": "success", 05:50:58 DEBUG "data": { 05:50:58 DEBUG "resultType": "vector", 05:50:58 DEBUG "result": [ 05:50:58 DEBUG { 05:50:58 DEBUG "metric": { 05:50:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:50:58 DEBUG }, 05:50:58 DEBUG "value": [ 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG [...] 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:50:58 DEBUG }, 05:50:58 INFO [http_cmd]: http status code OK 05:50:58 DEBUG "value": [ 05:50:58 DEBUG --- status code --- 05:50:58 DEBUG 1713502258, 05:50:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:50:58 DEBUG "0" 05:50:58 DEBUG --- http response --- 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG } 05:50:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:50:58 DEBUG ----- output ----- 05:50:58 DEBUG { 05:50:58 DEBUG "status": "success", 05:50:58 DEBUG "data": { 05:50:58 DEBUG "resultType": "vector", 05:50:58 DEBUG "result": [ 05:50:58 DEBUG { 05:50:58 DEBUG "metric": { 05:50:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:50:58 DEBUG }, 05:50:58 DEBUG "value": [ 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG [...] 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 05:50:58 DEBUG }, 05:50:58 DEBUG "value": [ 05:50:58 DEBUG 1713502258, 05:50:58 DEBUG "0" 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG } 05:50:58 INFO [http_cmd]: http status code OK 05:50:58 INFO [http_cmd]: http status code OK 05:50:58 DEBUG --- status code --- 05:50:58 DEBUG --- status code --- 05:50:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:50:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:50:58 DEBUG --- http response --- 05:50:58 DEBUG --- http response --- 05:50:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502258" 05:50:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:50:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:50:58 DEBUG ----- output ----- 05:50:58 DEBUG ----- output ----- 05:50:58 DEBUG { 05:50:58 DEBUG { 05:50:58 DEBUG "status": "success", 05:50:58 DEBUG "status": "success", 05:50:58 DEBUG "data": { 05:50:58 DEBUG "data": { 05:50:58 DEBUG "resultType": "vector", 05:50:58 DEBUG "resultType": "vector", 05:50:58 DEBUG "result": [ 05:50:58 DEBUG "result": [ 05:50:58 DEBUG { 05:50:58 DEBUG { 05:50:58 DEBUG "metric": { 05:50:58 DEBUG "metric": { 05:50:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:50:58 DEBUG "instance": "10.142.0.107:9100" 05:50:58 DEBUG }, 05:50:58 DEBUG }, 05:50:58 DEBUG "value": [ 05:50:58 DEBUG "value": [ 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG [...] 05:50:58 DEBUG [...] 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:50:58 DEBUG "instance": "10.142.0.43:9100" 05:50:58 DEBUG }, 05:50:58 DEBUG }, 05:50:58 DEBUG "value": [ 05:50:58 DEBUG "value": [ 05:50:58 DEBUG 1713502258, 05:50:58 DEBUG 1713502258, 05:50:58 DEBUG "55.176" 05:50:58 DEBUG "575106968576" 05:50:58 DEBUG ] 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG } 05:50:58 DEBUG ] 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG } 05:50:58 DEBUG } 05:50:58 DEBUG } 05:50:58 INFO [http_cmd]: http status code OK 05:50:58 DEBUG --- status code --- 05:50:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:50:58 DEBUG --- http response --- 05:50:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:50:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502258" 05:50:58 INFO [http_cmd]: http status code OK 05:50:58 DEBUG --- status code --- 05:50:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:50:58 DEBUG --- http response --- 05:50:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:50:58 DEBUG ----- output ----- 05:50:58 DEBUG { 05:50:58 DEBUG "status": "success", 05:50:58 DEBUG "data": { 05:50:58 DEBUG "resultType": "vector", 05:50:58 DEBUG "result": [ 05:50:58 DEBUG { 05:50:58 DEBUG "metric": { 05:50:58 DEBUG "pod": "ds-cts-0" 05:50:58 DEBUG }, 05:50:58 DEBUG "value": [ 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG [...] 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG "pod": "ds-cts-2" 05:50:58 DEBUG }, 05:50:58 DEBUG "value": [ 05:50:58 DEBUG 1713502258, 05:50:58 DEBUG "294.3666666666667" 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG } 05:50:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502258" 05:50:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502258" 05:50:58 INFO [http_cmd]: http status code OK 05:50:58 DEBUG --- status code --- 05:50:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:50:58 DEBUG --- http response --- 05:50:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:50:58 DEBUG ----- output ----- 05:50:58 DEBUG { 05:50:58 DEBUG "status": "success", 05:50:58 DEBUG "data": { 05:50:58 DEBUG "resultType": "vector", 05:50:58 DEBUG "result": [ 05:50:58 DEBUG { 05:50:58 DEBUG "metric": { 05:50:58 DEBUG "pod": "ds-idrepo-0" 05:50:58 DEBUG }, 05:50:58 DEBUG "value": [ 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG [...] 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG "pod": "ds-idrepo-2" 05:50:58 DEBUG }, 05:50:58 DEBUG "value": [ 05:50:58 DEBUG 1713502258, 05:50:58 DEBUG "0" 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG } 05:50:58 INFO [http_cmd]: http status code OK 05:50:58 DEBUG --- status code --- 05:50:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:50:58 DEBUG --- http response --- 05:50:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713502258" 05:50:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:50:58 DEBUG ----- output ----- 05:50:58 DEBUG { 05:50:58 DEBUG "status": "success", 05:50:58 DEBUG "data": { 05:50:58 DEBUG "resultType": "vector", 05:50:58 DEBUG "result": [ 05:50:58 DEBUG { 05:50:58 DEBUG "metric": { 05:50:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:50:58 DEBUG }, 05:50:58 DEBUG "value": [ 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG [...] 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:50:58 DEBUG }, 05:50:58 DEBUG "value": [ 05:50:58 DEBUG 1713502258, 05:50:58 DEBUG "0" 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG } 05:50:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502258" 05:50:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502258" 05:50:58 INFO [http_cmd]: http status code OK 05:50:58 DEBUG --- status code --- 05:50:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:50:58 DEBUG --- http response --- 05:50:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:50:58 DEBUG ----- output ----- 05:50:58 DEBUG { 05:50:58 DEBUG "status": "success", 05:50:58 DEBUG "data": { 05:50:58 DEBUG "resultType": "vector", 05:50:58 DEBUG "result": [ 05:50:58 DEBUG { 05:50:58 DEBUG "metric": { 05:50:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:50:58 DEBUG }, 05:50:58 DEBUG "value": [ 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG [...] 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:50:58 DEBUG }, 05:50:58 DEBUG "value": [ 05:50:58 DEBUG 1713502258, 05:50:58 DEBUG "1.0306487302989569" 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG } 05:50:58 INFO [http_cmd]: http status code OK 05:50:58 DEBUG --- status code --- 05:50:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:50:58 DEBUG --- http response --- 05:50:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:50:58 DEBUG ----- output ----- 05:50:58 DEBUG { 05:50:58 DEBUG "status": "success", 05:50:58 DEBUG "data": { 05:50:58 DEBUG "resultType": "vector", 05:50:58 DEBUG "result": [ 05:50:58 DEBUG { 05:50:58 DEBUG "metric": { 05:50:58 DEBUG "pod": "ds-idrepo-0" 05:50:58 DEBUG }, 05:50:58 DEBUG "value": [ 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG [...] 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG "pod": "ds-idrepo-2" 05:50:58 DEBUG }, 05:50:58 DEBUG "value": [ 05:50:58 DEBUG 1713502258, 05:50:58 DEBUG "0" 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG } 05:50:58 INFO [http_cmd]: http status code OK 05:50:58 DEBUG --- status code --- 05:50:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:50:58 DEBUG --- http response --- 05:50:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:50:58 DEBUG ----- output ----- 05:50:58 DEBUG { 05:50:58 DEBUG "status": "success", 05:50:58 DEBUG "data": { 05:50:58 DEBUG "resultType": "vector", 05:50:58 DEBUG "result": [ 05:50:58 DEBUG { 05:50:58 DEBUG "metric": { 05:50:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:50:58 DEBUG }, 05:50:58 DEBUG "value": [ 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG [...] 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 05:50:58 DEBUG }, 05:50:58 DEBUG "value": [ 05:50:58 DEBUG 1713502258, 05:50:58 DEBUG "0" 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG } 05:50:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502258" 05:50:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713502258" 05:50:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502258" 05:50:58 INFO [http_cmd]: http status code OK 05:50:58 DEBUG --- status code --- 05:50:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:50:58 DEBUG --- http response --- 05:50:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:50:58 DEBUG ----- output ----- 05:50:58 DEBUG { 05:50:58 DEBUG "status": "success", 05:50:58 DEBUG "data": { 05:50:58 DEBUG "resultType": "vector", 05:50:58 DEBUG "result": [ 05:50:58 DEBUG { 05:50:58 DEBUG "metric": { 05:50:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:50:58 DEBUG }, 05:50:58 DEBUG "value": [ 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG [...] 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:50:58 DEBUG }, 05:50:58 DEBUG "value": [ 05:50:58 DEBUG 1713502258, 05:50:58 DEBUG "0" 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG } 05:50:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502258" 05:50:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713502258" 05:50:58 INFO [http_cmd]: http status code OK 05:50:58 INFO [http_cmd]: http status code OK 05:50:58 DEBUG --- status code --- 05:50:58 DEBUG --- status code --- 05:50:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:50:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:50:58 DEBUG --- http response --- 05:50:58 DEBUG --- http response --- 05:50:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:50:58 DEBUG ----- output ----- 05:50:58 DEBUG { 05:50:58 DEBUG "status": "success", 05:50:58 DEBUG "data": { 05:50:58 DEBUG "resultType": "vector", 05:50:58 DEBUG "result": [ 05:50:58 DEBUG { 05:50:58 DEBUG "metric": { 05:50:58 DEBUG "instance": "10.142.0.107:9100" 05:50:58 DEBUG }, 05:50:58 DEBUG "value": [ 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG [...] 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG "instance": "10.142.0.43:9100" 05:50:58 DEBUG }, 05:50:58 DEBUG "value": [ 05:50:58 DEBUG 1713502258, 05:50:58 DEBUG "0.7949999999999999" 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG } 05:50:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:50:58 DEBUG ----- output ----- 05:50:58 DEBUG { 05:50:58 DEBUG "status": "success", 05:50:58 DEBUG "data": { 05:50:58 DEBUG "resultType": "vector", 05:50:58 DEBUG "result": [ 05:50:58 DEBUG { 05:50:58 DEBUG "metric": { 05:50:58 DEBUG "pod": "ds-idrepo-0" 05:50:58 DEBUG }, 05:50:58 DEBUG "value": [ 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG [...] 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG "pod": "ds-idrepo-2" 05:50:58 DEBUG }, 05:50:58 DEBUG "value": [ 05:50:58 DEBUG 1713502258, 05:50:58 DEBUG "0" 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG } 05:50:58 INFO [http_cmd]: http status code OK 05:50:58 DEBUG --- status code --- 05:50:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:50:58 DEBUG --- http response --- 05:50:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:50:58 DEBUG ----- output ----- 05:50:58 DEBUG { 05:50:58 DEBUG "status": "success", 05:50:58 DEBUG "data": { 05:50:58 DEBUG "resultType": "vector", 05:50:58 DEBUG "result": [ 05:50:58 DEBUG { 05:50:58 DEBUG "metric": { 05:50:58 DEBUG "pod": "ds-cts-0" 05:50:58 DEBUG }, 05:50:58 DEBUG "value": [ 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG [...] 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG "pod": "ds-cts-2" 05:50:58 DEBUG }, 05:50:58 DEBUG "value": [ 05:50:58 DEBUG 1713502258, 05:50:58 DEBUG "30" 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG } 05:50:58 INFO [http_cmd]: http status code OK 05:50:58 DEBUG --- status code --- 05:50:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:50:58 DEBUG --- http response --- 05:50:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:50:58 DEBUG ----- output ----- 05:50:58 DEBUG { 05:50:58 DEBUG "status": "success", 05:50:58 DEBUG "data": { 05:50:58 DEBUG "resultType": "vector", 05:50:58 DEBUG "result": [ 05:50:58 DEBUG { 05:50:58 DEBUG "metric": { 05:50:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:50:58 DEBUG }, 05:50:58 DEBUG "value": [ 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG [...] 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:50:58 DEBUG }, 05:50:58 DEBUG "value": [ 05:50:58 DEBUG 1713502258, 05:50:58 DEBUG "3298181120" 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG } 05:50:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502258" 05:50:58 INFO [http_cmd]: http status code OK 05:50:58 DEBUG --- status code --- 05:50:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:50:58 DEBUG --- http response --- 05:50:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502258" 05:50:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:50:58 DEBUG ----- output ----- 05:50:58 DEBUG { 05:50:58 DEBUG "status": "success", 05:50:58 DEBUG "data": { 05:50:58 DEBUG "resultType": "vector", 05:50:58 DEBUG "result": [ 05:50:58 DEBUG { 05:50:58 DEBUG "metric": { 05:50:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:50:58 DEBUG }, 05:50:58 DEBUG "value": [ 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG [...] 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:50:58 DEBUG }, 05:50:58 DEBUG "value": [ 05:50:58 DEBUG 1713502258, 05:50:58 DEBUG "2170.866666666667" 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG } 05:50:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502258" 05:50:58 INFO [http_cmd]: http status code OK 05:50:58 DEBUG --- status code --- 05:50:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:50:58 DEBUG --- http response --- 05:50:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 05:50:58 DEBUG ----- output ----- 05:50:58 DEBUG { 05:50:58 DEBUG "status": "success", 05:50:58 DEBUG "data": { 05:50:58 DEBUG "resultType": "vector", 05:50:58 DEBUG "result": [ 05:50:58 DEBUG { 05:50:58 DEBUG "metric": { 05:50:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:50:58 DEBUG }, 05:50:58 DEBUG "value": [ 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG [...] 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG "pod": "idm-b85f9d978-wnhxb" 05:50:58 DEBUG }, 05:50:58 DEBUG "value": [ 05:50:58 DEBUG 1713502258, 05:50:58 DEBUG "0" 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG } 05:50:58 INFO [http_cmd]: http status code OK 05:50:58 DEBUG --- status code --- 05:50:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:50:58 DEBUG --- http response --- 05:50:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 05:50:58 DEBUG ----- output ----- 05:50:58 DEBUG { 05:50:58 DEBUG "status": "success", 05:50:58 DEBUG "data": { 05:50:58 DEBUG "resultType": "vector", 05:50:58 DEBUG "result": [ 05:50:58 DEBUG { 05:50:58 DEBUG "metric": { 05:50:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:50:58 DEBUG }, 05:50:58 DEBUG "value": [ 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG [...] 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG "pod": "idm-b85f9d978-wnhxb" 05:50:58 DEBUG }, 05:50:58 DEBUG "value": [ 05:50:58 DEBUG 1713502258, 05:50:58 DEBUG "981.5892855536642" 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG } 05:50:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502258" 05:50:58 INFO [http_cmd]: http status code OK 05:50:58 DEBUG --- status code --- 05:50:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:50:58 DEBUG --- http response --- 05:50:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:50:58 DEBUG ----- output ----- 05:50:58 DEBUG { 05:50:58 DEBUG "status": "success", 05:50:58 DEBUG "data": { 05:50:58 DEBUG "resultType": "vector", 05:50:58 DEBUG "result": [ 05:50:58 DEBUG { 05:50:58 DEBUG "metric": { 05:50:58 DEBUG "pod": "ds-cts-0" 05:50:58 DEBUG }, 05:50:58 DEBUG "value": [ 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG [...] 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG "pod": "ds-cts-2" 05:50:58 DEBUG }, 05:50:58 DEBUG "value": [ 05:50:58 DEBUG 1713502258, 05:50:58 DEBUG "0.0003546133333333333" 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG } 05:50:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713502258" 05:50:58 INFO [http_cmd]: http status code OK 05:50:58 DEBUG --- status code --- 05:50:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:50:58 DEBUG --- http response --- 05:50:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:50:58 DEBUG ----- output ----- 05:50:58 DEBUG { 05:50:58 DEBUG "status": "success", 05:50:58 DEBUG "data": { 05:50:58 DEBUG "resultType": "vector", 05:50:58 DEBUG "result": [ 05:50:58 DEBUG { 05:50:58 DEBUG "metric": { 05:50:58 DEBUG "instance": "10.142.0.107:9100" 05:50:58 DEBUG }, 05:50:58 DEBUG "value": [ 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG [...] 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG "instance": "10.142.0.43:9100" 05:50:58 DEBUG }, 05:50:58 DEBUG "value": [ 05:50:58 DEBUG 1713502258, 05:50:58 DEBUG "14219644.166666666" 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG } 05:50:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713502258" 05:50:58 INFO [http_cmd]: http status code OK 05:50:58 DEBUG --- status code --- 05:50:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:50:58 DEBUG --- http response --- 05:50:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:50:58 DEBUG ----- output ----- 05:50:58 DEBUG { 05:50:58 DEBUG "status": "success", 05:50:58 DEBUG "data": { 05:50:58 DEBUG "resultType": "vector", 05:50:58 DEBUG "result": [ 05:50:58 DEBUG { 05:50:58 DEBUG "metric": { 05:50:58 DEBUG "instance": "10.142.0.107:9100" 05:50:58 DEBUG }, 05:50:58 DEBUG "value": [ 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG [...] 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG "instance": "10.142.0.43:9100" 05:50:58 DEBUG }, 05:50:58 DEBUG "value": [ 05:50:58 DEBUG 1713502258, 05:50:58 DEBUG "0.6768333333333193" 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG } 05:50:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713502258" 05:50:58 INFO [http_cmd]: http status code OK 05:50:58 DEBUG --- status code --- 05:50:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:50:58 DEBUG --- http response --- 05:50:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:50:58 DEBUG ----- output ----- 05:50:58 DEBUG { 05:50:58 DEBUG "status": "success", 05:50:58 DEBUG "data": { 05:50:58 DEBUG "resultType": "vector", 05:50:58 DEBUG "result": [ 05:50:58 DEBUG { 05:50:58 DEBUG "metric": { 05:50:58 DEBUG "instance": "10.142.0.107:9100" 05:50:58 DEBUG }, 05:50:58 DEBUG "value": [ 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG [...] 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG 05:50:58 DEBUG "instance": "10.142.0.43:9100" 05:50:58 DEBUG }, 05:50:58 DEBUG "value": [ 05:50:58 DEBUG 1713502258, 05:50:58 DEBUG "14210334.933333334" 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG ] 05:50:58 DEBUG } 05:50:58 DEBUG } 05:51:00 INFO 05:51:00 INFO [loop_until]: kubectl --namespace=xlou top pods 05:51:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:51:00 INFO [loop_until]: OK (rc = 0) 05:51:00 DEBUG --- stdout --- 05:51:00 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 2052m 9115Mi am-6fd554f8f7-hvvkw 2015m 8885Mi am-6fd554f8f7-pmj4b 2161m 8766Mi ds-cts-0 8333m 13682Mi ds-cts-1 7719m 13549Mi ds-cts-2 8652m 13915Mi ds-idrepo-0 620m 3297Mi ds-idrepo-1 611m 3212Mi ds-idrepo-2 582m 3093Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 3m 1216Mi lodemon-6b89d774d5-rf7sk 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 564m 2874Mi 05:51:00 DEBUG --- stderr --- 05:51:00 DEBUG 05:51:01 INFO 05:51:01 INFO [loop_until]: kubectl --namespace=xlou top node 05:51:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:51:02 INFO [loop_until]: OK (rc = 0) 05:51:02 DEBUG --- stdout --- 05:51:02 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 71m 0% 6243Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 690m 2% 6746Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 2058m 6% 11649Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 80m 0% 3262Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 8405m 28% 16357Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 156m 0% 5186Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 2214m 7% 11937Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 79m 0% 2687Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 2283m 7% 11002Mi 9% gke-xlou-cdm-ds-0f601751-j488 649m 2% 6193Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 7862m 26% 17763Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 704m 2% 6900Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 8720m 29% 14635Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 846m 5% 6570Mi 11% 05:51:02 DEBUG --- stderr --- 05:51:02 DEBUG 05:51:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502318" 05:51:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713502318" 05:51:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502318" 05:51:58 ERROR [http_cmd]: ERROR 05:51:58 ERROR --- status code --- 05:51:58 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:51:58 ERROR --- http response --- 05:51:58 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:51:58 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:51:58 INFO [http_cmd]: http status code OK 05:51:58 DEBUG --- status code --- 05:51:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:51:58 DEBUG --- http response --- 05:51:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:51:58 DEBUG ----- output ----- 05:51:58 DEBUG { 05:51:58 DEBUG "status": "success", 05:51:58 DEBUG "data": { 05:51:58 DEBUG "resultType": "vector", 05:51:58 DEBUG "result": [ 05:51:58 DEBUG { 05:51:58 DEBUG "metric": { 05:51:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:51:58 DEBUG }, 05:51:58 DEBUG "value": [ 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG [...] 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:51:58 DEBUG }, 05:51:58 DEBUG "value": [ 05:51:58 DEBUG 1713502318, 05:51:58 DEBUG "0" 05:51:58 DEBUG ] 05:51:58 DEBUG } 05:51:58 DEBUG ] 05:51:58 DEBUG } 05:51:58 DEBUG } 05:51:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713502318" 05:51:58 INFO [http_cmd]: http status code OK 05:51:58 DEBUG --- status code --- 05:51:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:51:58 DEBUG --- http response --- 05:51:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:51:58 DEBUG ----- output ----- 05:51:58 DEBUG { 05:51:58 DEBUG "status": "success", 05:51:58 DEBUG "data": { 05:51:58 DEBUG "resultType": "vector", 05:51:58 DEBUG "result": [ 05:51:58 DEBUG { 05:51:58 DEBUG "metric": { 05:51:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:51:58 DEBUG }, 05:51:58 DEBUG "value": [ 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG [...] 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:51:58 DEBUG }, 05:51:58 DEBUG "value": [ 05:51:58 DEBUG 1713502318, 05:51:58 DEBUG "3198971.5" 05:51:58 DEBUG ] 05:51:58 DEBUG } 05:51:58 DEBUG ] 05:51:58 DEBUG } 05:51:58 DEBUG } 05:51:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502318" 05:51:58 INFO [http_cmd]: http status code OK 05:51:58 DEBUG --- status code --- 05:51:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:51:58 DEBUG --- http response --- 05:51:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:51:58 DEBUG ----- output ----- 05:51:58 DEBUG { 05:51:58 DEBUG "status": "success", 05:51:58 DEBUG "data": { 05:51:58 DEBUG "resultType": "vector", 05:51:58 DEBUG "result": [ 05:51:58 DEBUG { 05:51:58 DEBUG "metric": { 05:51:58 DEBUG "pod": "ds-idrepo-0" 05:51:58 DEBUG }, 05:51:58 DEBUG "value": [ 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG [...] 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG "pod": "ds-cts-2" 05:51:58 DEBUG }, 05:51:58 DEBUG "value": [ 05:51:58 DEBUG 1713502318, 05:51:58 DEBUG "0" 05:51:58 DEBUG ] 05:51:58 DEBUG } 05:51:58 DEBUG ] 05:51:58 DEBUG } 05:51:58 DEBUG } 05:51:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502318" 05:51:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713502318" 05:51:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713502318" 05:51:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502318" 05:51:58 INFO [http_cmd]: http status code OK 05:51:58 DEBUG --- status code --- 05:51:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:51:58 DEBUG --- http response --- 05:51:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:51:58 DEBUG ----- output ----- 05:51:58 DEBUG { 05:51:58 DEBUG "status": "success", 05:51:58 DEBUG "data": { 05:51:58 DEBUG "resultType": "vector", 05:51:58 DEBUG "result": [ 05:51:58 DEBUG { 05:51:58 DEBUG "metric": { 05:51:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:51:58 DEBUG }, 05:51:58 DEBUG "value": [ 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG [...] 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:51:58 DEBUG }, 05:51:58 DEBUG "value": [ 05:51:58 DEBUG 1713502318, 05:51:58 DEBUG "0" 05:51:58 DEBUG ] 05:51:58 DEBUG } 05:51:58 DEBUG ] 05:51:58 DEBUG } 05:51:58 DEBUG } 05:51:58 INFO [http_cmd]: http status code OK 05:51:58 DEBUG --- status code --- 05:51:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:51:58 DEBUG --- http response --- 05:51:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:51:58 DEBUG ----- output ----- 05:51:58 DEBUG { 05:51:58 DEBUG "status": "success", 05:51:58 DEBUG "data": { 05:51:58 DEBUG "resultType": "vector", 05:51:58 DEBUG "result": [ 05:51:58 DEBUG { 05:51:58 DEBUG "metric": { 05:51:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:51:58 DEBUG }, 05:51:58 DEBUG "value": [ 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG [...] 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG "pod": "ds-idrepo-2" 05:51:58 DEBUG }, 05:51:58 DEBUG "value": [ 05:51:58 DEBUG 1713502318, 05:51:58 DEBUG "125305729.2909108" 05:51:58 DEBUG ] 05:51:58 DEBUG } 05:51:58 DEBUG ] 05:51:58 DEBUG } 05:51:58 DEBUG } 05:51:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502318" 05:51:58 INFO [http_cmd]: http status code OK 05:51:58 DEBUG --- status code --- 05:51:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:51:58 DEBUG --- http response --- 05:51:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:51:58 DEBUG ----- output ----- 05:51:58 DEBUG { 05:51:58 DEBUG "status": "success", 05:51:58 DEBUG "data": { 05:51:58 DEBUG "resultType": "vector", 05:51:58 DEBUG "result": [ 05:51:58 DEBUG { 05:51:58 DEBUG "metric": { 05:51:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:51:58 DEBUG }, 05:51:58 DEBUG "value": [ 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG [...] 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:51:58 DEBUG }, 05:51:58 DEBUG "value": [ 05:51:58 DEBUG 1713502318, 05:51:58 DEBUG "55.176" 05:51:58 DEBUG ] 05:51:58 DEBUG } 05:51:58 DEBUG ] 05:51:58 DEBUG } 05:51:58 DEBUG } 05:51:58 INFO [http_cmd]: http status code OK 05:51:58 INFO [http_cmd]: http status code OK 05:51:58 DEBUG --- status code --- 05:51:58 DEBUG --- status code --- 05:51:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:51:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:51:58 DEBUG --- http response --- 05:51:58 DEBUG --- http response --- 05:51:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:51:58 DEBUG ----- output ----- 05:51:58 DEBUG { 05:51:58 DEBUG "status": "success", 05:51:58 DEBUG "data": { 05:51:58 DEBUG "resultType": "vector", 05:51:58 DEBUG "result": [ 05:51:58 DEBUG { 05:51:58 DEBUG "metric": { 05:51:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:51:58 DEBUG }, 05:51:58 DEBUG "value": [ 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG [...] 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:51:58 DEBUG }, 05:51:58 DEBUG "value": [ 05:51:58 DEBUG 1713502318, 05:51:58 DEBUG "0" 05:51:58 DEBUG ] 05:51:58 DEBUG } 05:51:58 DEBUG ] 05:51:58 DEBUG } 05:51:58 DEBUG } 05:51:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:51:58 DEBUG ----- output ----- 05:51:58 DEBUG { 05:51:58 DEBUG "status": "success", 05:51:58 DEBUG "data": { 05:51:58 DEBUG "resultType": "vector", 05:51:58 DEBUG "result": [ 05:51:58 DEBUG { 05:51:58 DEBUG "metric": { 05:51:58 DEBUG "instance": "10.142.0.107:9100" 05:51:58 DEBUG }, 05:51:58 DEBUG "value": [ 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG [...] 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG "instance": "10.142.0.43:9100" 05:51:58 DEBUG }, 05:51:58 DEBUG "value": [ 05:51:58 DEBUG 1713502318, 05:51:58 DEBUG "576368073728" 05:51:58 DEBUG ] 05:51:58 DEBUG } 05:51:58 DEBUG ] 05:51:58 DEBUG } 05:51:58 DEBUG } 05:51:58 INFO [http_cmd]: http status code OK 05:51:58 DEBUG --- status code --- 05:51:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:51:58 DEBUG --- http response --- 05:51:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:51:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502318" 05:51:58 INFO [http_cmd]: http status code OK 05:51:58 DEBUG --- status code --- 05:51:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:51:58 DEBUG --- http response --- 05:51:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:51:58 DEBUG ----- output ----- 05:51:58 DEBUG { 05:51:58 DEBUG "status": "success", 05:51:58 DEBUG "data": { 05:51:58 DEBUG "resultType": "vector", 05:51:58 DEBUG "result": [ 05:51:58 DEBUG { 05:51:58 DEBUG "metric": { 05:51:58 DEBUG "pod": "ds-cts-0" 05:51:58 DEBUG }, 05:51:58 DEBUG "value": [ 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG [...] 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG "pod": "ds-cts-2" 05:51:58 DEBUG }, 05:51:58 DEBUG "value": [ 05:51:58 DEBUG 1713502318, 05:51:58 DEBUG "76.9" 05:51:58 DEBUG ] 05:51:58 DEBUG } 05:51:58 DEBUG ] 05:51:58 DEBUG } 05:51:58 DEBUG } 05:51:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502318" 05:51:58 INFO [http_cmd]: http status code OK 05:51:58 DEBUG --- status code --- 05:51:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:51:58 DEBUG --- http response --- 05:51:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:51:58 DEBUG ----- output ----- 05:51:58 DEBUG { 05:51:58 DEBUG "status": "success", 05:51:58 DEBUG "data": { 05:51:58 DEBUG "resultType": "vector", 05:51:58 DEBUG "result": [ 05:51:58 DEBUG { 05:51:58 DEBUG "metric": { 05:51:58 DEBUG "pod": "ds-idrepo-0" 05:51:58 DEBUG }, 05:51:58 DEBUG "value": [ 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG [...] 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG "pod": "ds-idrepo-2" 05:51:58 DEBUG }, 05:51:58 DEBUG "value": [ 05:51:58 DEBUG 1713502318, 05:51:58 DEBUG "0" 05:51:58 DEBUG ] 05:51:58 DEBUG } 05:51:58 DEBUG ] 05:51:58 DEBUG } 05:51:58 DEBUG } 05:51:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713502318" 05:51:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502318" 05:51:58 INFO [http_cmd]: http status code OK 05:51:58 DEBUG --- status code --- 05:51:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:51:58 DEBUG --- http response --- 05:51:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502318" 05:51:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:51:58 DEBUG ----- output ----- 05:51:58 DEBUG { 05:51:58 DEBUG "status": "success", 05:51:58 DEBUG "data": { 05:51:58 DEBUG "resultType": "vector", 05:51:58 DEBUG "result": [ 05:51:58 DEBUG { 05:51:58 DEBUG "metric": { 05:51:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:51:58 DEBUG }, 05:51:58 DEBUG "value": [ 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG [...] 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:51:58 DEBUG }, 05:51:58 DEBUG "value": [ 05:51:58 DEBUG 1713502318, 05:51:58 DEBUG "0.9645341053576046" 05:51:58 DEBUG ] 05:51:58 DEBUG } 05:51:58 DEBUG ] 05:51:58 DEBUG } 05:51:58 DEBUG } 05:51:58 INFO [http_cmd]: http status code OK 05:51:58 DEBUG --- status code --- 05:51:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:51:58 DEBUG --- http response --- 05:51:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:51:58 DEBUG ----- output ----- 05:51:58 DEBUG { 05:51:58 DEBUG "status": "success", 05:51:58 DEBUG "data": { 05:51:58 DEBUG "resultType": "vector", 05:51:58 DEBUG "result": [ 05:51:58 DEBUG { 05:51:58 DEBUG "metric": { 05:51:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:51:58 DEBUG }, 05:51:58 DEBUG "value": [ 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG [...] 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG "pod": "ds-idrepo-2" 05:51:58 DEBUG }, 05:51:58 DEBUG "value": [ 05:51:58 DEBUG 1713502318, 05:51:58 DEBUG "0.6079720860523877" 05:51:58 DEBUG ] 05:51:58 DEBUG } 05:51:58 DEBUG ] 05:51:58 DEBUG } 05:51:58 DEBUG } 05:51:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502318" 05:51:58 INFO [http_cmd]: http status code OK 05:51:58 DEBUG --- status code --- 05:51:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:51:58 DEBUG --- http response --- 05:51:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:51:58 DEBUG ----- output ----- 05:51:58 DEBUG { 05:51:58 DEBUG "status": "success", 05:51:58 DEBUG "data": { 05:51:58 DEBUG "resultType": "vector", 05:51:58 DEBUG "result": [ 05:51:58 DEBUG { 05:51:58 DEBUG "metric": { 05:51:58 DEBUG "pod": "ds-idrepo-0" 05:51:58 DEBUG }, 05:51:58 DEBUG "value": [ 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG [...] 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG "pod": "ds-idrepo-2" 05:51:58 DEBUG }, 05:51:58 DEBUG "value": [ 05:51:58 DEBUG 1713502318, 05:51:58 DEBUG "0" 05:51:58 DEBUG ] 05:51:58 DEBUG } 05:51:58 DEBUG ] 05:51:58 DEBUG } 05:51:58 DEBUG } 05:51:58 INFO [http_cmd]: http status code OK 05:51:58 DEBUG --- status code --- 05:51:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:51:58 DEBUG --- http response --- 05:51:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:51:58 DEBUG ----- output ----- 05:51:58 DEBUG { 05:51:58 DEBUG "status": "success", 05:51:58 DEBUG "data": { 05:51:58 DEBUG "resultType": "vector", 05:51:58 DEBUG "result": [ 05:51:58 DEBUG { 05:51:58 DEBUG "metric": { 05:51:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:51:58 DEBUG }, 05:51:58 DEBUG "value": [ 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG [...] 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:51:58 DEBUG }, 05:51:58 DEBUG "value": [ 05:51:58 DEBUG 1713502318, 05:51:58 DEBUG "0" 05:51:58 DEBUG ] 05:51:58 DEBUG } 05:51:58 DEBUG ] 05:51:58 DEBUG } 05:51:58 DEBUG } 05:51:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502318" 05:51:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713502318" 05:51:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502318" 05:51:58 INFO [http_cmd]: http status code OK 05:51:58 DEBUG --- status code --- 05:51:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:51:58 DEBUG --- http response --- 05:51:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502318" 05:51:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713502318" 05:51:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:51:58 DEBUG ----- output ----- 05:51:58 DEBUG { 05:51:58 DEBUG "status": "success", 05:51:58 DEBUG "data": { 05:51:58 DEBUG "resultType": "vector", 05:51:58 DEBUG "result": [ 05:51:58 DEBUG { 05:51:58 DEBUG "metric": { 05:51:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:51:58 DEBUG }, 05:51:58 DEBUG "value": [ 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG [...] 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:51:58 DEBUG }, 05:51:58 DEBUG "value": [ 05:51:58 DEBUG 1713502318, 05:51:58 DEBUG "2165.2" 05:51:58 DEBUG ] 05:51:58 DEBUG } 05:51:58 DEBUG ] 05:51:58 DEBUG } 05:51:58 DEBUG } 05:51:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502318" 05:51:58 INFO [http_cmd]: http status code OK 05:51:58 DEBUG --- status code --- 05:51:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:51:58 DEBUG --- http response --- 05:51:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:51:58 DEBUG ----- output ----- 05:51:58 DEBUG { 05:51:58 DEBUG "status": "success", 05:51:58 DEBUG "data": { 05:51:58 DEBUG "resultType": "vector", 05:51:58 DEBUG "result": [ 05:51:58 DEBUG { 05:51:58 DEBUG "metric": { 05:51:58 DEBUG "pod": "ds-idrepo-0" 05:51:58 DEBUG }, 05:51:58 DEBUG "value": [ 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG [...] 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG "pod": "ds-idrepo-2" 05:51:58 DEBUG }, 05:51:58 DEBUG "value": [ 05:51:58 DEBUG 1713502318, 05:51:58 DEBUG "0" 05:51:58 DEBUG ] 05:51:58 DEBUG } 05:51:58 DEBUG ] 05:51:58 DEBUG } 05:51:58 DEBUG } 05:51:58 INFO [http_cmd]: http status code OK 05:51:58 DEBUG --- status code --- 05:51:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:51:58 DEBUG --- http response --- 05:51:58 INFO [http_cmd]: http status code OK 05:51:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:51:58 DEBUG --- status code --- 05:51:58 DEBUG ----- output ----- 05:51:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:51:58 DEBUG { 05:51:58 DEBUG --- http response --- 05:51:58 DEBUG "status": "success", 05:51:58 DEBUG "data": { 05:51:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:51:58 DEBUG "resultType": "vector", 05:51:58 INFO [http_cmd]: http status code OK 05:51:58 DEBUG ----- output ----- 05:51:58 DEBUG "result": [ 05:51:58 DEBUG --- status code --- 05:51:58 DEBUG { 05:51:58 DEBUG { 05:51:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:51:58 DEBUG "status": "success", 05:51:58 DEBUG "metric": { 05:51:58 DEBUG --- http response --- 05:51:58 DEBUG "data": { 05:51:58 DEBUG "instance": "10.142.0.107:9100" 05:51:58 DEBUG "resultType": "vector", 05:51:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:51:58 DEBUG }, 05:51:58 DEBUG "result": [ 05:51:58 DEBUG ----- output ----- 05:51:58 DEBUG "value": [ 05:51:58 DEBUG { 05:51:58 DEBUG { 05:51:58 DEBUG 05:51:58 DEBUG "metric": { 05:51:58 DEBUG "status": "success", 05:51:58 DEBUG 05:51:58 DEBUG "pod": "ds-cts-0" 05:51:58 DEBUG "data": { 05:51:58 DEBUG 05:51:58 DEBUG }, 05:51:58 DEBUG "resultType": "vector", 05:51:58 DEBUG [...] 05:51:58 DEBUG "value": [ 05:51:58 DEBUG "result": [ 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG { 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG "metric": { 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:51:58 DEBUG "instance": "10.142.0.43:9100" 05:51:58 DEBUG [...] 05:51:58 DEBUG }, 05:51:58 DEBUG }, 05:51:58 DEBUG 05:51:58 DEBUG "value": [ 05:51:58 DEBUG "value": [ 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG 1713502318, 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG "0.6919999999999983" 05:51:58 DEBUG "pod": "ds-cts-2" 05:51:58 DEBUG 05:51:58 DEBUG ] 05:51:58 DEBUG }, 05:51:58 DEBUG [...] 05:51:58 DEBUG } 05:51:58 DEBUG "value": [ 05:51:58 DEBUG 05:51:58 DEBUG ] 05:51:58 DEBUG 1713502318, 05:51:58 DEBUG 05:51:58 DEBUG } 05:51:58 DEBUG "29.933333333333334" 05:51:58 DEBUG 05:51:58 DEBUG } 05:51:58 DEBUG ] 05:51:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:51:58 INFO [http_cmd]: http status code OK 05:51:58 DEBUG } 05:51:58 DEBUG }, 05:51:58 DEBUG --- status code --- 05:51:58 DEBUG ] 05:51:58 DEBUG "value": [ 05:51:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:51:58 DEBUG } 05:51:58 DEBUG 1713502318, 05:51:58 DEBUG --- http response --- 05:51:58 DEBUG } 05:51:58 DEBUG "3426705408" 05:51:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:51:58 DEBUG ] 05:51:58 DEBUG ----- output ----- 05:51:58 DEBUG } 05:51:58 DEBUG { 05:51:58 DEBUG ] 05:51:58 DEBUG "status": "success", 05:51:58 DEBUG } 05:51:58 DEBUG "data": { 05:51:58 DEBUG } 05:51:58 DEBUG "resultType": "vector", 05:51:58 DEBUG "result": [ 05:51:58 DEBUG { 05:51:58 DEBUG "metric": { 05:51:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:51:58 DEBUG }, 05:51:58 DEBUG "value": [ 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG [...] 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG "pod": "ds-idrepo-2" 05:51:58 DEBUG }, 05:51:58 DEBUG "value": [ 05:51:58 DEBUG 1713502318, 05:51:58 DEBUG "0" 05:51:58 DEBUG ] 05:51:58 DEBUG } 05:51:58 DEBUG ] 05:51:58 DEBUG } 05:51:58 DEBUG } 05:51:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502318" 05:51:58 INFO [http_cmd]: http status code OK 05:51:58 DEBUG --- status code --- 05:51:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:51:58 DEBUG --- http response --- 05:51:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:51:58 DEBUG ----- output ----- 05:51:58 DEBUG { 05:51:58 DEBUG "status": "success", 05:51:58 DEBUG "data": { 05:51:58 DEBUG "resultType": "vector", 05:51:58 DEBUG "result": [ 05:51:58 DEBUG { 05:51:58 DEBUG "metric": { 05:51:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:51:58 DEBUG }, 05:51:58 DEBUG "value": [ 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG [...] 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG "pod": "ds-idrepo-2" 05:51:58 DEBUG }, 05:51:58 DEBUG "value": [ 05:51:58 DEBUG 1713502318, 05:51:58 DEBUG "4092.5469135416993" 05:51:58 DEBUG ] 05:51:58 DEBUG } 05:51:58 DEBUG ] 05:51:58 DEBUG } 05:51:58 DEBUG } 05:51:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502318" 05:51:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713502318" 05:51:58 INFO [http_cmd]: http status code OK 05:51:58 DEBUG --- status code --- 05:51:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:51:58 DEBUG --- http response --- 05:51:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:51:58 DEBUG ----- output ----- 05:51:58 DEBUG { 05:51:58 DEBUG "status": "success", 05:51:58 DEBUG "data": { 05:51:58 DEBUG "resultType": "vector", 05:51:58 DEBUG "result": [ 05:51:58 DEBUG { 05:51:58 DEBUG "metric": { 05:51:58 DEBUG "pod": "ds-cts-0" 05:51:58 DEBUG }, 05:51:58 DEBUG "value": [ 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG [...] 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG "pod": "ds-cts-2" 05:51:58 DEBUG }, 05:51:58 DEBUG "value": [ 05:51:58 DEBUG 1713502318, 05:51:58 DEBUG "0.00011063999999999997" 05:51:58 DEBUG ] 05:51:58 DEBUG } 05:51:58 DEBUG ] 05:51:58 DEBUG } 05:51:58 DEBUG } 05:51:58 INFO [http_cmd]: http status code OK 05:51:58 DEBUG --- status code --- 05:51:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:51:58 DEBUG --- http response --- 05:51:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:51:58 DEBUG ----- output ----- 05:51:58 DEBUG { 05:51:58 DEBUG "status": "success", 05:51:58 DEBUG "data": { 05:51:58 DEBUG "resultType": "vector", 05:51:58 DEBUG "result": [ 05:51:58 DEBUG { 05:51:58 DEBUG "metric": { 05:51:58 DEBUG "instance": "10.142.0.107:9100" 05:51:58 DEBUG }, 05:51:58 DEBUG "value": [ 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG [...] 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG "instance": "10.142.0.43:9100" 05:51:58 DEBUG }, 05:51:58 DEBUG "value": [ 05:51:58 DEBUG 1713502318, 05:51:58 DEBUG "14155611.233333334" 05:51:58 DEBUG ] 05:51:58 DEBUG } 05:51:58 DEBUG ] 05:51:58 DEBUG } 05:51:58 DEBUG } 05:51:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713502318" 05:51:58 INFO [http_cmd]: http status code OK 05:51:58 DEBUG --- status code --- 05:51:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:51:58 DEBUG --- http response --- 05:51:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:51:58 DEBUG ----- output ----- 05:51:58 DEBUG { 05:51:58 DEBUG "status": "success", 05:51:58 DEBUG "data": { 05:51:58 DEBUG "resultType": "vector", 05:51:58 DEBUG "result": [ 05:51:58 DEBUG { 05:51:58 DEBUG "metric": { 05:51:58 DEBUG "instance": "10.142.0.107:9100" 05:51:58 DEBUG }, 05:51:58 DEBUG "value": [ 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG [...] 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG "instance": "10.142.0.43:9100" 05:51:58 DEBUG }, 05:51:58 DEBUG "value": [ 05:51:58 DEBUG 1713502318, 05:51:58 DEBUG "14163220.6" 05:51:58 DEBUG ] 05:51:58 DEBUG } 05:51:58 DEBUG ] 05:51:58 DEBUG } 05:51:58 DEBUG } 05:51:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713502318" 05:51:58 INFO [http_cmd]: http status code OK 05:51:58 DEBUG --- status code --- 05:51:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:51:58 DEBUG --- http response --- 05:51:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:51:58 DEBUG ----- output ----- 05:51:58 DEBUG { 05:51:58 DEBUG "status": "success", 05:51:58 DEBUG "data": { 05:51:58 DEBUG "resultType": "vector", 05:51:58 DEBUG "result": [ 05:51:58 DEBUG { 05:51:58 DEBUG "metric": { 05:51:58 DEBUG "instance": "10.142.0.107:9100" 05:51:58 DEBUG }, 05:51:58 DEBUG "value": [ 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG [...] 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG 05:51:58 DEBUG "instance": "10.142.0.43:9100" 05:51:58 DEBUG }, 05:51:58 DEBUG "value": [ 05:51:58 DEBUG 1713502318, 05:51:58 DEBUG "0.669599999999997" 05:51:58 DEBUG ] 05:51:58 DEBUG } 05:51:58 DEBUG ] 05:51:58 DEBUG } 05:51:58 DEBUG } 05:52:01 INFO 05:52:01 INFO [loop_until]: kubectl --namespace=xlou top pods 05:52:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:52:01 INFO [loop_until]: OK (rc = 0) 05:52:01 DEBUG --- stdout --- 05:52:01 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 2047m 9115Mi am-6fd554f8f7-hvvkw 2078m 8884Mi am-6fd554f8f7-pmj4b 2337m 8765Mi ds-cts-0 8952m 13272Mi ds-cts-1 8123m 12916Mi ds-cts-2 8885m 12991Mi ds-idrepo-0 591m 3297Mi ds-idrepo-1 612m 3212Mi ds-idrepo-2 596m 3092Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 3m 1216Mi lodemon-6b89d774d5-rf7sk 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 1018m 3609Mi 05:52:01 DEBUG --- stderr --- 05:52:01 DEBUG 05:52:02 INFO 05:52:02 INFO [loop_until]: kubectl --namespace=xlou top node 05:52:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:52:02 INFO [loop_until]: OK (rc = 0) 05:52:02 DEBUG --- stdout --- 05:52:02 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 71m 0% 6247Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 685m 2% 6741Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 2078m 6% 11643Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 82m 0% 3268Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 8784m 29% 15911Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 151m 0% 5157Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 2214m 7% 11928Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 80m 0% 2689Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 2469m 8% 10998Mi 9% gke-xlou-cdm-ds-0f601751-j488 681m 2% 6206Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 8015m 26% 17267Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 680m 2% 6905Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 9015m 30% 14980Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 1117m 7% 7234Mi 12% 05:52:02 DEBUG --- stderr --- 05:52:02 DEBUG 05:52:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502378" 05:52:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502378" 05:52:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713502378" 05:52:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502378" 05:52:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713502378" 05:52:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502378" 05:52:58 INFO [http_cmd]: http status code OK 05:52:58 INFO [http_cmd]: http status code OK 05:52:58 DEBUG --- status code --- 05:52:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502378" 05:52:58 DEBUG --- status code --- 05:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:52:58 DEBUG --- http response --- 05:52:58 ERROR [http_cmd]: ERROR 05:52:58 DEBUG --- http response --- 05:52:58 INFO [http_cmd]: http status code OK 05:52:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:52:58 ERROR --- status code --- 05:52:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:52:58 DEBUG --- status code --- 05:52:58 DEBUG ----- output ----- 05:52:58 ERROR http status code is 400 (expected [200, 401, 403, 404]) 05:52:58 DEBUG ----- output ----- 05:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:52:58 DEBUG { 05:52:58 ERROR --- http response --- 05:52:58 DEBUG { 05:52:58 DEBUG --- http response --- 05:52:58 DEBUG "status": "success", 05:52:58 DEBUG "status": "success", 05:52:58 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 05:52:58 DEBUG "data": { 05:52:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:52:58 DEBUG "data": { 05:52:58 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 05:52:58 DEBUG "resultType": "vector", 05:52:58 DEBUG ----- output ----- 05:52:58 DEBUG "resultType": "vector", 05:52:58 DEBUG "result": [ 05:52:58 DEBUG { 05:52:58 DEBUG "result": [ 05:52:58 DEBUG { 05:52:58 DEBUG "status": "success", 05:52:58 DEBUG { 05:52:58 DEBUG "metric": { 05:52:58 DEBUG "data": { 05:52:58 DEBUG "metric": { 05:52:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:52:58 DEBUG "resultType": "vector", 05:52:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:52:58 DEBUG }, 05:52:58 DEBUG "result": [ 05:52:58 DEBUG }, 05:52:58 INFO [http_cmd]: http status code OK 05:52:58 DEBUG "value": [ 05:52:58 DEBUG { 05:52:58 DEBUG "value": [ 05:52:58 DEBUG --- status code --- 05:52:58 DEBUG 05:52:58 DEBUG "metric": { 05:52:58 DEBUG 05:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:52:58 DEBUG 05:52:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:52:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713502378" 05:52:58 DEBUG 05:52:58 DEBUG --- http response --- 05:52:58 DEBUG 05:52:58 DEBUG }, 05:52:58 DEBUG 05:52:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:52:58 DEBUG [...] 05:52:58 INFO [http_cmd]: http status code OK 05:52:58 DEBUG "value": [ 05:52:58 DEBUG [...] 05:52:58 DEBUG ----- output ----- 05:52:58 DEBUG 05:52:58 DEBUG --- status code --- 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG { 05:52:58 DEBUG 05:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG "status": "success", 05:52:58 DEBUG 05:52:58 DEBUG --- http response --- 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG "data": { 05:52:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:52:58 DEBUG [...] 05:52:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:52:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:52:58 DEBUG "resultType": "vector", 05:52:58 DEBUG }, 05:52:58 DEBUG 05:52:58 DEBUG ----- output ----- 05:52:58 DEBUG }, 05:52:58 DEBUG "result": [ 05:52:58 DEBUG "value": [ 05:52:58 DEBUG 05:52:58 DEBUG { 05:52:58 DEBUG "value": [ 05:52:58 DEBUG { 05:52:58 DEBUG 1713502378, 05:52:58 DEBUG 05:52:58 DEBUG "status": "success", 05:52:58 DEBUG 1713502378, 05:52:58 DEBUG "metric": { 05:52:58 DEBUG "0" 05:52:58 DEBUG "pod": "ds-cts-0" 05:52:58 DEBUG "data": { 05:52:58 DEBUG "0" 05:52:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:52:58 DEBUG ] 05:52:58 DEBUG }, 05:52:58 DEBUG "resultType": "vector", 05:52:58 DEBUG ] 05:52:58 DEBUG }, 05:52:58 DEBUG } 05:52:58 DEBUG "value": [ 05:52:58 DEBUG "result": [ 05:52:58 DEBUG } 05:52:58 DEBUG "value": [ 05:52:58 DEBUG ] 05:52:58 DEBUG 1713502378, 05:52:58 DEBUG { 05:52:58 DEBUG ] 05:52:58 DEBUG 05:52:58 DEBUG } 05:52:58 DEBUG "356396277.78760934" 05:52:58 DEBUG "metric": { 05:52:58 DEBUG } 05:52:58 DEBUG 05:52:58 DEBUG } 05:52:58 DEBUG ] 05:52:58 DEBUG "instance": "10.142.0.107:9100" 05:52:58 DEBUG } 05:52:58 DEBUG 05:52:58 DEBUG } 05:52:58 DEBUG }, 05:52:58 DEBUG [...] 05:52:58 DEBUG ] 05:52:58 DEBUG "value": [ 05:52:58 DEBUG 05:52:58 DEBUG } 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG } 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:52:58 DEBUG [...] 05:52:58 DEBUG }, 05:52:58 DEBUG 05:52:58 DEBUG "value": [ 05:52:58 DEBUG 05:52:58 DEBUG 1713502378, 05:52:58 DEBUG 05:52:58 DEBUG "3068892.5" 05:52:58 DEBUG "instance": "10.142.0.43:9100" 05:52:58 DEBUG ] 05:52:58 DEBUG }, 05:52:58 DEBUG } 05:52:58 DEBUG "value": [ 05:52:58 DEBUG ] 05:52:58 DEBUG 1713502378, 05:52:58 DEBUG } 05:52:58 DEBUG "577371950080" 05:52:58 DEBUG } 05:52:58 DEBUG ] 05:52:58 DEBUG } 05:52:58 DEBUG ] 05:52:58 DEBUG } 05:52:58 DEBUG } 05:52:58 INFO [http_cmd]: http status code OK 05:52:58 DEBUG --- status code --- 05:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:52:58 DEBUG --- http response --- 05:52:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:52:58 DEBUG ----- output ----- 05:52:58 DEBUG { 05:52:58 DEBUG "status": "success", 05:52:58 DEBUG "data": { 05:52:58 DEBUG "resultType": "vector", 05:52:58 DEBUG "result": [ 05:52:58 DEBUG { 05:52:58 DEBUG "metric": { 05:52:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:52:58 DEBUG }, 05:52:58 DEBUG "value": [ 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG [...] 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:52:58 DEBUG }, 05:52:58 DEBUG "value": [ 05:52:58 DEBUG 1713502378, 05:52:58 DEBUG "0" 05:52:58 DEBUG ] 05:52:58 DEBUG } 05:52:58 DEBUG ] 05:52:58 DEBUG } 05:52:58 DEBUG } 05:52:58 INFO [http_cmd]: http status code OK 05:52:58 DEBUG --- status code --- 05:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:52:58 DEBUG --- http response --- 05:52:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:52:58 DEBUG ----- output ----- 05:52:58 DEBUG { 05:52:58 DEBUG "status": "success", 05:52:58 DEBUG "data": { 05:52:58 DEBUG "resultType": "vector", 05:52:58 DEBUG "result": [ 05:52:58 DEBUG { 05:52:58 DEBUG "metric": { 05:52:58 DEBUG "pod": "ds-idrepo-0" 05:52:58 DEBUG }, 05:52:58 DEBUG "value": [ 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG [...] 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG "pod": "ds-cts-2" 05:52:58 DEBUG }, 05:52:58 DEBUG "value": [ 05:52:58 DEBUG 1713502378, 05:52:58 DEBUG "0" 05:52:58 DEBUG ] 05:52:58 DEBUG } 05:52:58 DEBUG ] 05:52:58 DEBUG } 05:52:58 DEBUG } 05:52:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713502378" 05:52:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502378" 05:52:58 INFO [http_cmd]: http status code OK 05:52:58 DEBUG --- status code --- 05:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:52:58 DEBUG --- http response --- 05:52:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:52:58 DEBUG ----- output ----- 05:52:58 DEBUG { 05:52:58 DEBUG "status": "success", 05:52:58 DEBUG "data": { 05:52:58 DEBUG "resultType": "vector", 05:52:58 DEBUG "result": [ 05:52:58 DEBUG { 05:52:58 DEBUG "metric": { 05:52:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:52:58 DEBUG }, 05:52:58 DEBUG "value": [ 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG [...] 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:52:58 DEBUG }, 05:52:58 DEBUG "value": [ 05:52:58 DEBUG 1713502378, 05:52:58 DEBUG "55.176" 05:52:58 DEBUG ] 05:52:58 DEBUG } 05:52:58 DEBUG ] 05:52:58 DEBUG } 05:52:58 DEBUG } 05:52:58 INFO [http_cmd]: http status code OK 05:52:58 DEBUG --- status code --- 05:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:52:58 DEBUG --- http response --- 05:52:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 05:52:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502378" 05:52:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502378" 05:52:58 INFO [http_cmd]: http status code OK 05:52:58 DEBUG --- status code --- 05:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:52:58 DEBUG --- http response --- 05:52:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:52:58 DEBUG ----- output ----- 05:52:58 DEBUG { 05:52:58 DEBUG "status": "success", 05:52:58 DEBUG "data": { 05:52:58 DEBUG "resultType": "vector", 05:52:58 DEBUG "result": [ 05:52:58 DEBUG { 05:52:58 DEBUG "metric": { 05:52:58 DEBUG "pod": "ds-cts-0" 05:52:58 DEBUG }, 05:52:58 DEBUG "value": [ 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG [...] 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG "pod": "ds-cts-2" 05:52:58 DEBUG }, 05:52:58 DEBUG "value": [ 05:52:58 DEBUG 1713502378, 05:52:58 DEBUG "296.06666666666666" 05:52:58 DEBUG ] 05:52:58 DEBUG } 05:52:58 DEBUG ] 05:52:58 DEBUG } 05:52:58 DEBUG } 05:52:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502378" 05:52:58 INFO [http_cmd]: http status code OK 05:52:58 DEBUG --- status code --- 05:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:52:58 DEBUG --- http response --- 05:52:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:52:58 DEBUG ----- output ----- 05:52:58 DEBUG { 05:52:58 DEBUG "status": "success", 05:52:58 DEBUG "data": { 05:52:58 DEBUG "resultType": "vector", 05:52:58 DEBUG "result": [ 05:52:58 DEBUG { 05:52:58 DEBUG "metric": { 05:52:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:52:58 DEBUG }, 05:52:58 DEBUG "value": [ 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG [...] 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:52:58 DEBUG }, 05:52:58 DEBUG "value": [ 05:52:58 DEBUG 1713502378, 05:52:58 DEBUG "0" 05:52:58 DEBUG ] 05:52:58 DEBUG } 05:52:58 DEBUG ] 05:52:58 DEBUG } 05:52:58 DEBUG } 05:52:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502378" 05:52:58 INFO [http_cmd]: http status code OK 05:52:58 DEBUG --- status code --- 05:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:52:58 DEBUG --- http response --- 05:52:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:52:58 DEBUG ----- output ----- 05:52:58 DEBUG { 05:52:58 DEBUG "status": "success", 05:52:58 DEBUG "data": { 05:52:58 DEBUG "resultType": "vector", 05:52:58 DEBUG "result": [ 05:52:58 DEBUG { 05:52:58 DEBUG "metric": { 05:52:58 DEBUG "pod": "ds-idrepo-0" 05:52:58 DEBUG }, 05:52:58 DEBUG "value": [ 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG [...] 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG "pod": "ds-idrepo-2" 05:52:58 DEBUG }, 05:52:58 DEBUG "value": [ 05:52:58 DEBUG 1713502378, 05:52:58 DEBUG "0" 05:52:58 DEBUG ] 05:52:58 DEBUG } 05:52:58 DEBUG ] 05:52:58 DEBUG } 05:52:58 DEBUG } 05:52:58 INFO [http_cmd]: http status code OK 05:52:58 DEBUG --- status code --- 05:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:52:58 DEBUG --- http response --- 05:52:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:52:58 DEBUG ----- output ----- 05:52:58 DEBUG { 05:52:58 DEBUG "status": "success", 05:52:58 DEBUG "data": { 05:52:58 DEBUG "resultType": "vector", 05:52:58 DEBUG "result": [ 05:52:58 DEBUG { 05:52:58 DEBUG "metric": { 05:52:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:52:58 DEBUG }, 05:52:58 DEBUG "value": [ 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG [...] 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG "pod": "ds-cts-0" 05:52:58 DEBUG }, 05:52:58 DEBUG "value": [ 05:52:58 DEBUG 1713502378, 05:52:58 DEBUG "6.215001037783689" 05:52:58 DEBUG ] 05:52:58 DEBUG } 05:52:58 DEBUG ] 05:52:58 DEBUG } 05:52:58 DEBUG } 05:52:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713502378" 05:52:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502378" 05:52:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713502378" 05:52:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502378" 05:52:58 INFO [http_cmd]: http status code OK 05:52:58 DEBUG --- status code --- 05:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:52:58 DEBUG --- http response --- 05:52:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:52:58 DEBUG ----- output ----- 05:52:58 DEBUG { 05:52:58 DEBUG "status": "success", 05:52:58 DEBUG "data": { 05:52:58 DEBUG "resultType": "vector", 05:52:58 DEBUG "result": [ 05:52:58 DEBUG { 05:52:58 DEBUG "metric": { 05:52:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:52:58 DEBUG }, 05:52:58 DEBUG "value": [ 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG [...] 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:52:58 DEBUG }, 05:52:58 DEBUG "value": [ 05:52:58 DEBUG 1713502378, 05:52:58 DEBUG "1.0333557263417146" 05:52:58 DEBUG ] 05:52:58 DEBUG } 05:52:58 DEBUG ] 05:52:58 DEBUG } 05:52:58 DEBUG } 05:52:58 INFO [http_cmd]: http status code OK 05:52:58 DEBUG --- status code --- 05:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:52:58 DEBUG --- http response --- 05:52:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:52:58 DEBUG ----- output ----- 05:52:58 DEBUG { 05:52:58 DEBUG "status": "success", 05:52:58 DEBUG "data": { 05:52:58 DEBUG "resultType": "vector", 05:52:58 DEBUG "result": [ 05:52:58 DEBUG { 05:52:58 DEBUG "metric": { 05:52:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:52:58 DEBUG }, 05:52:58 DEBUG "value": [ 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG [...] 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG "pod": "ds-cts-0" 05:52:58 DEBUG }, 05:52:58 DEBUG "value": [ 05:52:58 DEBUG 1713502378, 05:52:58 DEBUG "36838092.17139349" 05:52:58 DEBUG ] 05:52:58 DEBUG } 05:52:58 DEBUG ] 05:52:58 DEBUG } 05:52:58 DEBUG } 05:52:58 INFO [http_cmd]: http status code OK 05:52:58 DEBUG --- status code --- 05:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:52:58 DEBUG --- http response --- 05:52:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:52:58 DEBUG ----- output ----- 05:52:58 DEBUG { 05:52:58 DEBUG "status": "success", 05:52:58 DEBUG "data": { 05:52:58 DEBUG "resultType": "vector", 05:52:58 DEBUG "result": [ 05:52:58 DEBUG { 05:52:58 DEBUG "metric": { 05:52:58 DEBUG "pod": "ds-idrepo-0" 05:52:58 DEBUG }, 05:52:58 DEBUG "value": [ 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG [...] 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG "pod": "ds-idrepo-2" 05:52:58 DEBUG }, 05:52:58 DEBUG "value": [ 05:52:58 DEBUG 1713502378, 05:52:58 DEBUG "0" 05:52:58 DEBUG ] 05:52:58 DEBUG } 05:52:58 DEBUG ] 05:52:58 DEBUG } 05:52:58 DEBUG } 05:52:58 INFO [http_cmd]: http status code OK 05:52:58 DEBUG --- status code --- 05:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:52:58 DEBUG --- http response --- 05:52:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:52:58 DEBUG ----- output ----- 05:52:58 DEBUG { 05:52:58 DEBUG "status": "success", 05:52:58 DEBUG "data": { 05:52:58 DEBUG "resultType": "vector", 05:52:58 DEBUG "result": [ 05:52:58 DEBUG { 05:52:58 DEBUG "metric": { 05:52:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 05:52:58 DEBUG }, 05:52:58 DEBUG "value": [ 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG [...] 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 05:52:58 DEBUG }, 05:52:58 DEBUG "value": [ 05:52:58 DEBUG 1713502378, 05:52:58 DEBUG "4302585856" 05:52:58 DEBUG ] 05:52:58 DEBUG } 05:52:58 DEBUG ] 05:52:58 DEBUG } 05:52:58 DEBUG } 05:52:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502378" 05:52:58 INFO [http_cmd]: http status code OK 05:52:58 DEBUG --- status code --- 05:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:52:58 DEBUG --- http response --- 05:52:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:52:58 DEBUG ----- output ----- 05:52:58 DEBUG { 05:52:58 DEBUG "status": "success", 05:52:58 DEBUG "data": { 05:52:58 DEBUG "resultType": "vector", 05:52:58 DEBUG "result": [ 05:52:58 DEBUG { 05:52:58 DEBUG "metric": { 05:52:58 DEBUG "pod": "ds-cts-0" 05:52:58 DEBUG }, 05:52:58 DEBUG "value": [ 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG [...] 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG "pod": "ds-cts-2" 05:52:58 DEBUG }, 05:52:58 DEBUG "value": [ 05:52:58 DEBUG 1713502378, 05:52:58 DEBUG "30" 05:52:58 DEBUG ] 05:52:58 DEBUG } 05:52:58 DEBUG ] 05:52:58 DEBUG } 05:52:58 DEBUG } 05:52:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502378" 05:52:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713502378" 05:52:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502378" 05:52:58 INFO [http_cmd]: http status code OK 05:52:58 INFO [http_cmd]: http status code OK 05:52:58 DEBUG --- status code --- 05:52:58 DEBUG --- status code --- 05:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:52:58 DEBUG --- http response --- 05:52:58 DEBUG --- http response --- 05:52:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:52:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713502378" 05:52:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:52:58 DEBUG ----- output ----- 05:52:58 DEBUG ----- output ----- 05:52:58 DEBUG { 05:52:58 DEBUG { 05:52:58 DEBUG "status": "success", 05:52:58 DEBUG "status": "success", 05:52:58 DEBUG "data": { 05:52:58 DEBUG "data": { 05:52:58 DEBUG "resultType": "vector", 05:52:58 DEBUG "resultType": "vector", 05:52:58 DEBUG "result": [ 05:52:58 DEBUG "result": [ 05:52:58 DEBUG { 05:52:58 DEBUG { 05:52:58 DEBUG "metric": { 05:52:58 DEBUG "metric": { 05:52:58 DEBUG "instance": "10.142.0.107:9100" 05:52:58 DEBUG "pod": "am-6fd554f8f7-hvvkw" 05:52:58 DEBUG }, 05:52:58 DEBUG }, 05:52:58 DEBUG "value": [ 05:52:58 DEBUG 05:52:58 DEBUG "value": [ 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG [...] 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG [...] 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG "instance": "10.142.0.43:9100" 05:52:58 DEBUG 05:52:58 DEBUG }, 05:52:58 DEBUG "pod": "am-6fd554f8f7-pmj4b" 05:52:58 DEBUG "value": [ 05:52:58 DEBUG }, 05:52:58 DEBUG 1713502378, 05:52:58 DEBUG "value": [ 05:52:58 DEBUG "14202742.5" 05:52:58 DEBUG 1713502378, 05:52:58 DEBUG ] 05:52:58 DEBUG "2173.1" 05:52:58 DEBUG } 05:52:58 DEBUG ] 05:52:58 DEBUG ] 05:52:58 DEBUG } 05:52:58 DEBUG } 05:52:58 DEBUG ] 05:52:58 DEBUG } 05:52:58 DEBUG } 05:52:58 DEBUG } 05:52:58 INFO [http_cmd]: http status code OK 05:52:58 DEBUG --- status code --- 05:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:52:58 DEBUG --- http response --- 05:52:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 05:52:58 DEBUG ----- output ----- 05:52:58 DEBUG { 05:52:58 DEBUG "status": "success", 05:52:58 DEBUG "data": { 05:52:58 DEBUG "resultType": "vector", 05:52:58 DEBUG "result": [ 05:52:58 DEBUG { 05:52:58 DEBUG "metric": { 05:52:58 DEBUG "pod": "ds-idrepo-0" 05:52:58 DEBUG }, 05:52:58 DEBUG "value": [ 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG [...] 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG "pod": "ds-idrepo-2" 05:52:58 DEBUG }, 05:52:58 DEBUG "value": [ 05:52:58 DEBUG 1713502378, 05:52:58 DEBUG "0" 05:52:58 DEBUG ] 05:52:58 DEBUG } 05:52:58 DEBUG ] 05:52:58 DEBUG } 05:52:58 DEBUG } 05:52:58 INFO [http_cmd]: http status code OK 05:52:58 DEBUG --- status code --- 05:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:52:58 DEBUG --- http response --- 05:52:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:52:58 DEBUG ----- output ----- 05:52:58 DEBUG { 05:52:58 DEBUG "status": "success", 05:52:58 DEBUG "data": { 05:52:58 DEBUG "resultType": "vector", 05:52:58 DEBUG "result": [ 05:52:58 DEBUG { 05:52:58 DEBUG "metric": { 05:52:58 DEBUG "instance": "10.142.0.107:9100" 05:52:58 DEBUG }, 05:52:58 DEBUG "value": [ 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG [...] 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG "instance": "10.142.0.43:9100" 05:52:58 DEBUG }, 05:52:58 DEBUG "value": [ 05:52:58 DEBUG 1713502378, 05:52:58 DEBUG "0.7593333333333329" 05:52:58 DEBUG ] 05:52:58 DEBUG } 05:52:58 DEBUG ] 05:52:58 DEBUG } 05:52:58 DEBUG } 05:52:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713502378" 05:52:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502378" 05:52:58 INFO [http_cmd]: http status code OK 05:52:58 INFO [http_cmd]: http status code OK 05:52:58 DEBUG --- status code --- 05:52:58 DEBUG --- status code --- 05:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:52:58 DEBUG --- http response --- 05:52:58 DEBUG --- http response --- 05:52:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:52:58 DEBUG ----- output ----- 05:52:58 DEBUG { 05:52:58 DEBUG "status": "success", 05:52:58 DEBUG "data": { 05:52:58 DEBUG "resultType": "vector", 05:52:58 DEBUG "result": [ 05:52:58 DEBUG { 05:52:58 DEBUG "metric": { 05:52:58 DEBUG "instance": "10.142.0.107:9100" 05:52:58 DEBUG }, 05:52:58 DEBUG "value": [ 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG [...] 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG "instance": "10.142.0.43:9100" 05:52:58 DEBUG }, 05:52:58 DEBUG "value": [ 05:52:58 DEBUG 1713502378, 05:52:58 DEBUG "0.62553333333335" 05:52:58 DEBUG ] 05:52:58 DEBUG } 05:52:58 DEBUG ] 05:52:58 DEBUG } 05:52:58 DEBUG } 05:52:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 05:52:58 DEBUG ----- output ----- 05:52:58 DEBUG { 05:52:58 DEBUG "status": "success", 05:52:58 DEBUG "data": { 05:52:58 DEBUG "resultType": "vector", 05:52:58 DEBUG "result": [ 05:52:58 DEBUG { 05:52:58 DEBUG "metric": { 05:52:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 05:52:58 DEBUG }, 05:52:58 DEBUG "value": [ 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG [...] 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG "pod": "ds-cts-0" 05:52:58 DEBUG }, 05:52:58 DEBUG "value": [ 05:52:58 DEBUG 1713502378, 05:52:58 DEBUG "11107325.060340157" 05:52:58 DEBUG ] 05:52:58 DEBUG } 05:52:58 DEBUG ] 05:52:58 DEBUG } 05:52:58 DEBUG } 05:52:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713502378" 05:52:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713502378" 05:52:58 INFO [http_cmd]: http status code OK 05:52:58 DEBUG --- status code --- 05:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:52:58 DEBUG --- http response --- 05:52:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 05:52:58 DEBUG ----- output ----- 05:52:58 DEBUG { 05:52:58 DEBUG "status": "success", 05:52:58 DEBUG "data": { 05:52:58 DEBUG "resultType": "vector", 05:52:58 DEBUG "result": [ 05:52:58 DEBUG { 05:52:58 DEBUG "metric": { 05:52:58 DEBUG "pod": "ds-cts-0" 05:52:58 DEBUG }, 05:52:58 DEBUG "value": [ 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG [...] 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG "pod": "ds-cts-2" 05:52:58 DEBUG }, 05:52:58 DEBUG "value": [ 05:52:58 DEBUG 1713502378, 05:52:58 DEBUG "0.00031596" 05:52:58 DEBUG ] 05:52:58 DEBUG } 05:52:58 DEBUG ] 05:52:58 DEBUG } 05:52:58 DEBUG } 05:52:58 INFO [http_cmd]: http status code OK 05:52:58 DEBUG --- status code --- 05:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 05:52:58 DEBUG --- http response --- 05:52:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 05:52:58 DEBUG ----- output ----- 05:52:58 DEBUG { 05:52:58 DEBUG "status": "success", 05:52:58 DEBUG "data": { 05:52:58 DEBUG "resultType": "vector", 05:52:58 DEBUG "result": [ 05:52:58 DEBUG { 05:52:58 DEBUG "metric": { 05:52:58 DEBUG "instance": "10.142.0.107:9100" 05:52:58 DEBUG }, 05:52:58 DEBUG "value": [ 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG [...] 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG 05:52:58 DEBUG "instance": "10.142.0.43:9100" 05:52:58 DEBUG }, 05:52:58 DEBUG "value": [ 05:52:58 DEBUG 1713502378, 05:52:58 DEBUG "14212062.133333333" 05:52:58 DEBUG ] 05:52:58 DEBUG } 05:52:58 DEBUG ] 05:52:58 DEBUG } 05:52:58 DEBUG } 05:53:01 INFO 05:53:01 INFO [loop_until]: kubectl --namespace=xlou top pods 05:53:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:53:01 INFO [loop_until]: OK (rc = 0) 05:53:01 DEBUG --- stdout --- 05:53:01 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-6rhpf 2109m 9113Mi am-6fd554f8f7-hvvkw 2043m 8885Mi am-6fd554f8f7-pmj4b 2205m 8768Mi ds-cts-0 8873m 12795Mi ds-cts-1 8690m 12601Mi ds-cts-2 8251m 13737Mi ds-idrepo-0 614m 3297Mi ds-idrepo-1 619m 3212Mi ds-idrepo-2 588m 3093Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1172Mi idm-b85f9d978-wnhxb 3m 1216Mi lodemon-6b89d774d5-rf7sk 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 998m 4279Mi 05:53:01 DEBUG --- stderr --- 05:53:01 DEBUG 05:53:02 INFO 05:53:02 INFO [loop_until]: kubectl --namespace=xlou top node 05:53:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 05:53:02 INFO [loop_until]: OK (rc = 0) 05:53:02 DEBUG --- stdout --- 05:53:02 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 71m 0% 6242Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 686m 2% 6739Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 2087m 6% 11650Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 88m 0% 3257Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 8618m 28% 15330Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 152m 0% 5144Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 2216m 7% 11919Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 79m 0% 2686Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 2246m 7% 11000Mi 9% gke-xlou-cdm-ds-0f601751-j488 657m 2% 6197Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 8260m 27% 16869Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 704m 2% 6905Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 8377m 28% 15892Mi 14% gke-xlou-cdm-frontend-a8771548-i82n 1078m 6% 8050Mi 13% 05:53:02 DEBUG --- stderr --- 05:53:02 DEBUG 05:53:27 INFO Finished: True 05:53:27 INFO Waiting for threads to register finish flag 05:54:02 INFO Done. Have a nice day! :) 127.0.0.1 - - [19/Apr/2024 05:54:02] "GET /monitoring/stop HTTP/1.1" 200 - 05:54:04 WARNING File /tmp/lodemon_data-20240419_044251/node_disk_read_bytes_total.json does not exist. Skipping graph generation... WARNING:bokeh.core.validation.check:W-1000 (MISSING_RENDERERS): Plot has no renderers: figure(id='p6521', ...) 127.0.0.1 - - [19/Apr/2024 05:54:09] "GET /monitoring/process HTTP/1.1" 200 -