==================================================================================================== ========================================= Pod describe ========================================= ==================================================================================================== Name: lodemon-7dd48676f6-tdxl9 Namespace: xlou Priority: 0 Service Account: k8s-svc-acct-lodemon Node: gke-xlou-cdm-ds-0f601751-mc5d/10.142.0.43 Start Time: Sat, 20 Apr 2024 20:39:50 +0000 Labels: app=lodemon app.kubernetes.io/name=lodemon pod-template-hash=7dd48676f6 skaffold.dev/run-id=cf3af129-4aff-47bd-bd52-736e26585b79 Annotations: Status: Running IP: 10.106.46.173 IPs: IP: 10.106.46.173 Controlled By: ReplicaSet/lodemon-7dd48676f6 Containers: lodemon: Container ID: containerd://44bac723133568bce6e7277dcb17c0508d4f02dc4e1c8c9a5f9040acaa392bbd Image: gcr.io/engineeringpit/lodestar-images/lodestarbox:master-stable Image ID: gcr.io/engineeringpit/lodestar-images/lodestarbox@sha256:03f76e6326d713f4d73dc474e0e25fbbbaecc7c1d8c0590bfd94d68ac0d98c97 Port: 8080/TCP Host Port: 0/TCP Command: python3 Args: /lodestar/scripts/lodemon_run.py -W default State: Running Started: Sat, 20 Apr 2024 20:39:51 +0000 Ready: True Restart Count: 0 Limits: cpu: 2 memory: 2Gi Requests: cpu: 1 memory: 1Gi Liveness: exec [cat /tmp/lodemon_alive] delay=20s timeout=1s period=10s #success=1 #failure=3 Readiness: exec [cat /tmp/lodemon_alive] delay=20s timeout=1s period=10s #success=1 #failure=3 Environment: SKAFFOLD_PROFILE: medium Mounts: /lodestar/config/config.yaml from config (rw,path="config.yaml") /lodestar/config/deployments_to_monitor.yaml from deployments-config (rw,path="deployments_to_monitor.yaml") /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-m2l42 (ro) Conditions: Type Status Initialized True Ready True ContainersReady True PodScheduled True Volumes: config: Type: ConfigMap (a volume populated by a ConfigMap) Name: lodemon-config Optional: false deployments-config: Type: ConfigMap (a volume populated by a ConfigMap) Name: lodemon-deployments Optional: false kube-api-access-m2l42: Type: Projected (a volume that contains injected data from multiple sources) TokenExpirationSeconds: 3607 ConfigMapName: kube-root-ca.crt ConfigMapOptional: DownwardAPI: true QoS Class: Burstable Node-Selectors: Tolerations: node.kubernetes.io/not-ready:NoExecute op=Exists for 300s node.kubernetes.io/unreachable:NoExecute op=Exists for 300s Events: Type Reason Age From Message ---- ------ ---- ---- ------- Normal Scheduled 46m default-scheduler Successfully assigned xlou/lodemon-7dd48676f6-tdxl9 to gke-xlou-cdm-ds-0f601751-mc5d Normal Pulling 46m kubelet Pulling image "gcr.io/engineeringpit/lodestar-images/lodestarbox:master-stable" Normal Pulled 46m kubelet Successfully pulled image "gcr.io/engineeringpit/lodestar-images/lodestarbox:master-stable" in 240.431419ms (240.436288ms including waiting) Normal Created 46m kubelet Created container lodemon Normal Started 46m kubelet Started container lodemon ==================================================================================================== =========================================== Pod logs =========================================== ==================================================================================================== * Serving Flask app 'lodemon_run' * Debug mode: off WARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead. * Running on all addresses (0.0.0.0) * Running on http://127.0.0.1:8080 * Running on http://10.106.46.173:8080 Press CTRL+C to quit 21:40:48 INFO 21:40:48 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 21:40:48 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:40:48 INFO [loop_until]: OK (rc = 0) 21:40:48 DEBUG --- stdout --- 21:40:48 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 21:40:48 DEBUG --- stderr --- 21:40:48 DEBUG 21:40:48 INFO 21:40:48 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 21:40:48 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:40:48 INFO [loop_until]: OK (rc = 0) 21:40:48 DEBUG --- stdout --- 21:40:48 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 21:40:48 DEBUG --- stderr --- 21:40:48 DEBUG 21:40:48 INFO 21:40:48 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 21:40:48 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:40:48 INFO [loop_until]: OK (rc = 0) 21:40:48 DEBUG --- stdout --- 21:40:48 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 21:40:48 DEBUG --- stderr --- 21:40:48 DEBUG 21:40:48 INFO 21:40:48 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 21:40:48 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:40:48 INFO [loop_until]: OK (rc = 0) 21:40:48 DEBUG --- stdout --- 21:40:48 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 21:40:48 DEBUG --- stderr --- 21:40:48 DEBUG 21:40:49 INFO 21:40:49 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 21:40:49 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:40:49 INFO [loop_until]: OK (rc = 0) 21:40:49 DEBUG --- stdout --- 21:40:49 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 21:40:49 DEBUG --- stderr --- 21:40:49 DEBUG 21:40:49 INFO 21:40:49 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 21:40:49 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:40:49 INFO [loop_until]: OK (rc = 0) 21:40:49 DEBUG --- stdout --- 21:40:49 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 21:40:49 DEBUG --- stderr --- 21:40:49 DEBUG 21:40:49 INFO 21:40:49 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 21:40:49 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:40:49 INFO [loop_until]: OK (rc = 0) 21:40:49 DEBUG --- stdout --- 21:40:49 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 21:40:49 DEBUG --- stderr --- 21:40:49 DEBUG 21:40:49 INFO 21:40:49 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 21:40:49 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:40:49 INFO [loop_until]: OK (rc = 0) 21:40:49 DEBUG --- stdout --- 21:40:49 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 21:40:49 DEBUG --- stderr --- 21:40:49 DEBUG 21:40:49 INFO 21:40:49 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 21:40:49 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:40:49 INFO [loop_until]: OK (rc = 0) 21:40:49 DEBUG --- stdout --- 21:40:49 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 21:40:49 DEBUG --- stderr --- 21:40:49 DEBUG 21:40:49 INFO 21:40:49 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 21:40:49 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:40:49 INFO [loop_until]: OK (rc = 0) 21:40:49 DEBUG --- stdout --- 21:40:49 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 21:40:49 DEBUG --- stderr --- 21:40:49 DEBUG 21:40:49 INFO 21:40:49 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 21:40:49 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:40:49 INFO [loop_until]: OK (rc = 0) 21:40:49 DEBUG --- stdout --- 21:40:49 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 21:40:49 DEBUG --- stderr --- 21:40:49 DEBUG 21:40:49 INFO 21:40:49 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 21:40:49 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:40:50 INFO [loop_until]: OK (rc = 0) 21:40:50 DEBUG --- stdout --- 21:40:50 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 21:40:50 DEBUG --- stderr --- 21:40:50 DEBUG 21:40:50 INFO 21:40:50 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 21:40:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:40:50 INFO [loop_until]: OK (rc = 0) 21:40:50 DEBUG --- stdout --- 21:40:50 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 21:40:50 DEBUG --- stderr --- 21:40:50 DEBUG 21:40:50 INFO 21:40:50 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 21:40:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:40:50 INFO [loop_until]: OK (rc = 0) 21:40:50 DEBUG --- stdout --- 21:40:50 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 21:40:50 DEBUG --- stderr --- 21:40:50 DEBUG 21:40:50 INFO 21:40:50 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 21:40:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:40:50 INFO [loop_until]: OK (rc = 0) 21:40:50 DEBUG --- stdout --- 21:40:50 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 21:40:50 DEBUG --- stderr --- 21:40:50 DEBUG 21:40:50 INFO 21:40:50 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 21:40:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:40:50 INFO [loop_until]: OK (rc = 0) 21:40:50 DEBUG --- stdout --- 21:40:50 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 21:40:50 DEBUG --- stderr --- 21:40:50 DEBUG 21:40:50 INFO 21:40:50 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 21:40:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:40:50 INFO [loop_until]: OK (rc = 0) 21:40:50 DEBUG --- stdout --- 21:40:50 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 21:40:50 DEBUG --- stderr --- 21:40:50 DEBUG 21:40:50 INFO 21:40:50 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 21:40:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:40:50 INFO [loop_until]: OK (rc = 0) 21:40:50 DEBUG --- stdout --- 21:40:50 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 21:40:50 DEBUG --- stderr --- 21:40:50 DEBUG 21:40:50 INFO 21:40:50 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 21:40:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:40:50 INFO [loop_until]: OK (rc = 0) 21:40:50 DEBUG --- stdout --- 21:40:50 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 21:40:50 DEBUG --- stderr --- 21:40:50 DEBUG 21:40:51 INFO 21:40:51 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 21:40:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:40:51 INFO [loop_until]: OK (rc = 0) 21:40:51 DEBUG --- stdout --- 21:40:51 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 21:40:51 DEBUG --- stderr --- 21:40:51 DEBUG 21:40:51 INFO 21:40:51 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 21:40:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:40:51 INFO [loop_until]: OK (rc = 0) 21:40:51 DEBUG --- stdout --- 21:40:51 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 21:40:51 DEBUG --- stderr --- 21:40:51 DEBUG 21:40:51 INFO 21:40:51 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 21:40:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:40:51 INFO [loop_until]: OK (rc = 0) 21:40:51 DEBUG --- stdout --- 21:40:51 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 21:40:51 DEBUG --- stderr --- 21:40:51 DEBUG 21:40:51 INFO 21:40:51 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 21:40:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:40:51 INFO [loop_until]: OK (rc = 0) 21:40:51 DEBUG --- stdout --- 21:40:51 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 21:40:51 DEBUG --- stderr --- 21:40:51 DEBUG 21:40:51 INFO 21:40:51 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 21:40:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:40:51 INFO [loop_until]: OK (rc = 0) 21:40:51 DEBUG --- stdout --- 21:40:51 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 21:40:51 DEBUG --- stderr --- 21:40:51 DEBUG 21:40:51 INFO 21:40:51 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 21:40:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:40:51 INFO [loop_until]: OK (rc = 0) 21:40:51 DEBUG --- stdout --- 21:40:51 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 21:40:51 DEBUG --- stderr --- 21:40:51 DEBUG 21:40:51 INFO 21:40:51 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 21:40:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:40:51 INFO [loop_until]: OK (rc = 0) 21:40:51 DEBUG --- stdout --- 21:40:51 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 21:40:51 DEBUG --- stderr --- 21:40:51 DEBUG 21:40:52 INFO 21:40:52 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 21:40:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:40:52 INFO [loop_until]: OK (rc = 0) 21:40:52 DEBUG --- stdout --- 21:40:52 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 21:40:52 DEBUG --- stderr --- 21:40:52 DEBUG 21:40:52 INFO Initializing monitoring instance threads 21:40:52 DEBUG Monitoring instance thread list: [, , , , , , , , , , , , , , , , , , , , , , , , , , , , ] 21:40:52 INFO Starting instance threads 21:40:52 INFO 21:40:52 INFO Thread started 21:40:52 INFO [loop_until]: kubectl --namespace=xlou top node 21:40:52 INFO 21:40:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:40:52 INFO Thread started 21:40:52 INFO [loop_until]: kubectl --namespace=xlou top pods 21:40:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:40:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645652" 21:40:52 INFO Thread started 21:40:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645652" 21:40:52 INFO Thread started 21:40:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645652" 21:40:52 INFO Thread started 21:40:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645652" 21:40:52 INFO Thread started 21:40:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713645652" 21:40:52 INFO Thread started 21:40:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713645652" 21:40:52 INFO Thread started 21:40:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713645652" 21:40:52 INFO Thread started 21:40:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713645652" 21:40:52 INFO Thread started 21:40:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713645652" 21:40:52 INFO Thread started 21:40:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645652" 21:40:52 INFO Thread started 21:40:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645652" 21:40:52 INFO Thread started 21:40:52 INFO [http_cmd]: http status code OK 21:40:52 INFO [http_cmd]: http status code OK 21:40:52 DEBUG --- status code --- 21:40:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645652" 21:40:52 INFO Thread started 21:40:52 DEBUG --- status code --- 21:40:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:40:52 INFO [http_cmd]: http status code OK 21:40:52 INFO [http_cmd]: http status code OK 21:40:52 INFO [http_cmd]: http status code OK 21:40:52 INFO [http_cmd]: http status code OK 21:40:52 INFO [http_cmd]: http status code OK 21:40:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645652" 21:40:52 INFO [http_cmd]: http status code OK 21:40:52 INFO [http_cmd]: http status code OK 21:40:52 DEBUG --- status code --- 21:40:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:40:52 DEBUG --- http response --- 21:40:52 DEBUG --- status code --- 21:40:52 DEBUG --- status code --- 21:40:52 DEBUG --- status code --- 21:40:52 DEBUG --- status code --- 21:40:52 DEBUG --- status code --- 21:40:52 DEBUG --- status code --- 21:40:52 INFO [http_cmd]: http status code OK 21:40:52 INFO [http_cmd]: http status code OK 21:40:52 INFO Thread started 21:40:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:40:52 DEBUG --- http response --- 21:40:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 21:40:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:40:52 INFO [http_cmd]: http status code OK 21:40:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:40:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:40:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:40:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:40:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:40:52 DEBUG --- status code --- 21:40:52 DEBUG --- status code --- 21:40:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645652" 21:40:52 DEBUG --- http response --- 21:40:52 INFO Thread started 21:40:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 21:40:52 DEBUG ----- output ----- 21:40:52 DEBUG --- http response --- 21:40:52 DEBUG --- status code --- 21:40:52 DEBUG --- http response --- 21:40:52 DEBUG --- http response --- 21:40:52 DEBUG --- http response --- 21:40:52 DEBUG --- http response --- 21:40:52 DEBUG --- http response --- 21:40:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:40:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:40:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:40:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645652" 21:40:52 INFO Thread started 21:40:52 DEBUG ----- output ----- 21:40:52 DEBUG { 21:40:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:40:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:40:52 INFO [http_cmd]: http status code OK 21:40:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:40:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:40:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:40:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:40:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:40:52 DEBUG ----- output ----- 21:40:52 DEBUG --- http response --- 21:40:52 DEBUG ----- output ----- 21:40:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645652" 21:40:52 DEBUG { 21:40:52 INFO Thread started 21:40:52 DEBUG "status": "success", 21:40:52 DEBUG --- http response --- 21:40:52 DEBUG ----- output ----- 21:40:52 DEBUG { 21:40:52 DEBUG ----- output ----- 21:40:52 DEBUG ----- output ----- 21:40:52 DEBUG ----- output ----- 21:40:52 DEBUG ----- output ----- 21:40:52 DEBUG { 21:40:52 DEBUG --- http response --- 21:40:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:40:52 DEBUG { 21:40:52 DEBUG "status": "success", 21:40:52 DEBUG "data": { 21:40:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645652" 21:40:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:40:52 DEBUG --- status code --- 21:40:52 DEBUG "status": "success", 21:40:52 INFO Thread started 21:40:52 DEBUG { 21:40:52 DEBUG { 21:40:52 DEBUG { 21:40:52 DEBUG { 21:40:52 DEBUG "status": "success", 21:40:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:40:52 DEBUG ----- output ----- 21:40:52 DEBUG "status": "success", 21:40:52 DEBUG "data": { 21:40:52 DEBUG "data": { 21:40:52 DEBUG "resultType": "vector", 21:40:52 DEBUG ----- output ----- 21:40:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:40:52 DEBUG "data": { 21:40:52 DEBUG "status": "success", 21:40:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645652" 21:40:52 INFO Thread started 21:40:52 DEBUG "status": "success", 21:40:52 DEBUG "status": "success", 21:40:52 DEBUG "status": "success", 21:40:52 DEBUG "data": { 21:40:52 DEBUG ----- output ----- 21:40:52 DEBUG { 21:40:52 DEBUG "resultType": "vector", 21:40:52 INFO [http_cmd]: http status code OK 21:40:52 DEBUG "resultType": "vector", 21:40:52 DEBUG "result": [ 21:40:52 DEBUG { 21:40:52 DEBUG --- http response --- 21:40:52 DEBUG "resultType": "vector", 21:40:52 INFO [http_cmd]: http status code OK 21:40:52 DEBUG "data": { 21:40:52 DEBUG "resultType": "vector", 21:40:52 DEBUG "data": { 21:40:52 DEBUG "data": { 21:40:52 INFO Thread started 21:40:52 DEBUG "data": { 21:40:52 DEBUG "resultType": "vector", 21:40:52 DEBUG { 21:40:52 INFO [http_cmd]: http status code OK 21:40:52 DEBUG "status": "success", 21:40:52 DEBUG "result": [ 21:40:52 DEBUG --- status code --- 21:40:52 DEBUG "result": [ 21:40:52 DEBUG { 21:40:52 DEBUG "status": "success", 21:40:52 DEBUG "result": [ 21:40:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:40:52 DEBUG --- status code --- 21:40:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645652" 21:40:52 DEBUG "result": [ 21:40:52 DEBUG "resultType": "vector", 21:40:52 DEBUG "resultType": "vector", 21:40:52 DEBUG "resultType": "vector", 21:40:52 DEBUG "result": [ 21:40:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645652" 21:40:52 DEBUG "status": "success", 21:40:52 INFO Thread started 21:40:52 DEBUG --- status code --- 21:40:52 DEBUG "data": { 21:40:52 DEBUG { 21:40:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:40:52 DEBUG { 21:40:52 DEBUG "metric": { 21:40:52 DEBUG "data": { 21:40:52 DEBUG { 21:40:52 DEBUG ----- output ----- 21:40:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:40:52 DEBUG { 21:40:52 DEBUG "result": [ 21:40:52 DEBUG "result": [ 21:40:52 DEBUG "result": [ 21:40:52 DEBUG { 21:40:52 INFO [loop_until]: OK (rc = 0) 21:40:52 DEBUG "data": { 21:40:52 INFO [http_cmd]: http status code OK 21:40:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645652" 21:40:52 INFO Thread started 21:40:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:40:52 DEBUG "resultType": "vector", 21:40:52 DEBUG "metric": { 21:40:52 DEBUG --- http response --- 21:40:52 DEBUG "metric": { 21:40:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:40:52 DEBUG "resultType": "vector", 21:40:52 DEBUG "metric": { 21:40:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:40:52 DEBUG --- http response --- 21:40:52 DEBUG "metric": { 21:40:52 DEBUG { 21:40:52 INFO [http_cmd]: http status code OK 21:40:52 DEBUG { 21:40:52 DEBUG { 21:40:52 DEBUG "metric": { 21:40:52 DEBUG --- stdout --- 21:40:52 DEBUG "resultType": "vector", 21:40:52 DEBUG --- status code --- 21:40:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713645652" 21:40:52 DEBUG --- http response --- 21:40:52 INFO Thread started 21:40:52 DEBUG "result": [ 21:40:52 DEBUG "instance": "10.142.0.107:9100" 21:40:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:40:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:40:52 DEBUG }, 21:40:52 INFO [loop_until]: OK (rc = 0) 21:40:52 DEBUG "result": [ 21:40:52 DEBUG { 21:40:52 DEBUG }, 21:40:52 INFO [http_cmd]: http status code OK 21:40:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:40:52 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 21:40:52 DEBUG "metric": { 21:40:52 DEBUG --- status code --- 21:40:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:40:52 DEBUG "metric": { 21:40:52 DEBUG "instance": "10.142.0.107:9100" 21:40:52 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 6m 8702Mi am-767f6b67b5-qcqz2 6m 8815Mi am-767f6b67b5-vbltz 7m 8865Mi ds-cts-0 15m 12182Mi ds-cts-1 11m 12215Mi ds-cts-2 13m 12275Mi ds-idrepo-0 13m 3982Mi ds-idrepo-1 11m 3979Mi ds-idrepo-2 13m 3888Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-7dd48676f6-tdxl9 1m 54Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-967d957f7-zk66v 1m 15Mi 21:40:52 DEBUG "result": [ 21:40:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:40:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:40:52 INFO [http_cmd]: http status code OK 21:40:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713645652" 21:40:52 DEBUG { 21:40:52 DEBUG "metric": { 21:40:52 DEBUG }, 21:40:52 DEBUG }, 21:40:52 DEBUG ----- output ----- 21:40:52 DEBUG "value": [ 21:40:52 DEBUG --- stdout --- 21:40:52 DEBUG { 21:40:52 DEBUG "status": "success", 21:40:52 DEBUG "value": [ 21:40:52 DEBUG --- status code --- 21:40:52 DEBUG ----- output ----- 21:40:52 DEBUG }, 21:40:52 DEBUG "instance": "10.142.0.107:9100" 21:40:52 DEBUG "metric": { 21:40:52 DEBUG --- http response --- 21:40:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:40:52 DEBUG }, 21:40:52 DEBUG --- stderr --- 21:40:52 DEBUG { 21:40:52 DEBUG --- http response --- 21:40:52 DEBUG ----- output ----- 21:40:52 DEBUG --- status code --- 21:40:52 INFO Thread started 21:40:52 DEBUG "pod": "am-767f6b67b5-9zw22" 21:40:52 DEBUG "value": [ 21:40:52 DEBUG "value": [ 21:40:52 DEBUG { 21:40:52 DEBUG 21:40:52 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 69m 0% 6355Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 75m 0% 7704Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 57m 0% 11397Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 84m 0% 3393Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 73m 0% 15411Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 155m 0% 5205Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 68m 0% 11690Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 80m 0% 3385Mi 3% gke-xlou-cdm-ds-0f601751-hmm7 62m 0% 11067Mi 9% gke-xlou-cdm-ds-0f601751-j488 66m 0% 7266Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 126m 0% 17260Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 94m 0% 7945Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 66m 0% 14969Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 66m 0% 7993Mi 13% 21:40:52 DEBUG "metric": { 21:40:52 DEBUG "data": { 21:40:52 DEBUG 21:40:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:40:52 DEBUG { 21:40:52 DEBUG "value": [ 21:40:52 DEBUG 21:40:52 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 21:40:52 DEBUG }, 21:40:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:40:52 INFO [http_cmd]: http status code OK 21:40:52 DEBUG "value": [ 21:40:52 DEBUG 21:40:52 DEBUG "metric": { 21:40:52 DEBUG { 21:40:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:40:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:40:52 DEBUG }, 21:40:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713645652" 21:40:52 INFO Thread started 21:40:52 DEBUG 21:40:52 DEBUG 21:40:52 DEBUG "status": "success", 21:40:52 DEBUG 21:40:52 DEBUG --- stderr --- 21:40:52 DEBUG "pod": "ds-cts-0" 21:40:52 DEBUG }, 21:40:52 DEBUG 21:40:52 DEBUG --- http response --- 21:40:52 DEBUG "status": "success", 21:40:52 DEBUG }, 21:40:52 DEBUG 21:40:52 DEBUG }, 21:40:52 DEBUG "value": [ 21:40:52 DEBUG ----- output ----- 21:40:52 DEBUG --- status code --- 21:40:52 DEBUG 21:40:52 INFO [http_cmd]: http status code OK 21:40:52 DEBUG "pod": "am-767f6b67b5-9zw22" 21:40:52 DEBUG "status": "success", 21:40:52 DEBUG ----- output ----- 21:40:52 DEBUG --- http response --- 21:40:52 DEBUG "value": [ 21:40:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713645652" 21:40:52 INFO Thread started 21:40:52 DEBUG 21:40:52 DEBUG 21:40:52 DEBUG "data": { 21:40:52 DEBUG 21:40:52 DEBUG 21:40:52 DEBUG "resultType": "vector", 21:40:52 DEBUG "value": [ 21:40:52 DEBUG 21:40:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:40:52 DEBUG "data": { 21:40:52 DEBUG "value": [ 21:40:52 DEBUG 21:40:52 DEBUG "value": [ 21:40:52 DEBUG 21:40:52 DEBUG { 21:40:52 INFO [http_cmd]: http status code OK 21:40:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:40:52 DEBUG 21:40:52 DEBUG --- status code --- 21:40:52 DEBUG }, 21:40:52 DEBUG "data": { 21:40:52 DEBUG { 21:40:52 DEBUG 21:40:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:40:52 DEBUG 21:40:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645652" 21:40:52 DEBUG 21:40:52 INFO Thread started 21:40:52 DEBUG "resultType": "vector", 21:40:52 DEBUG [...] 21:40:52 DEBUG "result": [ 21:40:52 DEBUG 21:40:52 DEBUG [...] 21:40:52 DEBUG ----- output ----- 21:40:52 DEBUG "resultType": "vector", 21:40:52 DEBUG 21:40:52 DEBUG [...] 21:40:52 DEBUG 21:40:52 DEBUG 21:40:52 DEBUG "status": "success", 21:40:52 DEBUG --- status code --- 21:40:52 ERROR [http_cmd]: ERROR 21:40:52 DEBUG --- http response --- 21:40:52 DEBUG 21:40:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:40:52 DEBUG "value": [ 21:40:52 DEBUG "resultType": "vector", 21:40:52 DEBUG "status": "success", 21:40:52 DEBUG 21:40:52 DEBUG ----- output ----- 21:40:52 DEBUG [...] 21:40:52 DEBUG [...] 21:40:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713645652" 21:40:52 DEBUG "result": [ 21:40:52 INFO Thread started 21:40:52 DEBUG 21:40:52 DEBUG { 21:40:52 DEBUG 21:40:52 DEBUG 21:40:52 DEBUG { 21:40:52 DEBUG "result": [ 21:40:52 DEBUG 21:40:52 DEBUG 21:40:52 DEBUG 21:40:52 DEBUG 21:40:52 DEBUG "data": { 21:40:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:40:52 DEBUG --- http response --- 21:40:52 ERROR --- status code --- 21:40:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:40:52 DEBUG --- http response --- 21:40:52 DEBUG 21:40:52 DEBUG "result": [ 21:40:52 DEBUG "data": { 21:40:52 DEBUG 21:40:52 DEBUG { 21:40:52 DEBUG 21:40:52 DEBUG 21:40:52 DEBUG { 21:40:52 INFO All threads has been started 127.0.0.1 - - [20/Apr/2024 21:40:52] "GET /monitoring/start HTTP/1.1" 200 - 21:40:52 DEBUG 21:40:52 DEBUG "metric": { 21:40:52 DEBUG 21:40:52 DEBUG [...] 21:40:52 DEBUG "status": "success", 21:40:52 DEBUG { 21:40:52 DEBUG 21:40:52 DEBUG 21:40:52 DEBUG 21:40:52 DEBUG [...] 21:40:52 DEBUG "resultType": "vector", 21:40:52 DEBUG [...] 21:40:52 ERROR http status code is 400 (expected [200, 401, 403, 404]) 21:40:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:40:52 DEBUG ----- output ----- 21:40:52 INFO [http_cmd]: http status code OK 21:40:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:40:52 DEBUG 21:40:52 DEBUG { 21:40:52 DEBUG "resultType": "vector", 21:40:52 DEBUG [...] 21:40:52 DEBUG "status": "success", 21:40:52 DEBUG 21:40:52 DEBUG 21:40:52 DEBUG "metric": { 21:40:52 DEBUG 21:40:52 DEBUG "pod": "ds-idrepo-0" 21:40:52 DEBUG 21:40:52 DEBUG 21:40:52 DEBUG "data": { 21:40:52 DEBUG "metric": { 21:40:52 DEBUG [...] 21:40:52 DEBUG 21:40:52 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 21:40:52 DEBUG 21:40:52 DEBUG "result": [ 21:40:52 DEBUG { 21:40:52 ERROR --- http response --- 21:40:52 DEBUG ----- output ----- 21:40:52 DEBUG { 21:40:52 DEBUG --- status code --- 21:40:52 DEBUG ----- output ----- 21:40:52 DEBUG 21:40:52 DEBUG "metric": { 21:40:52 INFO [http_cmd]: http status code OK 21:40:52 DEBUG "result": [ 21:40:52 DEBUG { 21:40:52 DEBUG 21:40:52 DEBUG 21:40:52 DEBUG 21:40:52 DEBUG "pod": "ds-idrepo-0" 21:40:52 DEBUG "pod": "am-767f6b67b5-vbltz" 21:40:52 DEBUG }, 21:40:52 DEBUG 21:40:52 DEBUG 21:40:52 DEBUG "resultType": "vector", 21:40:52 DEBUG "result": [ 21:40:52 DEBUG 21:40:52 DEBUG [...] 21:40:52 DEBUG }, 21:40:52 DEBUG 21:40:52 DEBUG 21:40:52 DEBUG "metric": { 21:40:52 DEBUG { 21:40:52 DEBUG "status": "success", 21:40:52 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 21:40:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:40:52 DEBUG { 21:40:52 DEBUG [...] 21:40:52 DEBUG "pod": "am-767f6b67b5-9zw22" 21:40:52 DEBUG --- status code --- 21:40:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:40:52 DEBUG "metric": { 21:40:52 DEBUG 21:40:52 DEBUG "instance": "10.142.0.43:9100" 21:40:52 DEBUG "pod": "am-767f6b67b5-vbltz" 21:40:52 DEBUG }, 21:40:52 DEBUG }, 21:40:52 DEBUG "value": [ 21:40:52 DEBUG "pod": "am-767f6b67b5-9zw22" 21:40:52 DEBUG 21:40:52 DEBUG "pod": "ds-idrepo-0" 21:40:52 DEBUG { 21:40:52 DEBUG 21:40:52 DEBUG 21:40:52 DEBUG "value": [ 21:40:52 DEBUG 21:40:52 DEBUG 21:40:52 DEBUG "pod": "am-767f6b67b5-9zw22" 21:40:52 DEBUG "status": "success", 21:40:52 DEBUG "data": { 21:40:52 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 21:40:52 DEBUG --- http response --- 21:40:52 DEBUG "status": "success", 21:40:52 DEBUG 21:40:52 DEBUG }, 21:40:52 DEBUG "data": { 21:40:52 DEBUG --- http response --- 21:40:52 DEBUG "pod": "am-767f6b67b5-9zw22" 21:40:52 DEBUG 21:40:52 DEBUG }, 21:40:52 DEBUG }, 21:40:52 DEBUG "value": [ 21:40:52 DEBUG "value": [ 21:40:52 DEBUG 21:40:52 DEBUG }, 21:40:52 DEBUG "pod": "ds-cts-2" 21:40:52 DEBUG }, 21:40:52 DEBUG "metric": { 21:40:52 DEBUG 21:40:52 DEBUG 21:40:52 DEBUG 1713645652, 21:40:52 DEBUG "pod": "am-767f6b67b5-9zw22" 21:40:52 DEBUG 21:40:52 DEBUG }, 21:40:52 DEBUG "data": { 21:40:52 DEBUG "resultType": "vector", 21:40:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:40:52 DEBUG "data": { 21:40:52 DEBUG 21:40:52 DEBUG "value": [ 21:40:52 DEBUG "resultType": "vector", 21:40:52 INFO [http_cmd]: http status code OK 21:40:52 DEBUG }, 21:40:52 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 21:40:52 DEBUG "pod": "am-767f6b67b5-vbltz" 21:40:52 DEBUG "value": [ 21:40:52 DEBUG "value": [ 21:40:52 DEBUG 21:40:52 DEBUG 1713645652, 21:40:52 DEBUG 21:40:52 DEBUG "value": [ 21:40:52 DEBUG }, 21:40:52 DEBUG "value": [ 21:40:52 DEBUG "pod": "ds-cts-0" 21:40:52 DEBUG "instance": "10.142.0.43:9100" 21:40:52 DEBUG 21:40:52 DEBUG "110743552" 21:40:52 DEBUG }, 21:40:52 DEBUG "instance": "10.142.0.43:9100" 21:40:52 DEBUG "value": [ 21:40:52 DEBUG "resultType": "vector", 21:40:52 DEBUG "result": [ 21:40:52 DEBUG ----- output ----- 21:40:52 DEBUG "resultType": "vector", 21:40:52 DEBUG 21:40:52 DEBUG 21:40:52 DEBUG "result": [ 21:40:52 DEBUG --- status code --- 21:40:52 DEBUG "value": [ 21:40:52 DEBUG }, 21:40:52 DEBUG 1713645652, 21:40:52 DEBUG "33408.73333333333" 21:40:52 DEBUG 21:40:52 DEBUG "0" 21:40:52 DEBUG 21:40:52 DEBUG 1713645652, 21:40:52 DEBUG "0" 21:40:52 DEBUG 21:40:52 DEBUG }, 21:40:52 DEBUG }, 21:40:52 DEBUG "value": [ 21:40:52 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 21:40:52 DEBUG "value": [ 21:40:52 DEBUG }, 21:40:52 DEBUG 21:40:52 DEBUG "result": [ 21:40:52 DEBUG { 21:40:52 DEBUG { 21:40:52 DEBUG "result": [ 21:40:52 DEBUG "pod": "am-767f6b67b5-vbltz" 21:40:52 DEBUG 21:40:52 DEBUG { 21:40:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:40:52 DEBUG 21:40:52 DEBUG "value": [ 21:40:52 DEBUG 1713645652, 21:40:52 DEBUG ] 21:40:52 DEBUG 21:40:52 DEBUG ] 21:40:52 DEBUG [...] 21:40:52 DEBUG "value": [ 21:40:52 DEBUG 1713645652, 21:40:52 DEBUG 21:40:52 DEBUG "value": [ 21:40:52 DEBUG ] 21:40:52 DEBUG } 21:40:52 DEBUG 1713645652, 21:40:52 DEBUG 1713645652, 21:40:52 DEBUG "value": [ 21:40:52 DEBUG 21:40:52 DEBUG { 21:40:52 DEBUG "metric": { 21:40:52 DEBUG "status": "success", 21:40:52 DEBUG { 21:40:52 DEBUG }, 21:40:52 DEBUG 21:40:52 DEBUG "metric": { 21:40:52 DEBUG --- http response --- 21:40:52 DEBUG 21:40:52 DEBUG 1713645652, 21:40:52 DEBUG "0" 21:40:52 DEBUG } 21:40:52 DEBUG [...] 21:40:52 DEBUG } 21:40:52 DEBUG 21:40:52 DEBUG ] 21:40:52 DEBUG } 21:40:52 DEBUG "0" 21:40:52 DEBUG 21:40:52 DEBUG }, 21:40:52 DEBUG ] 21:40:52 DEBUG "27513.533333333333" 21:40:52 DEBUG "0" 21:40:52 DEBUG 1713645652, 21:40:52 DEBUG 21:40:52 DEBUG "metric": { 21:40:52 DEBUG "pod": "ds-idrepo-0" 21:40:52 DEBUG "data": { 21:40:52 DEBUG "metric": { 21:40:52 DEBUG "value": [ 21:40:52 DEBUG [...] 21:40:52 DEBUG "pod": "ds-cts-0" 21:40:52 DEBUG 21:40:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:40:52 DEBUG "0" 21:40:52 DEBUG ] 21:40:52 DEBUG ] 21:40:52 DEBUG 21:40:52 DEBUG ] 21:40:52 DEBUG 21:40:52 DEBUG 21:40:52 DEBUG ] 21:40:52 DEBUG ] 21:40:52 DEBUG 21:40:52 DEBUG 21:40:52 DEBUG } 21:40:52 DEBUG ] 21:40:52 DEBUG ] 21:40:52 DEBUG "0" 21:40:52 DEBUG [...] 21:40:52 DEBUG "pod": "am-767f6b67b5-9zw22" 21:40:52 DEBUG }, 21:40:52 DEBUG "resultType": "vector", 21:40:52 DEBUG "pod": "am-767f6b67b5-9zw22" 21:40:52 DEBUG 1713645652, 21:40:52 DEBUG 21:40:52 DEBUG }, 21:40:52 DEBUG "value": [ 21:40:52 DEBUG ----- output ----- 21:40:52 DEBUG ] 21:40:52 DEBUG } 21:40:52 DEBUG } 21:40:52 DEBUG 21:40:52 DEBUG } 21:40:52 DEBUG 21:40:52 DEBUG [...] 21:40:52 DEBUG } 21:40:52 DEBUG } 21:40:52 DEBUG ] 21:40:52 DEBUG [...] 21:40:52 DEBUG } 21:40:52 DEBUG } 21:40:52 DEBUG } 21:40:52 DEBUG ] 21:40:52 DEBUG 21:40:52 DEBUG }, 21:40:52 DEBUG "value": [ 21:40:52 DEBUG "result": [ 21:40:52 DEBUG }, 21:40:52 DEBUG "0" 21:40:52 DEBUG 21:40:52 DEBUG [...] 21:40:52 DEBUG 21:40:52 DEBUG { 21:40:52 DEBUG } 21:40:52 DEBUG ] 21:40:52 DEBUG ] 21:40:52 DEBUG 21:40:52 DEBUG } 21:40:52 DEBUG "pod": "ds-idrepo-2" 21:40:52 DEBUG 21:40:52 DEBUG } 21:40:52 DEBUG "value": [ 21:40:52 DEBUG } 21:40:52 DEBUG 21:40:52 DEBUG ] 21:40:52 DEBUG } 21:40:52 DEBUG } 21:40:52 DEBUG 21:40:52 DEBUG "value": [ 21:40:52 DEBUG 21:40:52 DEBUG { 21:40:52 DEBUG "value": [ 21:40:52 DEBUG ] 21:40:52 DEBUG 21:40:52 DEBUG 21:40:52 DEBUG 21:40:52 DEBUG "status": "success", 21:40:52 DEBUG } 21:40:52 DEBUG } 21:40:52 DEBUG } 21:40:52 DEBUG "pod": "ds-idrepo-2" 21:40:52 DEBUG }, 21:40:52 DEBUG 21:40:52 DEBUG 21:40:52 DEBUG 1713645652, 21:40:52 DEBUG 21:40:52 DEBUG ] 21:40:52 DEBUG } 21:40:52 DEBUG ] 21:40:52 DEBUG 21:40:52 DEBUG 21:40:52 DEBUG 21:40:52 DEBUG "metric": { 21:40:52 DEBUG 21:40:52 DEBUG } 21:40:52 DEBUG "pod": "am-767f6b67b5-vbltz" 21:40:52 DEBUG 21:40:52 DEBUG 21:40:52 DEBUG "data": { 21:40:52 DEBUG } 21:40:52 DEBUG } 21:40:52 DEBUG }, 21:40:52 DEBUG "value": [ 21:40:52 DEBUG 1713645652, 21:40:52 DEBUG "pod": "ds-idrepo-2" 21:40:52 DEBUG "0.029049606165240917" 21:40:52 DEBUG 21:40:52 DEBUG } 21:40:52 DEBUG } 21:40:52 DEBUG "pod": "am-767f6b67b5-vbltz" 21:40:52 DEBUG 21:40:52 DEBUG 21:40:52 DEBUG "instance": "10.142.0.107:9100" 21:40:52 DEBUG 21:40:52 DEBUG ] 21:40:52 DEBUG }, 21:40:52 DEBUG 21:40:52 DEBUG [...] 21:40:52 DEBUG "resultType": "vector", 21:40:52 DEBUG "value": [ 21:40:52 DEBUG } 21:40:52 DEBUG "0" 21:40:52 DEBUG }, 21:40:52 DEBUG ] 21:40:52 DEBUG "pod": "ds-cts-2" 21:40:52 DEBUG } 21:40:52 DEBUG } 21:40:52 DEBUG }, 21:40:52 DEBUG 21:40:52 DEBUG [...] 21:40:52 DEBUG }, 21:40:52 DEBUG 21:40:52 DEBUG } 21:40:52 DEBUG "value": [ 21:40:52 DEBUG "pod": "am-767f6b67b5-vbltz" 21:40:52 DEBUG 21:40:52 DEBUG "result": [ 21:40:52 DEBUG 1713645652, 21:40:52 DEBUG ] 21:40:52 DEBUG "value": [ 21:40:52 DEBUG } 21:40:52 DEBUG }, 21:40:52 DEBUG "value": [ 21:40:52 DEBUG [...] 21:40:52 DEBUG 21:40:52 DEBUG "value": [ 21:40:52 DEBUG [...] 21:40:52 DEBUG } 21:40:52 DEBUG 1713645652, 21:40:52 DEBUG }, 21:40:52 DEBUG 21:40:52 DEBUG { 21:40:52 DEBUG "0" 21:40:52 DEBUG } 21:40:52 DEBUG 1713645652, 21:40:52 DEBUG ] 21:40:52 DEBUG "value": [ 21:40:52 DEBUG 1713645652, 21:40:52 DEBUG 21:40:52 DEBUG 21:40:52 DEBUG 21:40:52 DEBUG 21:40:52 DEBUG "0" 21:40:52 DEBUG "value": [ 21:40:52 DEBUG 21:40:52 DEBUG "pod": "ds-cts-2" 21:40:52 DEBUG ] 21:40:52 DEBUG ] 21:40:52 DEBUG "0" 21:40:52 DEBUG } 21:40:52 DEBUG 1713645652, 21:40:52 DEBUG "0" 21:40:52 DEBUG ] 21:40:52 DEBUG 21:40:52 DEBUG 21:40:52 DEBUG 21:40:52 DEBUG ] 21:40:52 DEBUG 1713645652, 21:40:52 DEBUG "metric": { 21:40:52 DEBUG }, 21:40:52 DEBUG } 21:40:52 DEBUG } 21:40:52 DEBUG ] 21:40:52 DEBUG } 21:40:52 DEBUG "0.03333333333333333" 21:40:52 DEBUG 21:40:52 DEBUG } 21:40:52 DEBUG "pod": "am-767f6b67b5-vbltz" 21:40:52 DEBUG 21:40:52 DEBUG 21:40:52 DEBUG } 21:40:52 DEBUG "0" 21:40:52 DEBUG "instance": "10.142.0.107:9100" 21:40:52 DEBUG "value": [ 21:40:52 DEBUG ] 21:40:52 DEBUG } 21:40:52 DEBUG } 21:40:52 DEBUG ] 21:40:52 DEBUG 21:40:52 DEBUG ] 21:40:52 DEBUG }, 21:40:52 DEBUG [...] 21:40:52 DEBUG "pod": "am-767f6b67b5-vbltz" 21:40:52 DEBUG ] 21:40:52 DEBUG ] 21:40:52 DEBUG }, 21:40:52 DEBUG 1713645652, 21:40:52 DEBUG } 21:40:52 DEBUG ] 21:40:52 DEBUG } 21:40:52 DEBUG ] 21:40:52 DEBUG } 21:40:52 DEBUG "value": [ 21:40:52 DEBUG 21:40:52 DEBUG }, 21:40:52 DEBUG } 21:40:52 DEBUG } 21:40:52 DEBUG "value": [ 21:40:52 DEBUG "0" 21:40:52 DEBUG } 21:40:52 DEBUG } 21:40:52 DEBUG } 21:40:52 DEBUG "pod": "ds-cts-2" 21:40:52 DEBUG } 21:40:52 DEBUG 1713645652, 21:40:52 DEBUG 21:40:52 DEBUG 21:40:52 DEBUG } 21:40:52 DEBUG ] 21:40:52 DEBUG 21:40:52 DEBUG ] 21:40:52 DEBUG } 21:40:52 DEBUG } 21:40:52 DEBUG }, 21:40:52 DEBUG "1" 21:40:52 DEBUG "instance": "10.142.0.43:9100" 21:40:52 DEBUG }, 21:40:52 DEBUG } 21:40:52 DEBUG 21:40:52 DEBUG } 21:40:52 DEBUG "value": [ 21:40:52 DEBUG ] 21:40:52 DEBUG "value": [ 21:40:52 DEBUG "value": [ 21:40:52 DEBUG } 21:40:52 DEBUG 21:40:52 DEBUG ] 21:40:52 DEBUG 1713645652, 21:40:52 DEBUG } 21:40:52 DEBUG 1713645652, 21:40:52 DEBUG 1713645652, 21:40:52 DEBUG [...] 21:40:52 DEBUG } 21:40:52 DEBUG "0" 21:40:52 DEBUG ] 21:40:52 DEBUG "49.377" 21:40:52 DEBUG "3944323437568" 21:40:52 DEBUG 21:40:52 DEBUG } 21:40:52 DEBUG ] 21:40:52 DEBUG } 21:40:52 DEBUG ] 21:40:52 DEBUG ] 21:40:52 DEBUG 21:40:52 DEBUG } 21:40:52 DEBUG } 21:40:52 DEBUG } 21:40:52 DEBUG } 21:40:52 DEBUG 21:40:52 DEBUG ] 21:40:52 DEBUG ] 21:40:52 DEBUG ] 21:40:52 DEBUG "instance": "10.142.0.43:9100" 21:40:52 DEBUG } 21:40:52 DEBUG } 21:40:52 DEBUG } 21:40:52 DEBUG }, 21:40:52 DEBUG } 21:40:52 DEBUG } 21:40:52 DEBUG } 21:40:52 DEBUG "value": [ 21:40:52 DEBUG 1713645652, 21:40:52 DEBUG "0.0007000000000478698" 21:40:52 DEBUG ] 21:40:52 DEBUG } 21:40:52 DEBUG ] 21:40:52 DEBUG } 21:40:52 DEBUG } 21:41:52 INFO 21:41:52 INFO [loop_until]: kubectl --namespace=xlou top pods 21:41:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:41:52 INFO 21:41:52 INFO [loop_until]: kubectl --namespace=xlou top node 21:41:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:41:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713645712" 21:41:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645712" 21:41:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713645712" 21:41:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645712" 21:41:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645712" 21:41:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713645712" 21:41:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645712" 21:41:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645712" 21:41:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645712" 21:41:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713645712" 21:41:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645712" 21:41:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713645712" 21:41:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645712" 21:41:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713645712" 21:41:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645712" 21:41:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645712" 21:41:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645712" 21:41:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645712" 21:41:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645712" 21:41:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645712" 21:41:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645712" 21:41:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645712" 21:41:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713645712" 21:41:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713645712" 21:41:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713645712" 21:41:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713645712" 21:41:52 INFO [http_cmd]: http status code OK 21:41:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645712" 21:41:52 INFO [http_cmd]: http status code OK 21:41:52 INFO [http_cmd]: http status code OK 21:41:52 ERROR [http_cmd]: ERROR 21:41:52 INFO [http_cmd]: http status code OK 21:41:52 INFO [http_cmd]: http status code OK 21:41:52 INFO [http_cmd]: http status code OK 21:41:52 INFO [http_cmd]: http status code OK 21:41:52 INFO [http_cmd]: http status code OK 21:41:52 INFO [http_cmd]: http status code OK 21:41:52 INFO [loop_until]: OK (rc = 0) 21:41:52 INFO [http_cmd]: http status code OK 21:41:52 DEBUG --- status code --- 21:41:52 INFO [http_cmd]: http status code OK 21:41:52 INFO [loop_until]: OK (rc = 0) 21:41:52 INFO [http_cmd]: http status code OK 21:41:52 INFO [http_cmd]: http status code OK 21:41:52 INFO [http_cmd]: http status code OK 21:41:52 INFO [http_cmd]: http status code OK 21:41:52 INFO [http_cmd]: http status code OK 21:41:52 INFO [http_cmd]: http status code OK 21:41:52 DEBUG --- status code --- 21:41:52 INFO [http_cmd]: http status code OK 21:41:52 DEBUG --- status code --- 21:41:52 DEBUG --- status code --- 21:41:52 INFO [http_cmd]: http status code OK 21:41:52 INFO [http_cmd]: http status code OK 21:41:52 INFO [http_cmd]: http status code OK 21:41:52 ERROR --- status code --- 21:41:52 DEBUG --- status code --- 21:41:52 DEBUG --- status code --- 21:41:52 DEBUG --- status code --- 21:41:52 DEBUG --- status code --- 21:41:52 DEBUG --- status code --- 21:41:52 DEBUG --- status code --- 21:41:52 INFO [http_cmd]: http status code OK 21:41:52 DEBUG --- stdout --- 21:41:52 INFO [http_cmd]: http status code OK 21:41:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:41:52 INFO [http_cmd]: http status code OK 21:41:52 DEBUG --- status code --- 21:41:52 DEBUG --- stdout --- 21:41:52 DEBUG --- status code --- 21:41:52 DEBUG --- status code --- 21:41:52 DEBUG --- status code --- 21:41:52 DEBUG --- status code --- 21:41:52 INFO [http_cmd]: http status code OK 21:41:52 DEBUG --- status code --- 21:41:52 DEBUG --- status code --- 21:41:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:41:52 DEBUG --- status code --- 21:41:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:41:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:41:52 DEBUG --- status code --- 21:41:52 DEBUG --- status code --- 21:41:52 DEBUG --- status code --- 21:41:52 ERROR http status code is 400 (expected [200, 401, 403, 404]) 21:41:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:41:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:41:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:41:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:41:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:41:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:41:52 DEBUG --- status code --- 21:41:52 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 774m 8705Mi am-767f6b67b5-qcqz2 851m 8818Mi am-767f6b67b5-vbltz 7m 8864Mi ds-cts-0 196m 12184Mi ds-cts-1 149m 12218Mi ds-cts-2 13m 12276Mi ds-idrepo-0 174m 3983Mi ds-idrepo-1 160m 3979Mi ds-idrepo-2 89m 3888Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-7dd48676f6-tdxl9 113m 61Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-967d957f7-zk66v 2789m 554Mi 21:41:52 DEBUG --- status code --- 21:41:52 DEBUG --- http response --- 21:41:52 DEBUG --- status code --- 21:41:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:41:52 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 6231Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 469m 1% 7713Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 909m 3% 11426Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 86m 0% 3387Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 269m 0% 15393Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 155m 0% 5229Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 325m 1% 11703Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 351m 1% 3166Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 772m 2% 11099Mi 9% gke-xlou-cdm-ds-0f601751-j488 685m 2% 7308Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 312m 1% 17226Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 518m 1% 7965Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 622m 2% 14964Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 3480m 21% 8505Mi 14% 21:41:52 INFO [http_cmd]: http status code OK 21:41:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:41:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:41:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:41:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:41:52 DEBUG --- status code --- 21:41:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:41:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:41:52 DEBUG --- http response --- 21:41:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:41:52 DEBUG --- http response --- 21:41:52 DEBUG --- http response --- 21:41:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:41:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:41:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:41:52 ERROR --- http response --- 21:41:52 DEBUG --- http response --- 21:41:52 DEBUG --- http response --- 21:41:52 DEBUG --- http response --- 21:41:52 DEBUG --- http response --- 21:41:52 DEBUG --- http response --- 21:41:52 DEBUG --- http response --- 21:41:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:41:52 DEBUG --- stderr --- 21:41:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:41:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:41:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:41:52 DEBUG --- http response --- 21:41:52 DEBUG --- stderr --- 21:41:52 DEBUG --- status code --- 21:41:52 DEBUG --- http response --- 21:41:52 DEBUG --- http response --- 21:41:52 DEBUG --- http response --- 21:41:52 DEBUG --- http response --- 21:41:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:41:52 DEBUG --- http response --- 21:41:52 DEBUG --- http response --- 21:41:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:41:52 DEBUG --- http response --- 21:41:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 21:41:52 DEBUG --- http response --- 21:41:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:41:52 DEBUG --- http response --- 21:41:52 DEBUG --- http response --- 21:41:52 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 21:41:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:41:52 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 21:41:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:41:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:41:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 21:41:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:41:52 DEBUG --- http response --- 21:41:52 DEBUG 21:41:52 DEBUG --- http response --- 21:41:52 DEBUG ----- output ----- 21:41:52 DEBUG --- http response --- 21:41:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:41:52 DEBUG 21:41:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:41:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:41:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:41:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:41:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:41:52 DEBUG --- http response --- 21:41:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:41:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:41:52 DEBUG ----- output ----- 21:41:52 DEBUG ----- output ----- 21:41:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:41:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:41:52 DEBUG ----- output ----- 21:41:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:41:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:41:52 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 21:41:52 DEBUG ----- output ----- 21:41:52 DEBUG ----- output ----- 21:41:52 DEBUG ----- output ----- 21:41:52 DEBUG ----- output ----- 21:41:52 DEBUG ----- output ----- 21:41:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:41:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:41:52 DEBUG { 21:41:52 DEBUG ----- output ----- 21:41:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:41:52 DEBUG --- http response --- 21:41:52 DEBUG ----- output ----- 21:41:52 DEBUG ----- output ----- 21:41:52 DEBUG ----- output ----- 21:41:52 DEBUG ----- output ----- 21:41:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:41:52 DEBUG ----- output ----- 21:41:52 DEBUG ----- output ----- 21:41:52 DEBUG { 21:41:52 DEBUG { 21:41:52 DEBUG ----- output ----- 21:41:52 DEBUG ----- output ----- 21:41:52 DEBUG { 21:41:52 DEBUG ----- output ----- 21:41:52 DEBUG ----- output ----- 21:41:52 DEBUG { 21:41:52 DEBUG { 21:41:52 DEBUG { 21:41:52 DEBUG { 21:41:52 DEBUG { 21:41:52 DEBUG ----- output ----- 21:41:52 DEBUG ----- output ----- 21:41:52 DEBUG "status": "success", 21:41:52 DEBUG { 21:41:52 DEBUG ----- output ----- 21:41:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:41:52 DEBUG { 21:41:52 DEBUG { 21:41:52 DEBUG { 21:41:52 DEBUG { 21:41:52 DEBUG ----- output ----- 21:41:52 DEBUG { 21:41:52 DEBUG { 21:41:52 DEBUG "status": "success", 21:41:52 DEBUG "status": "success", 21:41:52 DEBUG { 21:41:52 DEBUG { 21:41:52 DEBUG "status": "success", 21:41:52 DEBUG "data": { 21:41:52 DEBUG { 21:41:52 DEBUG "status": "success", 21:41:52 DEBUG "status": "success", 21:41:52 DEBUG "status": "success", 21:41:52 DEBUG "status": "success", 21:41:52 DEBUG "status": "success", 21:41:52 DEBUG { 21:41:52 DEBUG { 21:41:52 DEBUG "data": { 21:41:52 DEBUG "status": "success", 21:41:52 DEBUG { 21:41:52 DEBUG ----- output ----- 21:41:52 DEBUG "status": "success", 21:41:52 DEBUG "status": "success", 21:41:52 DEBUG "status": "success", 21:41:52 DEBUG "status": "success", 21:41:52 DEBUG { 21:41:52 DEBUG "status": "success", 21:41:52 DEBUG "status": "success", 21:41:52 DEBUG "data": { 21:41:52 DEBUG "data": { 21:41:52 DEBUG "status": "success", 21:41:52 DEBUG "status": "success", 21:41:52 DEBUG { 21:41:52 DEBUG "resultType": "vector", 21:41:52 DEBUG "status": "success", 21:41:52 DEBUG "data": { 21:41:52 DEBUG "data": { 21:41:52 DEBUG "data": { 21:41:52 DEBUG "data": { 21:41:52 DEBUG "data": { 21:41:52 DEBUG "status": "success", 21:41:52 DEBUG "status": "success", 21:41:52 DEBUG "resultType": "vector", 21:41:52 DEBUG "data": { 21:41:52 DEBUG "status": "success", 21:41:52 DEBUG { 21:41:52 DEBUG "data": { 21:41:52 DEBUG "resultType": "vector", 21:41:52 DEBUG "data": { 21:41:52 DEBUG "resultType": "vector", 21:41:52 DEBUG "status": "success", 21:41:52 DEBUG "data": { 21:41:52 DEBUG "resultType": "vector", 21:41:52 DEBUG "data": { 21:41:52 DEBUG "resultType": "vector", 21:41:52 DEBUG "data": { 21:41:52 DEBUG "data": { 21:41:52 DEBUG "status": "success", 21:41:52 DEBUG "result": [ 21:41:52 DEBUG "data": { 21:41:52 DEBUG "resultType": "vector", 21:41:52 DEBUG "resultType": "vector", 21:41:52 DEBUG "resultType": "vector", 21:41:52 DEBUG "resultType": "vector", 21:41:52 DEBUG "resultType": "vector", 21:41:52 DEBUG "data": { 21:41:52 DEBUG "data": { 21:41:52 DEBUG "result": [ 21:41:52 DEBUG { 21:41:52 DEBUG "metric": { 21:41:52 DEBUG "status": "success", 21:41:52 DEBUG "data": { 21:41:52 DEBUG "result": [ 21:41:52 DEBUG "data": { 21:41:52 DEBUG "result": [ 21:41:52 DEBUG "data": { 21:41:52 DEBUG "resultType": "vector", 21:41:52 DEBUG "result": [ 21:41:52 DEBUG "resultType": "vector", 21:41:52 DEBUG "result": [ 21:41:52 DEBUG "resultType": "vector", 21:41:52 DEBUG "resultType": "vector", 21:41:52 DEBUG "data": { 21:41:52 DEBUG { 21:41:52 DEBUG "resultType": "vector", 21:41:52 DEBUG "result": [ 21:41:52 DEBUG "result": [ 21:41:52 DEBUG "result": [ 21:41:52 DEBUG "result": [ 21:41:52 DEBUG "result": [ 21:41:52 DEBUG "resultType": "vector", 21:41:52 DEBUG "resultType": "vector", 21:41:52 DEBUG "resultType": "vector", 21:41:52 DEBUG "pod": "ds-idrepo-0" 21:41:52 DEBUG "data": { 21:41:52 DEBUG "data": { 21:41:52 DEBUG "resultType": "vector", 21:41:52 DEBUG { 21:41:52 DEBUG "resultType": "vector", 21:41:52 DEBUG { 21:41:52 DEBUG "resultType": "vector", 21:41:52 DEBUG "result": [ 21:41:52 DEBUG { 21:41:52 DEBUG "result": [ 21:41:52 DEBUG { 21:41:52 DEBUG "result": [ 21:41:52 DEBUG "result": [ 21:41:52 DEBUG "resultType": "vector", 21:41:52 DEBUG "metric": { 21:41:52 DEBUG "result": [ 21:41:52 DEBUG { 21:41:52 DEBUG { 21:41:52 DEBUG { 21:41:52 DEBUG { 21:41:52 DEBUG { 21:41:52 DEBUG "result": [ 21:41:52 DEBUG "result": [ 21:41:52 DEBUG "result": [ 21:41:52 DEBUG }, 21:41:52 DEBUG "resultType": "vector", 21:41:52 DEBUG "resultType": "vector", 21:41:52 DEBUG "result": [ 21:41:52 DEBUG "metric": { 21:41:52 DEBUG "result": [ 21:41:52 DEBUG "metric": { 21:41:52 DEBUG "result": [ 21:41:52 DEBUG { 21:41:52 DEBUG "metric": { 21:41:52 DEBUG { 21:41:52 DEBUG "metric": { 21:41:52 DEBUG { 21:41:52 DEBUG { 21:41:52 DEBUG "result": [ 21:41:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:41:52 DEBUG { 21:41:52 DEBUG "metric": { 21:41:52 DEBUG "metric": { 21:41:52 DEBUG "metric": { 21:41:52 DEBUG "metric": { 21:41:52 DEBUG "metric": { 21:41:52 DEBUG { 21:41:52 DEBUG { 21:41:52 DEBUG { 21:41:52 DEBUG "value": [ 21:41:52 DEBUG "result": [ 21:41:52 DEBUG "result": [ 21:41:52 DEBUG { 21:41:52 DEBUG "instance": "10.142.0.107:9100" 21:41:52 DEBUG { 21:41:52 DEBUG "pod": "am-767f6b67b5-9zw22" 21:41:52 DEBUG { 21:41:52 DEBUG "metric": { 21:41:52 DEBUG "pod": "am-767f6b67b5-9zw22" 21:41:52 DEBUG "metric": { 21:41:52 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 21:41:52 DEBUG "metric": { 21:41:52 DEBUG "metric": { 21:41:52 DEBUG { 21:41:52 DEBUG }, 21:41:52 DEBUG "metric": { 21:41:52 DEBUG "instance": "10.142.0.107:9100" 21:41:52 DEBUG "pod": "ds-cts-0" 21:41:52 DEBUG "pod": "am-767f6b67b5-9zw22" 21:41:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:41:52 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 21:41:52 DEBUG "metric": { 21:41:52 DEBUG "metric": { 21:41:52 DEBUG "metric": { 21:41:52 DEBUG 21:41:52 DEBUG { 21:41:52 DEBUG { 21:41:52 DEBUG "metric": { 21:41:52 DEBUG }, 21:41:52 DEBUG "metric": { 21:41:52 DEBUG }, 21:41:52 DEBUG "metric": { 21:41:52 DEBUG "pod": "ds-idrepo-0" 21:41:52 DEBUG }, 21:41:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:41:52 DEBUG }, 21:41:52 DEBUG "pod": "am-767f6b67b5-9zw22" 21:41:52 DEBUG "pod": "am-767f6b67b5-9zw22" 21:41:52 DEBUG "metric": { 21:41:52 DEBUG "value": [ 21:41:52 DEBUG "pod": "ds-idrepo-0" 21:41:52 DEBUG }, 21:41:52 DEBUG }, 21:41:52 DEBUG }, 21:41:52 DEBUG }, 21:41:52 DEBUG }, 21:41:52 DEBUG "instance": "10.142.0.107:9100" 21:41:52 DEBUG "instance": "10.142.0.107:9100" 21:41:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:41:52 DEBUG 21:41:52 DEBUG "metric": { 21:41:52 DEBUG "metric": { 21:41:52 DEBUG "pod": "ds-idrepo-0" 21:41:52 DEBUG "value": [ 21:41:52 DEBUG "pod": "ds-cts-0" 21:41:52 DEBUG "value": [ 21:41:52 DEBUG "pod": "am-767f6b67b5-9zw22" 21:41:52 DEBUG }, 21:41:52 DEBUG "value": [ 21:41:52 DEBUG }, 21:41:52 DEBUG "value": [ 21:41:52 DEBUG }, 21:41:52 DEBUG }, 21:41:52 DEBUG "pod": "am-767f6b67b5-9zw22" 21:41:52 DEBUG 21:41:52 DEBUG }, 21:41:52 DEBUG "value": [ 21:41:52 DEBUG "value": [ 21:41:52 DEBUG "value": [ 21:41:52 DEBUG "value": [ 21:41:52 DEBUG "value": [ 21:41:52 DEBUG }, 21:41:52 DEBUG }, 21:41:52 DEBUG }, 21:41:52 DEBUG 21:41:52 DEBUG "instance": "10.142.0.107:9100" 21:41:52 DEBUG "pod": "ds-cts-0" 21:41:52 DEBUG }, 21:41:52 DEBUG 21:41:52 DEBUG }, 21:41:52 DEBUG 21:41:52 DEBUG }, 21:41:52 DEBUG "value": [ 21:41:52 DEBUG 21:41:52 DEBUG "value": [ 21:41:52 DEBUG 21:41:52 DEBUG "value": [ 21:41:52 DEBUG "value": [ 21:41:52 DEBUG }, 21:41:52 DEBUG 21:41:52 DEBUG "value": [ 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG "value": [ 21:41:52 DEBUG "value": [ 21:41:52 DEBUG "value": [ 21:41:52 DEBUG [...] 21:41:52 DEBUG }, 21:41:52 DEBUG }, 21:41:52 DEBUG "value": [ 21:41:52 DEBUG 21:41:52 DEBUG "value": [ 21:41:52 DEBUG 21:41:52 DEBUG "value": [ 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG "value": [ 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG "value": [ 21:41:52 DEBUG "value": [ 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG [...] 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG [...] 21:41:52 DEBUG 21:41:52 DEBUG [...] 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG [...] 21:41:52 DEBUG 21:41:52 DEBUG [...] 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG [...] 21:41:52 DEBUG [...] 21:41:52 DEBUG [...] 21:41:52 DEBUG [...] 21:41:52 DEBUG [...] 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG [...] 21:41:52 DEBUG 21:41:52 DEBUG [...] 21:41:52 DEBUG 21:41:52 DEBUG [...] 21:41:52 DEBUG [...] 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG [...] 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG [...] 21:41:52 DEBUG [...] 21:41:52 DEBUG [...] 21:41:52 DEBUG "pod": "ds-idrepo-2" 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG [...] 21:41:52 DEBUG 21:41:52 DEBUG [...] 21:41:52 DEBUG 21:41:52 DEBUG [...] 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG }, 21:41:52 DEBUG [...] 21:41:52 DEBUG [...] 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG [...] 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG "pod": "ds-idrepo-0" 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG "pod": "am-767f6b67b5-vbltz" 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG "value": [ 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG "instance": "10.142.0.43:9100" 21:41:52 DEBUG 21:41:52 DEBUG "pod": "am-767f6b67b5-vbltz" 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG "pod": "am-767f6b67b5-vbltz" 21:41:52 DEBUG 21:41:52 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 21:41:52 DEBUG 21:41:52 DEBUG "pod": "am-767f6b67b5-vbltz" 21:41:52 DEBUG 21:41:52 DEBUG }, 21:41:52 DEBUG 21:41:52 DEBUG "instance": "10.142.0.43:9100" 21:41:52 DEBUG "pod": "ds-cts-2" 21:41:52 DEBUG "pod": "am-767f6b67b5-vbltz" 21:41:52 DEBUG 21:41:52 DEBUG }, 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 1713645712, 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG }, 21:41:52 DEBUG 21:41:52 DEBUG }, 21:41:52 DEBUG 21:41:52 DEBUG "pod": "am-767f6b67b5-vbltz" 21:41:52 DEBUG }, 21:41:52 DEBUG "pod": "am-767f6b67b5-vbltz" 21:41:52 DEBUG }, 21:41:52 DEBUG 21:41:52 DEBUG }, 21:41:52 DEBUG "pod": "am-767f6b67b5-vbltz" 21:41:52 DEBUG "value": [ 21:41:52 DEBUG "pod": "ds-cts-2" 21:41:52 DEBUG }, 21:41:52 DEBUG }, 21:41:52 DEBUG }, 21:41:52 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 21:41:52 DEBUG "value": [ 21:41:52 DEBUG "instance": "10.142.0.43:9100" 21:41:52 DEBUG "instance": "10.142.0.43:9100" 21:41:52 DEBUG "pod": "ds-idrepo-0" 21:41:52 DEBUG "0" 21:41:52 DEBUG 21:41:52 DEBUG 21:41:52 DEBUG "pod": "ds-idrepo-2" 21:41:52 DEBUG "value": [ 21:41:52 DEBUG "pod": "ds-cts-2" 21:41:52 DEBUG "value": [ 21:41:52 DEBUG "pod": "ds-idrepo-2" 21:41:52 DEBUG }, 21:41:52 DEBUG "value": [ 21:41:52 DEBUG }, 21:41:52 DEBUG "value": [ 21:41:52 DEBUG 21:41:52 DEBUG "value": [ 21:41:52 DEBUG }, 21:41:52 DEBUG 1713645712, 21:41:52 DEBUG }, 21:41:52 DEBUG "value": [ 21:41:52 DEBUG "value": [ 21:41:52 DEBUG "value": [ 21:41:52 DEBUG }, 21:41:52 DEBUG 1713645712, 21:41:52 DEBUG }, 21:41:52 DEBUG }, 21:41:52 DEBUG }, 21:41:52 DEBUG ] 21:41:52 DEBUG "instance": "10.142.0.43:9100" 21:41:52 DEBUG "pod": "ds-cts-2" 21:41:52 DEBUG }, 21:41:52 DEBUG 1713645712, 21:41:52 DEBUG }, 21:41:52 DEBUG 1713645712, 21:41:52 DEBUG }, 21:41:52 DEBUG "value": [ 21:41:52 DEBUG 1713645712, 21:41:52 DEBUG "value": [ 21:41:52 DEBUG 1713645712, 21:41:52 DEBUG "pod": "am-767f6b67b5-vbltz" 21:41:52 DEBUG 1713645712, 21:41:52 DEBUG "value": [ 21:41:52 DEBUG "4672.727901268711" 21:41:52 DEBUG "value": [ 21:41:52 DEBUG 1713645712, 21:41:52 DEBUG 1713645712, 21:41:52 DEBUG 1713645712, 21:41:52 DEBUG "value": [ 21:41:52 DEBUG "0" 21:41:52 DEBUG "value": [ 21:41:52 DEBUG "value": [ 21:41:52 DEBUG "value": [ 21:41:52 DEBUG } 21:41:52 DEBUG }, 21:41:52 DEBUG }, 21:41:52 DEBUG "value": [ 21:41:52 DEBUG "26558155.5" 21:41:52 DEBUG "value": [ 21:41:52 DEBUG "0" 21:41:52 DEBUG "value": [ 21:41:52 DEBUG 1713645712, 21:41:52 DEBUG "0" 21:41:52 DEBUG 1713645712, 21:41:52 DEBUG "658485248" 21:41:52 DEBUG }, 21:41:52 DEBUG "37922.5" 21:41:52 DEBUG 1713645712, 21:41:52 DEBUG ] 21:41:52 DEBUG 1713645712, 21:41:52 DEBUG "0" 21:41:52 DEBUG "0" 21:41:52 DEBUG "2349.1000000000004" 21:41:52 DEBUG 1713645712, 21:41:52 DEBUG ] 21:41:52 DEBUG 1713645712, 21:41:52 DEBUG 1713645712, 21:41:52 DEBUG 1713645712, 21:41:52 DEBUG ] 21:41:52 DEBUG "value": [ 21:41:52 DEBUG "value": [ 21:41:52 DEBUG 1713645712, 21:41:52 DEBUG ] 21:41:52 DEBUG 1713645712, 21:41:52 DEBUG ] 21:41:52 DEBUG 1713645712, 21:41:52 DEBUG "49.377" 21:41:52 DEBUG ] 21:41:52 DEBUG "0" 21:41:52 DEBUG ] 21:41:52 DEBUG "value": [ 21:41:52 DEBUG ] 21:41:52 DEBUG "2350.6666666666665" 21:41:52 DEBUG } 21:41:52 DEBUG "26553149" 21:41:52 DEBUG ] 21:41:52 DEBUG ] 21:41:52 DEBUG ] 21:41:52 DEBUG "3.2362736144490674" 21:41:52 DEBUG } 21:41:52 DEBUG "3944598180864" 21:41:52 DEBUG "0.08096666666668852" 21:41:52 DEBUG "0" 21:41:52 DEBUG } 21:41:52 DEBUG 1713645712, 21:41:52 DEBUG 1713645712, 21:41:52 DEBUG "0" 21:41:52 DEBUG } 21:41:52 DEBUG "0" 21:41:52 DEBUG } 21:41:52 DEBUG "0" 21:41:52 DEBUG ] 21:41:52 DEBUG } 21:41:52 DEBUG ] 21:41:52 DEBUG } 21:41:52 DEBUG 1713645712, 21:41:52 DEBUG } 21:41:52 DEBUG ] 21:41:52 DEBUG ] 21:41:52 DEBUG ] 21:41:52 DEBUG } 21:41:52 DEBUG } 21:41:52 DEBUG } 21:41:52 DEBUG ] 21:41:52 DEBUG ] 21:41:52 DEBUG ] 21:41:52 DEBUG ] 21:41:52 DEBUG ] 21:41:52 DEBUG } 21:41:52 DEBUG "0.052333333333333114" 21:41:52 DEBUG "0" 21:41:52 DEBUG ] 21:41:52 DEBUG ] 21:41:52 DEBUG } 21:41:52 DEBUG ] 21:41:52 DEBUG ] 21:41:52 DEBUG } 21:41:52 DEBUG ] 21:41:52 DEBUG } 21:41:52 DEBUG ] 21:41:52 DEBUG "2350.3666666666672" 21:41:52 DEBUG ] 21:41:52 DEBUG } 21:41:52 DEBUG } 21:41:52 DEBUG } 21:41:52 DEBUG ] 21:41:52 DEBUG ] 21:41:52 DEBUG ] 21:41:52 DEBUG } 21:41:52 DEBUG } 21:41:52 DEBUG } 21:41:52 DEBUG } 21:41:52 DEBUG } 21:41:52 DEBUG ] 21:41:52 DEBUG } 21:41:52 DEBUG ] 21:41:52 DEBUG ] 21:41:52 DEBUG } 21:41:52 DEBUG } 21:41:52 DEBUG } 21:41:52 DEBUG ] 21:41:52 DEBUG } 21:41:52 DEBUG ] 21:41:52 DEBUG } 21:41:52 DEBUG } 21:41:52 DEBUG ] 21:41:52 DEBUG ] 21:41:52 DEBUG } 21:41:52 DEBUG ] 21:41:52 DEBUG } 21:41:52 DEBUG } 21:41:52 DEBUG } 21:41:52 DEBUG ] 21:41:52 DEBUG } 21:41:52 DEBUG ] 21:41:52 DEBUG ] 21:41:52 DEBUG ] 21:41:52 DEBUG } 21:41:52 DEBUG ] 21:41:52 DEBUG } 21:41:52 DEBUG } 21:41:52 DEBUG } 21:41:52 DEBUG } 21:41:52 DEBUG } 21:41:52 DEBUG } 21:41:52 DEBUG } 21:41:52 DEBUG ] 21:41:52 DEBUG } 21:41:52 DEBUG } 21:41:52 DEBUG } 21:41:52 DEBUG } 21:41:52 DEBUG } 21:41:52 DEBUG } 21:41:52 DEBUG } 21:41:52 DEBUG } 21:41:52 DEBUG } 21:41:52 DEBUG } 21:41:52 DEBUG ] 21:41:52 DEBUG } 21:41:52 DEBUG ] 21:41:52 DEBUG ] 21:41:52 DEBUG ] 21:41:52 DEBUG } 21:41:52 DEBUG } 21:41:52 DEBUG } 21:41:52 DEBUG } 21:41:52 DEBUG } 21:41:52 DEBUG } 21:41:52 DEBUG } 21:41:52 DEBUG } 21:41:52 DEBUG } 21:41:52 DEBUG } 21:41:52 DEBUG } 21:41:52 DEBUG } 21:41:52 DEBUG } 21:41:52 DEBUG } 21:41:52 DEBUG } 21:41:52 DEBUG } 21:41:52 DEBUG } 21:41:52 DEBUG } 21:41:52 DEBUG } 21:41:52 DEBUG } 21:42:52 INFO 21:42:52 INFO [loop_until]: kubectl --namespace=xlou top pods 21:42:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:42:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645772" 21:42:52 INFO 21:42:52 INFO [loop_until]: kubectl --namespace=xlou top node 21:42:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:42:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713645772" 21:42:52 INFO [http_cmd]: http status code OK 21:42:52 DEBUG --- status code --- 21:42:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:42:52 DEBUG --- http response --- 21:42:52 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 21:42:52 ERROR [http_cmd]: ERROR 21:42:52 ERROR --- status code --- 21:42:52 ERROR http status code is 400 (expected [200, 401, 403, 404]) 21:42:52 ERROR --- http response --- 21:42:52 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 21:42:52 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 21:42:52 INFO [loop_until]: OK (rc = 0) 21:42:52 DEBUG --- stdout --- 21:42:52 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17322m 8719Mi am-767f6b67b5-qcqz2 16686m 8833Mi am-767f6b67b5-vbltz 18943m 8864Mi ds-cts-0 9951m 8673Mi ds-cts-1 9185m 9325Mi ds-cts-2 10104m 10267Mi ds-idrepo-0 8421m 3984Mi ds-idrepo-1 8102m 3980Mi ds-idrepo-2 8482m 3889Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-7dd48676f6-tdxl9 2m 61Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-967d957f7-zk66v 11714m 622Mi 21:42:52 DEBUG --- stderr --- 21:42:52 DEBUG 21:42:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645772" 21:42:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645772" 21:42:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645772" 21:42:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713645772" 21:42:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645772" 21:42:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645772" 21:42:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713645772" 21:42:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645772" 21:42:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645772" 21:42:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645772" 21:42:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713645772" 21:42:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645772" 21:42:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713645772" 21:42:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645772" 21:42:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713645772" 21:42:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645772" 21:42:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713645772" 21:42:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713645772" 21:42:52 INFO [http_cmd]: http status code OK 21:42:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713645772" 21:42:52 INFO [http_cmd]: http status code OK 21:42:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645772" 21:42:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645772" 21:42:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713645772" 21:42:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645772" 21:42:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645772" 21:42:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645772" 21:42:52 INFO [http_cmd]: http status code OK 21:42:52 INFO [http_cmd]: http status code OK 21:42:52 INFO [http_cmd]: http status code OK 21:42:52 INFO [http_cmd]: http status code OK 21:42:52 DEBUG --- status code --- 21:42:52 INFO [loop_until]: OK (rc = 0) 21:42:52 INFO [http_cmd]: http status code OK 21:42:52 INFO [http_cmd]: http status code OK 21:42:52 INFO [http_cmd]: http status code OK 21:42:52 DEBUG --- status code --- 21:42:52 INFO [http_cmd]: http status code OK 21:42:52 INFO [http_cmd]: http status code OK 21:42:52 INFO [http_cmd]: http status code OK 21:42:52 DEBUG --- status code --- 21:42:52 INFO [http_cmd]: http status code OK 21:42:52 INFO [http_cmd]: http status code OK 21:42:52 INFO [http_cmd]: http status code OK 21:42:52 INFO [http_cmd]: http status code OK 21:42:52 INFO [http_cmd]: http status code OK 21:42:52 DEBUG --- status code --- 21:42:52 INFO [http_cmd]: http status code OK 21:42:52 DEBUG --- status code --- 21:42:52 DEBUG --- status code --- 21:42:52 INFO [http_cmd]: http status code OK 21:42:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:42:52 DEBUG --- stdout --- 21:42:52 DEBUG --- status code --- 21:42:52 INFO [http_cmd]: http status code OK 21:42:52 INFO [http_cmd]: http status code OK 21:42:52 DEBUG --- status code --- 21:42:52 DEBUG --- status code --- 21:42:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:42:52 DEBUG --- status code --- 21:42:52 DEBUG --- status code --- 21:42:52 DEBUG --- status code --- 21:42:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:42:52 INFO [http_cmd]: http status code OK 21:42:52 DEBUG --- status code --- 21:42:52 DEBUG --- status code --- 21:42:52 INFO [http_cmd]: http status code OK 21:42:52 DEBUG --- status code --- 21:42:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:42:52 INFO [http_cmd]: http status code OK 21:42:52 DEBUG --- status code --- 21:42:52 DEBUG --- status code --- 21:42:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:42:52 DEBUG --- status code --- 21:42:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:42:52 DEBUG --- http response --- 21:42:52 DEBUG --- status code --- 21:42:52 DEBUG --- http response --- 21:42:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:42:52 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6214Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8598m 28% 7737Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 17541m 58% 11443Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 84m 0% 3386Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 9815m 32% 11924Mi 10% gke-xlou-cdm-default-pool-1b0769c8-wpwl 141m 0% 5230Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 18000m 60% 11712Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19348m 64% 3096Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 20129m 67% 11085Mi 9% gke-xlou-cdm-ds-0f601751-j488 8936m 29% 7286Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 9258m 31% 14448Mi 12% gke-xlou-cdm-ds-0f601751-nj6q 9072m 30% 7968Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 9976m 33% 12992Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 12091m 76% 8587Mi 14% 21:42:52 DEBUG --- status code --- 21:42:52 DEBUG --- status code --- 21:42:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:42:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:42:52 DEBUG --- http response --- 21:42:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:42:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:42:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:42:52 DEBUG --- http response --- 21:42:52 DEBUG --- status code --- 21:42:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:42:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:42:52 DEBUG --- status code --- 21:42:52 INFO [http_cmd]: http status code OK 21:42:52 DEBUG --- http response --- 21:42:52 DEBUG --- status code --- 21:42:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:42:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:42:52 DEBUG --- http response --- 21:42:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:42:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:42:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:42:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:42:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 21:42:52 DEBUG --- http response --- 21:42:52 DEBUG --- stderr --- 21:42:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:42:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:42:52 DEBUG --- http response --- 21:42:52 DEBUG --- http response --- 21:42:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:42:52 DEBUG --- http response --- 21:42:52 DEBUG --- http response --- 21:42:52 DEBUG --- http response --- 21:42:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:42:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:42:52 DEBUG --- http response --- 21:42:52 DEBUG --- http response --- 21:42:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:42:52 DEBUG --- status code --- 21:42:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:42:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:42:52 DEBUG --- http response --- 21:42:52 DEBUG --- http response --- 21:42:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:42:52 DEBUG --- http response --- 21:42:52 DEBUG --- http response --- 21:42:52 DEBUG ----- output ----- 21:42:52 DEBUG --- http response --- 21:42:52 DEBUG ----- output ----- 21:42:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:42:52 DEBUG 21:42:52 DEBUG --- http response --- 21:42:52 DEBUG --- http response --- 21:42:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:42:52 DEBUG ----- output ----- 21:42:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:42:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:42:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:42:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:42:52 DEBUG --- http response --- 21:42:52 DEBUG ----- output ----- 21:42:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:42:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:42:52 DEBUG --- http response --- 21:42:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:42:52 DEBUG ----- output ----- 21:42:52 DEBUG --- http response --- 21:42:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:42:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:42:52 DEBUG ----- output ----- 21:42:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:42:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:42:52 DEBUG { 21:42:52 DEBUG { 21:42:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:42:52 DEBUG ----- output ----- 21:42:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 21:42:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:42:52 DEBUG ----- output ----- 21:42:52 DEBUG { 21:42:52 DEBUG ----- output ----- 21:42:52 DEBUG ----- output ----- 21:42:52 DEBUG ----- output ----- 21:42:52 DEBUG ----- output ----- 21:42:52 DEBUG { 21:42:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:42:52 DEBUG ----- output ----- 21:42:52 DEBUG ----- output ----- 21:42:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:42:52 DEBUG --- http response --- 21:42:52 DEBUG { 21:42:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:42:52 DEBUG ----- output ----- 21:42:52 DEBUG ----- output ----- 21:42:52 DEBUG { 21:42:52 DEBUG ----- output ----- 21:42:52 DEBUG ----- output ----- 21:42:52 DEBUG "status": "success", 21:42:52 DEBUG "status": "success", 21:42:52 DEBUG ----- output ----- 21:42:52 DEBUG { 21:42:52 DEBUG { 21:42:52 DEBUG ----- output ----- 21:42:52 DEBUG { 21:42:52 DEBUG "status": "success", 21:42:52 DEBUG { 21:42:52 DEBUG { 21:42:52 DEBUG { 21:42:52 DEBUG { 21:42:52 DEBUG "status": "success", 21:42:52 DEBUG ----- output ----- 21:42:52 DEBUG { 21:42:52 DEBUG { 21:42:52 DEBUG ----- output ----- 21:42:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:42:52 DEBUG "status": "success", 21:42:52 DEBUG ----- output ----- 21:42:52 DEBUG { 21:42:52 DEBUG { 21:42:52 DEBUG "status": "success", 21:42:52 DEBUG { 21:42:52 DEBUG { 21:42:52 DEBUG "data": { 21:42:52 DEBUG "data": { 21:42:52 DEBUG ----- output ----- 21:42:52 DEBUG "status": "success", 21:42:52 DEBUG "status": "success", 21:42:52 DEBUG { 21:42:52 DEBUG "status": "success", 21:42:52 DEBUG "data": { 21:42:52 DEBUG "status": "success", 21:42:52 DEBUG "status": "success", 21:42:52 DEBUG "status": "success", 21:42:52 DEBUG "status": "success", 21:42:52 DEBUG "data": { 21:42:52 DEBUG { 21:42:52 DEBUG "status": "success", 21:42:52 DEBUG "status": "success", 21:42:52 DEBUG { 21:42:52 DEBUG ----- output ----- 21:42:52 DEBUG "data": { 21:42:52 DEBUG { 21:42:52 DEBUG "status": "success", 21:42:52 DEBUG "status": "success", 21:42:52 DEBUG "data": { 21:42:52 DEBUG "status": "success", 21:42:52 DEBUG "status": "success", 21:42:52 DEBUG "resultType": "vector", 21:42:52 DEBUG "resultType": "vector", 21:42:52 DEBUG { 21:42:52 DEBUG "data": { 21:42:52 DEBUG "data": { 21:42:52 DEBUG "status": "success", 21:42:52 DEBUG "data": { 21:42:52 DEBUG "resultType": "vector", 21:42:52 DEBUG "data": { 21:42:52 DEBUG "data": { 21:42:52 DEBUG "data": { 21:42:52 DEBUG "resultType": "vector", 21:42:52 DEBUG "resultType": "vector", 21:42:52 DEBUG "status": "success", 21:42:52 DEBUG "data": { 21:42:52 DEBUG "data": { 21:42:52 DEBUG "status": "success", 21:42:52 DEBUG { 21:42:52 DEBUG "resultType": "vector", 21:42:52 DEBUG "status": "success", 21:42:52 DEBUG "data": { 21:42:52 DEBUG "data": { 21:42:52 DEBUG "resultType": "vector", 21:42:52 DEBUG "data": { 21:42:52 DEBUG "data": { 21:42:52 DEBUG "result": [ 21:42:52 DEBUG "result": [ 21:42:52 DEBUG "status": "success", 21:42:52 DEBUG "resultType": "vector", 21:42:52 DEBUG "resultType": "vector", 21:42:52 DEBUG "result": [ 21:42:52 DEBUG "data": { 21:42:52 DEBUG "result": [ 21:42:52 DEBUG "resultType": "vector", 21:42:52 DEBUG "resultType": "vector", 21:42:52 DEBUG "data": { 21:42:52 DEBUG "result": [ 21:42:52 DEBUG "result": [ 21:42:52 DEBUG "data": { 21:42:52 DEBUG "resultType": "vector", 21:42:52 DEBUG "resultType": "vector", 21:42:52 DEBUG "data": { 21:42:52 DEBUG "status": "success", 21:42:52 DEBUG "result": [ 21:42:52 DEBUG "data": { 21:42:52 DEBUG "resultType": "vector", 21:42:52 DEBUG "resultType": "vector", 21:42:52 DEBUG "result": [ 21:42:52 DEBUG "resultType": "vector", 21:42:52 DEBUG "resultType": "vector", 21:42:52 DEBUG { 21:42:52 DEBUG { 21:42:52 DEBUG "data": { 21:42:52 DEBUG "result": [ 21:42:52 DEBUG "resultType": "vector", 21:42:52 DEBUG { 21:42:52 DEBUG "resultType": "vector", 21:42:52 DEBUG { 21:42:52 DEBUG "result": [ 21:42:52 DEBUG "result": [ 21:42:52 DEBUG "resultType": "vector", 21:42:52 DEBUG { 21:42:52 DEBUG { 21:42:52 DEBUG "resultType": "vector", 21:42:52 DEBUG "result": [ 21:42:52 DEBUG "result": [ 21:42:52 DEBUG "resultType": "vector", 21:42:52 DEBUG "data": { 21:42:52 DEBUG { 21:42:52 DEBUG "resultType": "vector", 21:42:52 DEBUG "result": [ 21:42:52 DEBUG "result": [ 21:42:52 DEBUG { 21:42:52 DEBUG "result": [ 21:42:52 DEBUG "result": [ 21:42:52 DEBUG "metric": { 21:42:52 DEBUG "metric": { 21:42:52 DEBUG "resultType": "vector", 21:42:52 DEBUG { 21:42:52 DEBUG "result": [ 21:42:52 DEBUG "metric": { 21:42:52 DEBUG "result": [ 21:42:52 DEBUG "metric": { 21:42:52 DEBUG { 21:42:52 DEBUG { 21:42:52 DEBUG "result": [ 21:42:52 DEBUG "metric": { 21:42:52 DEBUG "metric": { 21:42:52 DEBUG "result": [ 21:42:52 DEBUG { 21:42:52 DEBUG { 21:42:52 DEBUG "result": [ 21:42:52 DEBUG "resultType": "vector", 21:42:52 DEBUG "metric": { 21:42:52 DEBUG "result": [ 21:42:52 DEBUG { 21:42:52 DEBUG { 21:42:52 DEBUG "metric": { 21:42:52 DEBUG { 21:42:52 DEBUG { 21:42:52 DEBUG "pod": "am-767f6b67b5-9zw22" 21:42:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:42:52 DEBUG "result": [ 21:42:52 DEBUG "metric": { 21:42:52 DEBUG { 21:42:52 DEBUG "pod": "ds-cts-0" 21:42:52 DEBUG { 21:42:52 DEBUG "pod": "ds-idrepo-0" 21:42:52 DEBUG "metric": { 21:42:52 DEBUG "metric": { 21:42:52 DEBUG { 21:42:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:42:52 DEBUG "pod": "ds-cts-0" 21:42:52 DEBUG { 21:42:52 DEBUG "metric": { 21:42:52 DEBUG "metric": { 21:42:52 DEBUG { 21:42:52 DEBUG "result": [ 21:42:52 DEBUG "instance": "10.142.0.107:9100" 21:42:52 DEBUG { 21:42:52 DEBUG "metric": { 21:42:52 DEBUG "metric": { 21:42:52 DEBUG "instance": "10.142.0.107:9100" 21:42:52 DEBUG "metric": { 21:42:52 DEBUG "metric": { 21:42:52 DEBUG }, 21:42:52 DEBUG }, 21:42:52 DEBUG { 21:42:52 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 21:42:52 DEBUG "metric": { 21:42:52 DEBUG }, 21:42:52 DEBUG "metric": { 21:42:52 DEBUG }, 21:42:52 DEBUG "pod": "am-767f6b67b5-9zw22" 21:42:52 DEBUG "pod": "ds-idrepo-0" 21:42:52 DEBUG "metric": { 21:42:52 DEBUG }, 21:42:52 DEBUG }, 21:42:52 DEBUG "metric": { 21:42:52 DEBUG "pod": "am-767f6b67b5-9zw22" 21:42:52 DEBUG "pod": "am-767f6b67b5-9zw22" 21:42:52 DEBUG "metric": { 21:42:52 DEBUG { 21:42:52 DEBUG }, 21:42:52 DEBUG "metric": { 21:42:52 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 21:42:52 DEBUG "instance": "10.142.0.107:9100" 21:42:52 DEBUG }, 21:42:52 DEBUG "instance": "10.142.0.107:9100" 21:42:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:42:52 DEBUG "value": [ 21:42:52 DEBUG "value": [ 21:42:52 DEBUG "metric": { 21:42:52 DEBUG }, 21:42:52 DEBUG "pod": "am-767f6b67b5-9zw22" 21:42:52 DEBUG "value": [ 21:42:52 DEBUG "pod": "am-767f6b67b5-9zw22" 21:42:52 DEBUG "value": [ 21:42:52 DEBUG }, 21:42:52 DEBUG }, 21:42:52 DEBUG "pod": "am-767f6b67b5-9zw22" 21:42:52 DEBUG "value": [ 21:42:52 DEBUG "value": [ 21:42:52 DEBUG "pod": "ds-idrepo-0" 21:42:52 DEBUG }, 21:42:52 DEBUG }, 21:42:52 DEBUG "pod": "ds-cts-0" 21:42:52 DEBUG "metric": { 21:42:52 DEBUG "value": [ 21:42:52 DEBUG "pod": "ds-idrepo-0" 21:42:52 DEBUG }, 21:42:52 DEBUG }, 21:42:52 DEBUG "value": [ 21:42:52 DEBUG }, 21:42:52 DEBUG }, 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:42:52 DEBUG "value": [ 21:42:52 DEBUG }, 21:42:52 DEBUG 21:42:52 DEBUG }, 21:42:52 DEBUG 21:42:52 DEBUG "value": [ 21:42:52 DEBUG "value": [ 21:42:52 DEBUG }, 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG }, 21:42:52 DEBUG "value": [ 21:42:52 DEBUG "value": [ 21:42:52 DEBUG 21:42:52 DEBUG "instance": "10.142.0.107:9100" 21:42:52 DEBUG 21:42:52 DEBUG }, 21:42:52 DEBUG "value": [ 21:42:52 DEBUG "value": [ 21:42:52 DEBUG 21:42:52 DEBUG "value": [ 21:42:52 DEBUG "value": [ 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG }, 21:42:52 DEBUG 21:42:52 DEBUG "value": [ 21:42:52 DEBUG 21:42:52 DEBUG "value": [ 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG "value": [ 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG "value": [ 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG }, 21:42:52 DEBUG }, 21:42:52 DEBUG 21:42:52 DEBUG "value": [ 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG "value": [ 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG "value": [ 21:42:52 DEBUG "value": [ 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG [...] 21:42:52 DEBUG [...] 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG [...] 21:42:52 DEBUG 21:42:52 DEBUG [...] 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG [...] 21:42:52 DEBUG [...] 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG [...] 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG [...] 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG [...] 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG [...] 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG [...] 21:42:52 DEBUG 21:42:52 DEBUG [...] 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG [...] 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG [...] 21:42:52 DEBUG [...] 21:42:52 DEBUG 21:42:52 DEBUG [...] 21:42:52 DEBUG [...] 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG "pod": "am-767f6b67b5-vbltz" 21:42:52 DEBUG }, 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG [...] 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG [...] 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG [...] 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG [...] 21:42:52 DEBUG "value": [ 21:42:52 DEBUG 1713645772, 21:42:52 DEBUG "0" 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG [...] 21:42:52 DEBUG 21:42:52 DEBUG "pod": "am-767f6b67b5-vbltz" 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG [...] 21:42:52 DEBUG [...] 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG [...] 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG "pod": "ds-cts-2" 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG "pod": "ds-idrepo-2" 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG }, 21:42:52 DEBUG 21:42:52 DEBUG "pod": "ds-cts-2" 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG "pod": "am-767f6b67b5-vbltz" 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG "instance": "10.142.0.43:9100" 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG "instance": "10.142.0.43:9100" 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 21:42:52 DEBUG 21:42:52 DEBUG ] 21:42:52 DEBUG }, 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG }, 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG "value": [ 21:42:52 DEBUG "pod": "ds-cts-2" 21:42:52 DEBUG }, 21:42:52 DEBUG 21:42:52 DEBUG "pod": "am-767f6b67b5-vbltz" 21:42:52 DEBUG }, 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG }, 21:42:52 DEBUG 21:42:52 DEBUG "pod": "ds-idrepo-2" 21:42:52 DEBUG "instance": "10.142.0.43:9100" 21:42:52 DEBUG }, 21:42:52 DEBUG "instance": "10.142.0.43:9100" 21:42:52 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 21:42:52 DEBUG 21:42:52 DEBUG }, 21:42:52 DEBUG 21:42:52 DEBUG } 21:42:52 DEBUG "value": [ 21:42:52 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 21:42:52 DEBUG "pod": "am-767f6b67b5-vbltz" 21:42:52 DEBUG "value": [ 21:42:52 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 21:42:52 DEBUG "pod": "am-767f6b67b5-vbltz" 21:42:52 DEBUG 1713645772, 21:42:52 DEBUG }, 21:42:52 DEBUG "value": [ 21:42:52 DEBUG "pod": "ds-idrepo-2" 21:42:52 DEBUG }, 21:42:52 DEBUG "value": [ 21:42:52 DEBUG 21:42:52 DEBUG 21:42:52 DEBUG "value": [ 21:42:52 DEBUG 1713645772, 21:42:52 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 21:42:52 DEBUG }, 21:42:52 DEBUG "value": [ 21:42:52 DEBUG }, 21:42:52 DEBUG "value": [ 21:42:52 DEBUG 21:42:52 DEBUG "value": [ 21:42:52 DEBUG "pod": "am-767f6b67b5-vbltz" 21:42:52 DEBUG ] 21:42:52 DEBUG 1713645772, 21:42:52 DEBUG }, 21:42:52 DEBUG }, 21:42:52 DEBUG 1713645772, 21:42:52 DEBUG }, 21:42:52 DEBUG }, 21:42:52 DEBUG "3335.3000000000006" 21:42:52 DEBUG "value": [ 21:42:52 DEBUG 1713645772, 21:42:52 DEBUG "0" 21:42:52 DEBUG ] 21:42:52 DEBUG } 21:42:52 DEBUG "pod": "ds-cts-2" 21:42:52 DEBUG "instance": "10.142.0.43:9100" 21:42:52 DEBUG }, 21:42:52 DEBUG "0.06526666666667798" 21:42:52 DEBUG }, 21:42:52 DEBUG "value": [ 21:42:52 DEBUG 1713645772, 21:42:52 DEBUG }, 21:42:52 DEBUG 1713645772, 21:42:52 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 21:42:52 DEBUG 1713645772, 21:42:52 DEBUG }, 21:42:52 DEBUG } 21:42:52 DEBUG "0" 21:42:52 DEBUG "value": [ 21:42:52 DEBUG "value": [ 21:42:52 DEBUG "0" 21:42:52 DEBUG "value": [ 21:42:52 DEBUG "value": [ 21:42:52 DEBUG 1713645772, 21:42:52 DEBUG "49.377" 21:42:52 DEBUG ] 21:42:52 DEBUG "value": [ 21:42:52 DEBUG 1713645772, 21:42:52 DEBUG ] 21:42:52 DEBUG }, 21:42:52 DEBUG }, 21:42:52 DEBUG "value": [ 21:42:52 DEBUG ] 21:42:52 DEBUG "value": [ 21:42:52 DEBUG 1713645772, 21:42:52 DEBUG "79937502.16666666" 21:42:52 DEBUG "value": [ 21:42:52 DEBUG "3947532540928" 21:42:52 DEBUG }, 21:42:52 DEBUG "0" 21:42:52 DEBUG "value": [ 21:42:52 DEBUG } 21:42:52 DEBUG ] 21:42:52 DEBUG 1713645772, 21:42:52 DEBUG 1713645772, 21:42:52 DEBUG ] 21:42:52 DEBUG 1713645772, 21:42:52 DEBUG ] 21:42:52 DEBUG }, 21:42:52 DEBUG 1713645772, 21:42:52 DEBUG } 21:42:52 DEBUG 1713645772, 21:42:52 DEBUG "223144.5" 21:42:52 DEBUG } 21:42:52 DEBUG "value": [ 21:42:52 DEBUG "value": [ 21:42:52 DEBUG 1713645772, 21:42:52 DEBUG } 21:42:52 DEBUG 1713645772, 21:42:52 DEBUG "79941474.26666667" 21:42:52 DEBUG ] 21:42:52 DEBUG 1713645772, 21:42:52 DEBUG ] 21:42:52 DEBUG "value": [ 21:42:52 DEBUG ] 21:42:52 DEBUG 1713645772, 21:42:52 DEBUG } 21:42:52 DEBUG "750673920" 21:42:52 DEBUG "3335.466666666667" 21:42:52 DEBUG } 21:42:52 DEBUG "0" 21:42:52 DEBUG } 21:42:52 DEBUG "value": [ 21:42:52 DEBUG "0.00013266" 21:42:52 DEBUG ] 21:42:52 DEBUG "3335.3" 21:42:52 DEBUG ] 21:42:52 DEBUG } 21:42:52 DEBUG 1713645772, 21:42:52 DEBUG 1713645772, 21:42:52 DEBUG "0" 21:42:52 DEBUG ] 21:42:52 DEBUG "11.85466254393334" 21:42:52 DEBUG ] 21:42:52 DEBUG } 21:42:52 DEBUG "0.000022878002646164212" 21:42:52 DEBUG } 21:42:52 DEBUG 1713645772, 21:42:52 DEBUG } 21:42:52 DEBUG "0" 21:42:52 DEBUG ] 21:42:52 DEBUG ] 21:42:52 DEBUG ] 21:42:52 DEBUG } 21:42:52 DEBUG ] 21:42:52 DEBUG ] 21:42:52 DEBUG 1713645772, 21:42:52 DEBUG ] 21:42:52 DEBUG } 21:42:52 DEBUG ] 21:42:52 DEBUG } 21:42:52 DEBUG "0" 21:42:52 DEBUG "0.04633333333333098" 21:42:52 DEBUG ] 21:42:52 DEBUG } 21:42:52 DEBUG ] 21:42:52 DEBUG } 21:42:52 DEBUG ] 21:42:52 DEBUG } 21:42:52 DEBUG ] 21:42:52 DEBUG "0" 21:42:52 DEBUG ] 21:42:52 DEBUG ] 21:42:52 DEBUG } 21:42:52 DEBUG } 21:42:52 DEBUG ] 21:42:52 DEBUG } 21:42:52 DEBUG } 21:42:52 DEBUG } 21:42:52 DEBUG "0" 21:42:52 DEBUG } 21:42:52 DEBUG } 21:42:52 DEBUG } 21:42:52 DEBUG ] 21:42:52 DEBUG ] 21:42:52 DEBUG ] 21:42:52 DEBUG } 21:42:52 DEBUG ] 21:42:52 DEBUG } 21:42:52 DEBUG ] 21:42:52 DEBUG } 21:42:52 DEBUG } 21:42:52 DEBUG ] 21:42:52 DEBUG ] 21:42:52 DEBUG } 21:42:52 DEBUG } 21:42:52 DEBUG } 21:42:52 DEBUG ] 21:42:52 DEBUG ] 21:42:52 DEBUG } 21:42:52 DEBUG ] 21:42:52 DEBUG } 21:42:52 DEBUG ] 21:42:52 DEBUG ] 21:42:52 DEBUG ] 21:42:52 DEBUG } 21:42:52 DEBUG } 21:42:52 DEBUG ] 21:42:52 DEBUG } 21:42:52 DEBUG ] 21:42:52 DEBUG } 21:42:52 DEBUG } 21:42:52 DEBUG } 21:42:52 DEBUG } 21:42:52 DEBUG } 21:42:52 DEBUG } 21:42:52 DEBUG ] 21:42:52 DEBUG } 21:42:52 DEBUG } 21:42:52 DEBUG } 21:42:52 DEBUG } 21:42:52 DEBUG } 21:42:52 DEBUG } 21:42:52 DEBUG } 21:42:52 DEBUG ] 21:42:52 DEBUG } 21:42:52 DEBUG } 21:42:52 DEBUG } 21:42:52 DEBUG } 21:42:52 DEBUG } 21:42:52 DEBUG ] 21:42:52 DEBUG ] 21:42:52 DEBUG } 21:42:52 DEBUG } 21:42:52 DEBUG } 21:42:52 DEBUG } 21:42:52 DEBUG ] 21:42:52 DEBUG } 21:42:52 DEBUG } 21:42:52 DEBUG } 21:42:52 DEBUG } 21:42:52 DEBUG } 21:42:52 DEBUG } 21:42:52 DEBUG } 21:42:52 DEBUG } 21:42:52 DEBUG } 21:42:52 DEBUG } 21:42:52 DEBUG } 21:42:52 DEBUG } 21:42:52 DEBUG } 21:43:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645832" 21:43:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713645832" 21:43:52 INFO [http_cmd]: http status code OK 21:43:52 DEBUG --- status code --- 21:43:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:43:52 DEBUG --- http response --- 21:43:52 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 21:43:52 ERROR [http_cmd]: ERROR 21:43:52 ERROR --- status code --- 21:43:52 ERROR http status code is 400 (expected [200, 401, 403, 404]) 21:43:52 ERROR --- http response --- 21:43:52 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 21:43:52 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 21:43:52 INFO 21:43:52 INFO [loop_until]: kubectl --namespace=xlou top pods 21:43:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:43:52 INFO [loop_until]: OK (rc = 0) 21:43:52 DEBUG --- stdout --- 21:43:52 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17985m 8729Mi am-767f6b67b5-qcqz2 17310m 8844Mi am-767f6b67b5-vbltz 19198m 8868Mi ds-cts-0 10142m 8742Mi ds-cts-1 9515m 9396Mi ds-cts-2 10129m 10160Mi ds-idrepo-0 8277m 3984Mi ds-idrepo-1 8204m 3980Mi ds-idrepo-2 8170m 3889Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-7dd48676f6-tdxl9 15m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-967d957f7-zk66v 11642m 631Mi 21:43:52 DEBUG --- stderr --- 21:43:52 DEBUG 21:43:52 INFO 21:43:52 INFO [loop_until]: kubectl --namespace=xlou top node 21:43:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:43:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713645832" 21:43:52 INFO [http_cmd]: http status code OK 21:43:52 DEBUG --- status code --- 21:43:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:43:52 DEBUG --- http response --- 21:43:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:43:52 DEBUG ----- output ----- 21:43:52 DEBUG { 21:43:52 DEBUG "status": "success", 21:43:52 DEBUG "data": { 21:43:52 DEBUG "resultType": "vector", 21:43:52 DEBUG "result": [ 21:43:52 DEBUG { 21:43:52 DEBUG "metric": { 21:43:52 DEBUG "pod": "ds-idrepo-0" 21:43:52 DEBUG }, 21:43:52 DEBUG "value": [ 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG [...] 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG "pod": "ds-cts-2" 21:43:52 DEBUG }, 21:43:52 DEBUG "value": [ 21:43:52 DEBUG 1713645832, 21:43:52 DEBUG "0" 21:43:52 DEBUG ] 21:43:52 DEBUG } 21:43:52 DEBUG ] 21:43:52 DEBUG } 21:43:52 DEBUG } 21:43:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645832" 21:43:52 INFO [http_cmd]: http status code OK 21:43:52 DEBUG --- status code --- 21:43:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:43:52 DEBUG --- http response --- 21:43:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 21:43:52 DEBUG ----- output ----- 21:43:52 DEBUG { 21:43:52 DEBUG "status": "success", 21:43:52 DEBUG "data": { 21:43:52 DEBUG "resultType": "vector", 21:43:52 DEBUG "result": [ 21:43:52 DEBUG { 21:43:52 DEBUG "metric": { 21:43:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:43:52 DEBUG }, 21:43:52 DEBUG "value": [ 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG [...] 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG "pod": "ds-cts-0" 21:43:52 DEBUG }, 21:43:52 DEBUG "value": [ 21:43:52 DEBUG 1713645832, 21:43:52 DEBUG "128302.14020929983" 21:43:52 DEBUG ] 21:43:52 DEBUG } 21:43:52 DEBUG ] 21:43:52 DEBUG } 21:43:52 DEBUG } 21:43:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645832" 21:43:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713645832" 21:43:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713645832" 21:43:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645832" 21:43:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645832" 21:43:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645832" 21:43:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713645832" 21:43:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713645832" 21:43:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713645832" 21:43:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713645832" 21:43:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645832" 21:43:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645832" 21:43:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645832" 21:43:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645832" 21:43:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713645832" 21:43:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645832" 21:43:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645832" 21:43:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645832" 21:43:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645832" 21:43:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645832" 21:43:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645832" 21:43:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713645832" 21:43:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645832" 21:43:52 INFO [http_cmd]: http status code OK 21:43:52 INFO [loop_until]: OK (rc = 0) 21:43:52 INFO [http_cmd]: http status code OK 21:43:52 INFO [http_cmd]: http status code OK 21:43:52 INFO [http_cmd]: http status code OK 21:43:52 INFO [http_cmd]: http status code OK 21:43:52 INFO [http_cmd]: http status code OK 21:43:52 INFO [http_cmd]: http status code OK 21:43:52 INFO [http_cmd]: http status code OK 21:43:52 INFO [http_cmd]: http status code OK 21:43:52 INFO [http_cmd]: http status code OK 21:43:52 INFO [http_cmd]: http status code OK 21:43:52 INFO [http_cmd]: http status code OK 21:43:52 INFO [http_cmd]: http status code OK 21:43:52 INFO [http_cmd]: http status code OK 21:43:52 INFO [http_cmd]: http status code OK 21:43:52 DEBUG --- status code --- 21:43:52 INFO [http_cmd]: http status code OK 21:43:52 DEBUG --- stdout --- 21:43:52 INFO [http_cmd]: http status code OK 21:43:52 INFO [http_cmd]: http status code OK 21:43:52 INFO [http_cmd]: http status code OK 21:43:52 INFO [http_cmd]: http status code OK 21:43:52 DEBUG --- status code --- 21:43:52 DEBUG --- status code --- 21:43:52 INFO [http_cmd]: http status code OK 21:43:52 DEBUG --- status code --- 21:43:52 DEBUG --- status code --- 21:43:52 DEBUG --- status code --- 21:43:52 DEBUG --- status code --- 21:43:52 DEBUG --- status code --- 21:43:52 DEBUG --- status code --- 21:43:52 DEBUG --- status code --- 21:43:52 DEBUG --- status code --- 21:43:52 DEBUG --- status code --- 21:43:52 DEBUG --- status code --- 21:43:52 DEBUG --- status code --- 21:43:52 DEBUG --- status code --- 21:43:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:43:52 DEBUG --- status code --- 21:43:52 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 6215Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8547m 28% 7749Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 17569m 58% 11442Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 81m 0% 3386Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 10463m 35% 11982Mi 10% gke-xlou-cdm-default-pool-1b0769c8-wpwl 158m 0% 5210Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 18002m 60% 11724Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19499m 65% 3070Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 20068m 67% 11102Mi 9% gke-xlou-cdm-ds-0f601751-j488 9233m 30% 7305Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 9579m 32% 14497Mi 12% gke-xlou-cdm-ds-0f601751-nj6q 9087m 30% 7946Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 9885m 33% 12870Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 12023m 75% 8593Mi 14% 21:43:52 DEBUG --- status code --- 21:43:52 DEBUG --- status code --- 21:43:52 DEBUG --- status code --- 21:43:52 DEBUG --- status code --- 21:43:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:43:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:43:52 DEBUG --- status code --- 21:43:52 INFO [http_cmd]: http status code OK 21:43:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:43:52 INFO [http_cmd]: http status code OK 21:43:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:43:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:43:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:43:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:43:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:43:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:43:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:43:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:43:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:43:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:43:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:43:52 DEBUG --- http response --- 21:43:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:43:52 DEBUG --- stderr --- 21:43:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:43:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:43:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:43:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:43:52 DEBUG --- http response --- 21:43:52 DEBUG --- http response --- 21:43:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:43:52 DEBUG --- status code --- 21:43:52 DEBUG --- http response --- 21:43:52 DEBUG --- status code --- 21:43:52 DEBUG --- http response --- 21:43:52 DEBUG --- http response --- 21:43:52 DEBUG --- http response --- 21:43:52 DEBUG --- http response --- 21:43:52 DEBUG --- http response --- 21:43:52 DEBUG --- http response --- 21:43:52 DEBUG --- http response --- 21:43:52 DEBUG --- http response --- 21:43:52 DEBUG --- http response --- 21:43:52 DEBUG --- http response --- 21:43:52 DEBUG --- http response --- 21:43:52 DEBUG --- http response --- 21:43:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:43:52 DEBUG 21:43:52 DEBUG --- http response --- 21:43:52 DEBUG --- http response --- 21:43:52 DEBUG --- http response --- 21:43:52 DEBUG --- http response --- 21:43:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:43:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:43:52 DEBUG --- http response --- 21:43:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:43:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:43:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:43:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:43:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:43:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:43:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:43:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:43:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:43:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:43:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:43:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:43:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:43:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:43:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:43:52 DEBUG ----- output ----- 21:43:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 21:43:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:43:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:43:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:43:52 DEBUG ----- output ----- 21:43:52 DEBUG ----- output ----- 21:43:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:43:52 DEBUG --- http response --- 21:43:52 DEBUG --- http response --- 21:43:52 DEBUG ----- output ----- 21:43:52 DEBUG ----- output ----- 21:43:52 DEBUG ----- output ----- 21:43:52 DEBUG ----- output ----- 21:43:52 DEBUG ----- output ----- 21:43:52 DEBUG { 21:43:52 DEBUG ----- output ----- 21:43:52 DEBUG ----- output ----- 21:43:52 DEBUG ----- output ----- 21:43:52 DEBUG ----- output ----- 21:43:52 DEBUG ----- output ----- 21:43:52 DEBUG ----- output ----- 21:43:52 DEBUG ----- output ----- 21:43:52 DEBUG { 21:43:52 DEBUG ----- output ----- 21:43:52 DEBUG ----- output ----- 21:43:52 DEBUG ----- output ----- 21:43:52 DEBUG ----- output ----- 21:43:52 DEBUG { 21:43:52 DEBUG { 21:43:52 DEBUG ----- output ----- 21:43:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:43:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:43:52 DEBUG { 21:43:52 DEBUG { 21:43:52 DEBUG { 21:43:52 DEBUG { 21:43:52 DEBUG ----- output ----- 21:43:52 DEBUG "status": "success", 21:43:52 DEBUG { 21:43:52 DEBUG { 21:43:52 DEBUG { 21:43:52 DEBUG { 21:43:52 DEBUG { 21:43:52 DEBUG { 21:43:52 DEBUG { 21:43:52 DEBUG "status": "success", 21:43:52 DEBUG { 21:43:52 DEBUG { 21:43:52 DEBUG { 21:43:52 DEBUG { 21:43:52 DEBUG "status": "success", 21:43:52 DEBUG "status": "success", 21:43:52 DEBUG { 21:43:52 DEBUG ----- output ----- 21:43:52 DEBUG ----- output ----- 21:43:52 DEBUG "status": "success", 21:43:52 DEBUG "status": "success", 21:43:52 DEBUG "status": "success", 21:43:52 DEBUG "status": "success", 21:43:52 DEBUG { 21:43:52 DEBUG "data": { 21:43:52 DEBUG "resultType": "vector", 21:43:52 DEBUG "status": "success", 21:43:52 DEBUG "status": "success", 21:43:52 DEBUG "status": "success", 21:43:52 DEBUG "status": "success", 21:43:52 DEBUG "status": "success", 21:43:52 DEBUG "status": "success", 21:43:52 DEBUG "data": { 21:43:52 DEBUG "status": "success", 21:43:52 DEBUG "status": "success", 21:43:52 DEBUG "status": "success", 21:43:52 DEBUG "status": "success", 21:43:52 DEBUG "data": { 21:43:52 DEBUG "data": { 21:43:52 DEBUG "status": "success", 21:43:52 DEBUG { 21:43:52 DEBUG { 21:43:52 DEBUG "data": { 21:43:52 DEBUG "data": { 21:43:52 DEBUG "data": { 21:43:52 DEBUG "data": { 21:43:52 DEBUG "status": "success", 21:43:52 DEBUG "status": "success", 21:43:52 DEBUG "result": [ 21:43:52 DEBUG "data": { 21:43:52 DEBUG "data": { 21:43:52 DEBUG "data": { 21:43:52 DEBUG "data": { 21:43:52 DEBUG "data": { 21:43:52 DEBUG "data": { 21:43:52 DEBUG "resultType": "vector", 21:43:52 DEBUG "data": { 21:43:52 DEBUG "data": { 21:43:52 DEBUG "data": { 21:43:52 DEBUG "data": { 21:43:52 DEBUG "resultType": "vector", 21:43:52 DEBUG "resultType": "vector", 21:43:52 DEBUG "data": { 21:43:52 DEBUG "status": "success", 21:43:52 DEBUG "status": "success", 21:43:52 DEBUG "resultType": "vector", 21:43:52 DEBUG "resultType": "vector", 21:43:52 DEBUG "resultType": "vector", 21:43:52 DEBUG "resultType": "vector", 21:43:52 DEBUG "data": { 21:43:52 DEBUG "data": { 21:43:52 DEBUG { 21:43:52 DEBUG "resultType": "vector", 21:43:52 DEBUG "resultType": "vector", 21:43:52 DEBUG "resultType": "vector", 21:43:52 DEBUG "resultType": "vector", 21:43:52 DEBUG "resultType": "vector", 21:43:52 DEBUG "resultType": "vector", 21:43:52 DEBUG "result": [ 21:43:52 DEBUG "resultType": "vector", 21:43:52 DEBUG "resultType": "vector", 21:43:52 DEBUG "resultType": "vector", 21:43:52 DEBUG "resultType": "vector", 21:43:52 DEBUG "result": [ 21:43:52 DEBUG "result": [ 21:43:52 DEBUG "resultType": "vector", 21:43:52 DEBUG "data": { 21:43:52 DEBUG "data": { 21:43:52 DEBUG "result": [ 21:43:52 DEBUG "result": [ 21:43:52 DEBUG "result": [ 21:43:52 DEBUG "result": [ 21:43:52 DEBUG "resultType": "vector", 21:43:52 DEBUG "resultType": "vector", 21:43:52 DEBUG "metric": { 21:43:52 DEBUG "result": [ 21:43:52 DEBUG "result": [ 21:43:52 DEBUG "result": [ 21:43:52 DEBUG "result": [ 21:43:52 DEBUG "result": [ 21:43:52 DEBUG "result": [ 21:43:52 DEBUG { 21:43:52 DEBUG "result": [ 21:43:52 DEBUG "result": [ 21:43:52 DEBUG "result": [ 21:43:52 DEBUG "result": [ 21:43:52 DEBUG { 21:43:52 DEBUG { 21:43:52 DEBUG "result": [ 21:43:52 DEBUG "resultType": "vector", 21:43:52 DEBUG "resultType": "vector", 21:43:52 DEBUG { 21:43:52 DEBUG { 21:43:52 DEBUG { 21:43:52 DEBUG { 21:43:52 DEBUG "result": [ 21:43:52 DEBUG "result": [ 21:43:52 DEBUG "instance": "10.142.0.107:9100" 21:43:52 DEBUG { 21:43:52 DEBUG { 21:43:52 DEBUG { 21:43:52 DEBUG { 21:43:52 DEBUG { 21:43:52 DEBUG { 21:43:52 DEBUG "metric": { 21:43:52 DEBUG { 21:43:52 DEBUG { 21:43:52 DEBUG { 21:43:52 DEBUG { 21:43:52 DEBUG "metric": { 21:43:52 DEBUG "metric": { 21:43:52 DEBUG { 21:43:52 DEBUG "result": [ 21:43:52 DEBUG "result": [ 21:43:52 DEBUG "metric": { 21:43:52 DEBUG "metric": { 21:43:52 DEBUG "metric": { 21:43:52 DEBUG "metric": { 21:43:52 DEBUG { 21:43:52 DEBUG { 21:43:52 DEBUG }, 21:43:52 DEBUG "metric": { 21:43:52 DEBUG "metric": { 21:43:52 DEBUG "metric": { 21:43:52 DEBUG "metric": { 21:43:52 DEBUG "metric": { 21:43:52 DEBUG "metric": { 21:43:52 DEBUG "pod": "am-767f6b67b5-9zw22" 21:43:52 DEBUG "metric": { 21:43:52 DEBUG "metric": { 21:43:52 DEBUG "metric": { 21:43:52 DEBUG "metric": { 21:43:52 DEBUG "pod": "ds-cts-0" 21:43:52 DEBUG "instance": "10.142.0.107:9100" 21:43:52 DEBUG "metric": { 21:43:52 DEBUG { 21:43:52 DEBUG { 21:43:52 DEBUG "pod": "am-767f6b67b5-9zw22" 21:43:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:43:52 DEBUG "pod": "am-767f6b67b5-9zw22" 21:43:52 DEBUG "instance": "10.142.0.107:9100" 21:43:52 DEBUG "metric": { 21:43:52 DEBUG "metric": { 21:43:52 DEBUG "value": [ 21:43:52 DEBUG "pod": "am-767f6b67b5-9zw22" 21:43:52 DEBUG "pod": "ds-idrepo-0" 21:43:52 DEBUG "pod": "am-767f6b67b5-9zw22" 21:43:52 DEBUG "pod": "ds-cts-0" 21:43:52 DEBUG "pod": "ds-idrepo-0" 21:43:52 DEBUG "pod": "am-767f6b67b5-9zw22" 21:43:52 DEBUG }, 21:43:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:43:52 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 21:43:52 DEBUG "pod": "am-767f6b67b5-9zw22" 21:43:52 DEBUG "pod": "ds-cts-0" 21:43:52 DEBUG }, 21:43:52 DEBUG }, 21:43:52 DEBUG "pod": "ds-idrepo-0" 21:43:52 DEBUG "metric": { 21:43:52 DEBUG "metric": { 21:43:52 DEBUG }, 21:43:52 DEBUG }, 21:43:52 DEBUG }, 21:43:52 DEBUG }, 21:43:52 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 21:43:52 DEBUG "instance": "10.142.0.107:9100" 21:43:52 DEBUG 21:43:52 DEBUG }, 21:43:52 DEBUG }, 21:43:52 DEBUG }, 21:43:52 DEBUG }, 21:43:52 DEBUG }, 21:43:52 DEBUG }, 21:43:52 DEBUG "value": [ 21:43:52 DEBUG }, 21:43:52 DEBUG }, 21:43:52 DEBUG "value": [ 21:43:52 DEBUG 21:43:52 DEBUG "value": [ 21:43:52 DEBUG "value": [ 21:43:52 DEBUG }, 21:43:52 DEBUG "instance": "10.142.0.107:9100" 21:43:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:43:52 DEBUG "value": [ 21:43:52 DEBUG "value": [ 21:43:52 DEBUG "value": [ 21:43:52 DEBUG "value": [ 21:43:52 DEBUG }, 21:43:52 DEBUG }, 21:43:52 DEBUG 21:43:52 DEBUG "value": [ 21:43:52 DEBUG "value": [ 21:43:52 DEBUG "value": [ 21:43:52 DEBUG "value": [ 21:43:52 DEBUG "value": [ 21:43:52 DEBUG "value": [ 21:43:52 DEBUG 21:43:52 DEBUG "value": [ 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG "value": [ 21:43:52 DEBUG }, 21:43:52 DEBUG }, 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG "value": [ 21:43:52 DEBUG "value": [ 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG [...] 21:43:52 DEBUG }, 21:43:52 DEBUG }, 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG "value": [ 21:43:52 DEBUG "value": [ 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG [...] 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG "value": [ 21:43:52 DEBUG "value": [ 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG [...] 21:43:52 DEBUG 21:43:52 DEBUG [...] 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG [...] 21:43:52 DEBUG [...] 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG [...] 21:43:52 DEBUG [...] 21:43:52 DEBUG [...] 21:43:52 DEBUG [...] 21:43:52 DEBUG 21:43:52 DEBUG [...] 21:43:52 DEBUG 21:43:52 DEBUG [...] 21:43:52 DEBUG 21:43:52 DEBUG [...] 21:43:52 DEBUG [...] 21:43:52 DEBUG [...] 21:43:52 DEBUG 21:43:52 DEBUG [...] 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG [...] 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG [...] 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG "pod": "ds-cts-0" 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG [...] 21:43:52 DEBUG [...] 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG "pod": "am-767f6b67b5-vbltz" 21:43:52 DEBUG }, 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG }, 21:43:52 DEBUG [...] 21:43:52 DEBUG [...] 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG "value": [ 21:43:52 DEBUG 21:43:52 DEBUG "instance": "10.142.0.43:9100" 21:43:52 DEBUG "pod": "am-767f6b67b5-vbltz" 21:43:52 DEBUG 21:43:52 DEBUG [...] 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG "pod": "am-767f6b67b5-vbltz" 21:43:52 DEBUG 21:43:52 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 21:43:52 DEBUG "value": [ 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG "pod": "ds-cts-2" 21:43:52 DEBUG "instance": "10.142.0.43:9100" 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG "pod": "am-767f6b67b5-vbltz" 21:43:52 DEBUG "pod": "ds-cts-0" 21:43:52 DEBUG 21:43:52 DEBUG 1713645832, 21:43:52 DEBUG "instance": "10.142.0.43:9100" 21:43:52 DEBUG }, 21:43:52 DEBUG }, 21:43:52 DEBUG 21:43:52 DEBUG "pod": "ds-idrepo-2" 21:43:52 DEBUG "pod": "am-767f6b67b5-vbltz" 21:43:52 DEBUG "pod": "ds-cts-2" 21:43:52 DEBUG "pod": "ds-idrepo-2" 21:43:52 DEBUG }, 21:43:52 DEBUG "pod": "am-767f6b67b5-vbltz" 21:43:52 DEBUG }, 21:43:52 DEBUG 1713645832, 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG }, 21:43:52 DEBUG }, 21:43:52 DEBUG "pod": "ds-idrepo-2" 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG }, 21:43:52 DEBUG }, 21:43:52 DEBUG "instance": "10.142.0.43:9100" 21:43:52 DEBUG "3328.5000000000005" 21:43:52 DEBUG }, 21:43:52 DEBUG "value": [ 21:43:52 DEBUG "value": [ 21:43:52 DEBUG 21:43:52 DEBUG }, 21:43:52 DEBUG }, 21:43:52 DEBUG }, 21:43:52 DEBUG }, 21:43:52 DEBUG "value": [ 21:43:52 DEBUG 1713645832, 21:43:52 DEBUG "value": [ 21:43:52 DEBUG "6.227399847448016" 21:43:52 DEBUG 21:43:52 DEBUG 21:43:52 DEBUG "value": [ 21:43:52 DEBUG "value": [ 21:43:52 DEBUG }, 21:43:52 DEBUG "instance": "10.142.0.43:9100" 21:43:52 DEBUG "pod": "ds-cts-0" 21:43:52 DEBUG "value": [ 21:43:52 DEBUG "value": [ 21:43:52 DEBUG }, 21:43:52 DEBUG ] 21:43:52 DEBUG "value": [ 21:43:52 DEBUG 1713645832, 21:43:52 DEBUG 1713645832, 21:43:52 DEBUG 21:43:52 DEBUG "value": [ 21:43:52 DEBUG "value": [ 21:43:52 DEBUG "value": [ 21:43:52 DEBUG "value": [ 21:43:52 DEBUG }, 21:43:52 DEBUG "0" 21:43:52 DEBUG 1713645832, 21:43:52 DEBUG ] 21:43:52 DEBUG "pod": "am-767f6b67b5-vbltz" 21:43:52 DEBUG "pod": "ds-cts-2" 21:43:52 DEBUG 1713645832, 21:43:52 DEBUG 1713645832, 21:43:52 DEBUG "value": [ 21:43:52 DEBUG }, 21:43:52 DEBUG }, 21:43:52 DEBUG 1713645832, 21:43:52 DEBUG 1713645832, 21:43:52 DEBUG "value": [ 21:43:52 DEBUG } 21:43:52 DEBUG 1713645832, 21:43:52 DEBUG "79370652.53333333" 21:43:52 DEBUG "3328.266666666667" 21:43:52 DEBUG 21:43:52 DEBUG 1713645832, 21:43:52 DEBUG 1713645832, 21:43:52 DEBUG 1713645832, 21:43:52 DEBUG 1713645832, 21:43:52 DEBUG "value": [ 21:43:52 DEBUG ] 21:43:52 DEBUG "758095872" 21:43:52 DEBUG } 21:43:52 DEBUG }, 21:43:52 DEBUG }, 21:43:52 DEBUG "0" 21:43:52 DEBUG "79373426.53333333" 21:43:52 DEBUG 1713645832, 21:43:52 DEBUG "value": [ 21:43:52 DEBUG "value": [ 21:43:52 DEBUG "49.377" 21:43:52 DEBUG "36077721.36247708" 21:43:52 DEBUG 1713645832, 21:43:52 DEBUG ] 21:43:52 DEBUG "0.06206666666664565" 21:43:52 DEBUG ] 21:43:52 DEBUG ] 21:43:52 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 21:43:52 DEBUG "0" 21:43:52 DEBUG "3328.5333333333338" 21:43:52 DEBUG "0.0007666666666666667" 21:43:52 DEBUG "0" 21:43:52 DEBUG 1713645832, 21:43:52 DEBUG } 21:43:52 DEBUG ] 21:43:52 DEBUG } 21:43:52 DEBUG "value": [ 21:43:52 DEBUG "value": [ 21:43:52 DEBUG ] 21:43:52 DEBUG ] 21:43:52 DEBUG "0" 21:43:52 DEBUG 1713645832, 21:43:52 DEBUG 1713645832, 21:43:52 DEBUG ] 21:43:52 DEBUG ] 21:43:52 DEBUG "3951198202880" 21:43:52 DEBUG } 21:43:52 DEBUG ] 21:43:52 DEBUG } 21:43:52 DEBUG } 21:43:52 DEBUG ] 21:43:52 DEBUG ] 21:43:52 DEBUG ] 21:43:52 DEBUG ] 21:43:52 DEBUG ] 21:43:52 DEBUG "0" 21:43:52 DEBUG ] 21:43:52 DEBUG ] 21:43:52 DEBUG ] 21:43:52 DEBUG 1713645832, 21:43:52 DEBUG 1713645832, 21:43:52 DEBUG } 21:43:52 DEBUG ] 21:43:52 DEBUG ] 21:43:52 DEBUG "0.042000000000005394" 21:43:52 DEBUG "1969.8383860179094" 21:43:52 DEBUG } 21:43:52 DEBUG } 21:43:52 DEBUG ] 21:43:52 DEBUG } 21:43:52 DEBUG } 21:43:52 DEBUG ] 21:43:52 DEBUG } 21:43:52 DEBUG }, 21:43:52 DEBUG } 21:43:52 DEBUG } 21:43:52 DEBUG } 21:43:52 DEBUG } 21:43:52 DEBUG ] 21:43:52 DEBUG } 21:43:52 DEBUG } 21:43:52 DEBUG } 21:43:52 DEBUG "423115.5" 21:43:52 DEBUG ] 21:43:52 DEBUG } 21:43:52 DEBUG } 21:43:52 DEBUG } 21:43:52 DEBUG ] 21:43:52 DEBUG ] 21:43:52 DEBUG ] 21:43:52 DEBUG ] 21:43:52 DEBUG } 21:43:52 DEBUG ] 21:43:52 DEBUG } 21:43:52 DEBUG } 21:43:52 DEBUG "value": [ 21:43:52 DEBUG ] 21:43:52 DEBUG ] 21:43:52 DEBUG ] 21:43:52 DEBUG ] 21:43:52 DEBUG } 21:43:52 DEBUG } 21:43:52 DEBUG } 21:43:52 DEBUG "0" 21:43:52 DEBUG } 21:43:52 DEBUG } 21:43:52 DEBUG ] 21:43:52 DEBUG ] 21:43:52 DEBUG } 21:43:52 DEBUG } 21:43:52 DEBUG } 21:43:52 DEBUG } 21:43:52 DEBUG ] 21:43:52 DEBUG } 21:43:52 DEBUG } 21:43:52 DEBUG 1713645832, 21:43:52 DEBUG } 21:43:52 DEBUG } 21:43:52 DEBUG } 21:43:52 DEBUG } 21:43:52 DEBUG ] 21:43:52 DEBUG } 21:43:52 DEBUG ] 21:43:52 DEBUG ] 21:43:52 DEBUG } 21:43:52 DEBUG } 21:43:52 DEBUG } 21:43:52 DEBUG ] 21:43:52 DEBUG } 21:43:52 DEBUG } 21:43:52 DEBUG } 21:43:52 DEBUG } 21:43:52 DEBUG "11.800897295609857" 21:43:52 DEBUG ] 21:43:52 DEBUG } 21:43:52 DEBUG ] 21:43:52 DEBUG } 21:43:52 DEBUG } 21:43:52 DEBUG } 21:43:52 DEBUG } 21:43:52 DEBUG } 21:43:52 DEBUG ] 21:43:52 DEBUG } 21:43:52 DEBUG } 21:43:52 DEBUG } 21:43:52 DEBUG } 21:43:52 DEBUG } 21:43:52 DEBUG } 21:43:52 DEBUG } 21:43:52 DEBUG ] 21:43:52 DEBUG } 21:43:52 DEBUG } 21:43:52 DEBUG } 21:43:52 DEBUG } 21:43:52 DEBUG } 21:43:52 DEBUG } 21:43:52 DEBUG } 21:44:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645892" 21:44:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713645892" 21:44:52 ERROR [http_cmd]: ERROR 21:44:52 ERROR --- status code --- 21:44:52 INFO [http_cmd]: http status code OK 21:44:52 ERROR http status code is 400 (expected [200, 401, 403, 404]) 21:44:52 DEBUG --- status code --- 21:44:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:44:52 DEBUG --- http response --- 21:44:52 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 21:44:52 ERROR --- http response --- 21:44:52 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 21:44:52 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 21:44:52 INFO 21:44:52 INFO [loop_until]: kubectl --namespace=xlou top pods 21:44:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:44:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713645892" 21:44:52 INFO [http_cmd]: http status code OK 21:44:52 DEBUG --- status code --- 21:44:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:44:52 DEBUG --- http response --- 21:44:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:44:52 DEBUG ----- output ----- 21:44:52 DEBUG { 21:44:52 DEBUG "status": "success", 21:44:52 DEBUG "data": { 21:44:52 DEBUG "resultType": "vector", 21:44:52 DEBUG "result": [ 21:44:52 DEBUG { 21:44:52 DEBUG "metric": { 21:44:52 DEBUG "pod": "ds-idrepo-0" 21:44:52 DEBUG }, 21:44:52 DEBUG "value": [ 21:44:52 DEBUG 21:44:52 DEBUG 21:44:52 DEBUG 21:44:52 DEBUG [...] 21:44:52 DEBUG 21:44:52 DEBUG 21:44:52 DEBUG 21:44:52 DEBUG "pod": "ds-cts-2" 21:44:52 DEBUG }, 21:44:52 DEBUG "value": [ 21:44:52 DEBUG 1713645892, 21:44:52 DEBUG "0" 21:44:52 DEBUG ] 21:44:52 DEBUG } 21:44:52 DEBUG ] 21:44:52 DEBUG } 21:44:52 DEBUG } 21:44:52 INFO [loop_until]: OK (rc = 0) 21:44:52 DEBUG --- stdout --- 21:44:52 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 18203m 8732Mi am-767f6b67b5-qcqz2 17628m 8846Mi am-767f6b67b5-vbltz 19620m 8870Mi ds-cts-0 11421m 10069Mi ds-cts-1 10201m 9799Mi ds-cts-2 11000m 10169Mi ds-idrepo-0 8281m 3967Mi ds-idrepo-1 8452m 3980Mi ds-idrepo-2 8269m 3870Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 4m 1239Mi lodemon-7dd48676f6-tdxl9 14m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-967d957f7-zk66v 11998m 636Mi 21:44:52 DEBUG --- stderr --- 21:44:52 DEBUG 21:44:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645892" 21:44:52 INFO [http_cmd]: http status code OK 21:44:52 DEBUG --- status code --- 21:44:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:44:52 DEBUG --- http response --- 21:44:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:44:52 DEBUG ----- output ----- 21:44:52 DEBUG { 21:44:52 DEBUG "status": "success", 21:44:52 DEBUG "data": { 21:44:52 DEBUG "resultType": "vector", 21:44:52 DEBUG "result": [ 21:44:52 DEBUG { 21:44:52 DEBUG "metric": { 21:44:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:44:52 DEBUG }, 21:44:52 DEBUG "value": [ 21:44:52 DEBUG 21:44:52 DEBUG 21:44:52 DEBUG 21:44:52 DEBUG [...] 21:44:52 DEBUG 21:44:52 DEBUG 21:44:52 DEBUG 21:44:52 DEBUG "pod": "overseer-0-967d957f7-zk66v" 21:44:52 DEBUG }, 21:44:52 DEBUG "value": [ 21:44:52 DEBUG 1713645892, 21:44:52 DEBUG "120229.5717411691" 21:44:52 DEBUG ] 21:44:52 DEBUG } 21:44:52 DEBUG ] 21:44:52 DEBUG } 21:44:52 DEBUG } 21:44:52 INFO 21:44:52 INFO [loop_until]: kubectl --namespace=xlou top node 21:44:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:44:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645892" 21:44:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645892" 21:44:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713645892" 21:44:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713645892" 21:44:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645892" 21:44:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645892" 21:44:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645892" 21:44:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645892" 21:44:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713645892" 21:44:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713645892" 21:44:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645892" 21:44:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645892" 21:44:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713645892" 21:44:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713645892" 21:44:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645892" 21:44:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645892" 21:44:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645892" 21:44:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713645892" 21:44:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713645892" 21:44:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645892" 21:44:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645892" 21:44:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645892" 21:44:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645892" 21:44:52 INFO [http_cmd]: http status code OK 21:44:52 INFO [http_cmd]: http status code OK 21:44:52 INFO [http_cmd]: http status code OK 21:44:52 INFO [http_cmd]: http status code OK 21:44:52 INFO [http_cmd]: http status code OK 21:44:53 DEBUG --- status code --- 21:44:52 INFO [http_cmd]: http status code OK 21:44:52 INFO [http_cmd]: http status code OK 21:44:53 INFO [http_cmd]: http status code OK 21:44:53 INFO [http_cmd]: http status code OK 21:44:53 INFO [http_cmd]: http status code OK 21:44:53 INFO [http_cmd]: http status code OK 21:44:53 INFO [http_cmd]: http status code OK 21:44:53 INFO [http_cmd]: http status code OK 21:44:53 INFO [http_cmd]: http status code OK 21:44:53 INFO [http_cmd]: http status code OK 21:44:53 INFO [http_cmd]: http status code OK 21:44:53 INFO [http_cmd]: http status code OK 21:44:53 DEBUG --- status code --- 21:44:53 DEBUG --- status code --- 21:44:53 DEBUG --- status code --- 21:44:53 DEBUG --- status code --- 21:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:44:52 INFO [loop_until]: OK (rc = 0) 21:44:53 DEBUG --- status code --- 21:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:44:53 DEBUG --- status code --- 21:44:53 DEBUG --- status code --- 21:44:53 DEBUG --- status code --- 21:44:53 DEBUG --- status code --- 21:44:53 INFO [http_cmd]: http status code OK 21:44:53 INFO [http_cmd]: http status code OK 21:44:53 DEBUG --- status code --- 21:44:53 INFO [http_cmd]: http status code OK 21:44:53 INFO [http_cmd]: http status code OK 21:44:53 DEBUG --- status code --- 21:44:53 DEBUG --- status code --- 21:44:53 DEBUG --- status code --- 21:44:53 DEBUG --- status code --- 21:44:53 DEBUG --- status code --- 21:44:53 DEBUG --- status code --- 21:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:44:53 INFO [http_cmd]: http status code OK 21:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:44:53 DEBUG --- http response --- 21:44:53 DEBUG --- stdout --- 21:44:53 INFO [http_cmd]: http status code OK 21:44:53 DEBUG --- http response --- 21:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:44:53 DEBUG --- http response --- 21:44:53 DEBUG --- status code --- 21:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:44:53 DEBUG --- status code --- 21:44:53 DEBUG --- status code --- 21:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:44:53 DEBUG --- http response --- 21:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:44:53 DEBUG --- http response --- 21:44:53 DEBUG --- http response --- 21:44:53 DEBUG --- http response --- 21:44:53 DEBUG --- status code --- 21:44:53 DEBUG --- http response --- 21:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:44:53 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6215Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8682m 29% 7719Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 17786m 59% 11451Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 3379Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 11537m 38% 13313Mi 11% gke-xlou-cdm-default-pool-1b0769c8-wpwl 161m 0% 5213Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 18178m 60% 11729Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19580m 65% 3059Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 20460m 68% 11090Mi 9% gke-xlou-cdm-ds-0f601751-j488 9307m 31% 7280Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 10499m 35% 15028Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 9217m 30% 7956Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 11660m 39% 12932Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 11482m 72% 8596Mi 14% 21:44:53 DEBUG --- status code --- 21:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:44:53 DEBUG --- http response --- 21:44:53 DEBUG --- http response --- 21:44:53 DEBUG --- http response --- 21:44:53 DEBUG --- status code --- 21:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:44:53 DEBUG --- http response --- 21:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:44:53 DEBUG --- http response --- 21:44:53 DEBUG --- http response --- 21:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:44:53 DEBUG --- http response --- 21:44:53 DEBUG --- http response --- 21:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:44:53 DEBUG ----- output ----- 21:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:44:53 DEBUG ----- output ----- 21:44:53 DEBUG --- stderr --- 21:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:44:53 DEBUG ----- output ----- 21:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:44:53 DEBUG --- http response --- 21:44:53 DEBUG ----- output ----- 21:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:44:53 DEBUG --- http response --- 21:44:53 DEBUG --- http response --- 21:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:44:53 DEBUG --- http response --- 21:44:53 DEBUG ----- output ----- 21:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:44:53 DEBUG ----- output ----- 21:44:53 DEBUG ----- output ----- 21:44:53 DEBUG { 21:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:44:53 DEBUG ----- output ----- 21:44:53 DEBUG { 21:44:53 DEBUG 21:44:53 DEBUG --- http response --- 21:44:53 DEBUG { 21:44:53 DEBUG "status": "success", 21:44:53 DEBUG ----- output ----- 21:44:53 DEBUG --- http response --- 21:44:53 DEBUG ----- output ----- 21:44:53 DEBUG { 21:44:53 DEBUG { 21:44:53 DEBUG ----- output ----- 21:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:44:53 DEBUG ----- output ----- 21:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:44:53 DEBUG ----- output ----- 21:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:44:53 DEBUG { 21:44:53 DEBUG ----- output ----- 21:44:53 DEBUG { 21:44:53 DEBUG ----- output ----- 21:44:53 DEBUG { 21:44:53 DEBUG "status": "success", 21:44:53 DEBUG --- http response --- 21:44:53 DEBUG { 21:44:53 DEBUG "status": "success", 21:44:53 DEBUG ----- output ----- 21:44:53 DEBUG "data": { 21:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:44:53 DEBUG { 21:44:53 DEBUG "status": "success", 21:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:44:53 DEBUG "status": "success", 21:44:53 DEBUG "status": "success", 21:44:53 DEBUG { 21:44:53 DEBUG ----- output ----- 21:44:53 DEBUG { 21:44:53 DEBUG ----- output ----- 21:44:53 DEBUG { 21:44:53 DEBUG ----- output ----- 21:44:53 DEBUG "status": "success", 21:44:53 DEBUG "status": "success", 21:44:53 DEBUG "status": "success", 21:44:53 DEBUG { 21:44:53 DEBUG { 21:44:53 DEBUG "data": { 21:44:53 DEBUG "status": "success", 21:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:44:53 DEBUG "data": { 21:44:53 DEBUG { 21:44:53 DEBUG "resultType": "vector", 21:44:53 DEBUG ----- output ----- 21:44:53 DEBUG "data": { 21:44:53 DEBUG ----- output ----- 21:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:44:53 DEBUG "data": { 21:44:53 DEBUG "data": { 21:44:53 DEBUG "status": "success", 21:44:53 DEBUG { 21:44:53 DEBUG "status": "success", 21:44:53 DEBUG { 21:44:53 DEBUG "status": "success", 21:44:53 DEBUG { 21:44:53 DEBUG "data": { 21:44:53 DEBUG "data": { 21:44:53 DEBUG "data": { 21:44:53 DEBUG "status": "success", 21:44:53 DEBUG "status": "success", 21:44:53 DEBUG "resultType": "vector", 21:44:53 DEBUG "result": [ 21:44:53 DEBUG ----- output ----- 21:44:53 DEBUG "resultType": "vector", 21:44:53 DEBUG "status": "success", 21:44:53 DEBUG "result": [ 21:44:53 DEBUG { 21:44:53 DEBUG "resultType": "vector", 21:44:53 DEBUG { 21:44:53 DEBUG ----- output ----- 21:44:53 DEBUG "resultType": "vector", 21:44:53 DEBUG "resultType": "vector", 21:44:53 DEBUG "data": { 21:44:53 DEBUG "status": "success", 21:44:53 DEBUG "data": { 21:44:53 DEBUG "status": "success", 21:44:53 DEBUG "data": { 21:44:53 DEBUG "status": "success", 21:44:53 DEBUG "resultType": "vector", 21:44:53 DEBUG "resultType": "vector", 21:44:53 DEBUG "resultType": "vector", 21:44:53 DEBUG "data": { 21:44:53 DEBUG "data": { 21:44:53 DEBUG "data": { 21:44:53 DEBUG { 21:44:53 DEBUG "metric": { 21:44:53 DEBUG "result": [ 21:44:53 DEBUG "data": { 21:44:53 DEBUG { 21:44:53 DEBUG "status": "success", 21:44:53 DEBUG "result": [ 21:44:53 DEBUG "status": "success", 21:44:53 DEBUG { 21:44:53 DEBUG "result": [ 21:44:53 DEBUG "result": [ 21:44:53 DEBUG "resultType": "vector", 21:44:53 DEBUG "data": { 21:44:53 DEBUG "resultType": "vector", 21:44:53 DEBUG "data": { 21:44:53 DEBUG "resultType": "vector", 21:44:53 DEBUG "data": { 21:44:53 DEBUG "result": [ 21:44:53 DEBUG "result": [ 21:44:53 DEBUG "result": [ 21:44:53 DEBUG "resultType": "vector", 21:44:53 DEBUG "resultType": "vector", 21:44:53 DEBUG "resultType": "vector", 21:44:53 DEBUG { 21:44:53 DEBUG "pod": "am-767f6b67b5-9zw22" 21:44:53 DEBUG { 21:44:53 DEBUG "resultType": "vector", 21:44:53 DEBUG "metric": { 21:44:53 DEBUG "data": { 21:44:53 DEBUG { 21:44:53 DEBUG "data": { 21:44:53 DEBUG "status": "success", 21:44:53 DEBUG { 21:44:53 DEBUG { 21:44:53 DEBUG "result": [ 21:44:53 DEBUG "resultType": "vector", 21:44:53 DEBUG "result": [ 21:44:53 DEBUG "resultType": "vector", 21:44:53 DEBUG "result": [ 21:44:53 DEBUG "resultType": "vector", 21:44:53 DEBUG { 21:44:53 DEBUG { 21:44:53 DEBUG { 21:44:53 DEBUG "result": [ 21:44:53 DEBUG "result": [ 21:44:53 DEBUG "result": [ 21:44:53 DEBUG "status": "success", 21:44:53 DEBUG }, 21:44:53 DEBUG "metric": { 21:44:53 DEBUG "result": [ 21:44:53 DEBUG "instance": "10.142.0.107:9100" 21:44:53 DEBUG "resultType": "vector", 21:44:53 DEBUG "metric": { 21:44:53 DEBUG "pod": "ds-idrepo-0" 21:44:53 DEBUG }, 21:44:53 DEBUG "metric": { 21:44:53 DEBUG "metric": { 21:44:53 DEBUG { 21:44:53 DEBUG "result": [ 21:44:53 DEBUG { 21:44:53 DEBUG "result": [ 21:44:53 DEBUG { 21:44:53 DEBUG "result": [ 21:44:53 DEBUG "metric": { 21:44:53 DEBUG "metric": { 21:44:53 DEBUG "metric": { 21:44:53 DEBUG "pod": "ds-cts-0" 21:44:53 DEBUG { 21:44:53 DEBUG { 21:44:53 DEBUG "data": { 21:44:53 DEBUG "value": [ 21:44:53 DEBUG "pod": "am-767f6b67b5-9zw22" 21:44:53 DEBUG { 21:44:53 DEBUG }, 21:44:53 DEBUG "result": [ 21:44:53 DEBUG "value": [ 21:44:53 DEBUG "instance": "10.142.0.107:9100" 21:44:53 DEBUG "data": { 21:44:53 DEBUG "resultType": "vector", 21:44:53 DEBUG "pod": "am-767f6b67b5-9zw22" 21:44:53 DEBUG "metric": { 21:44:53 DEBUG { 21:44:53 DEBUG "metric": { 21:44:53 DEBUG { 21:44:53 DEBUG "metric": { 21:44:53 DEBUG { 21:44:53 DEBUG "instance": "10.142.0.107:9100" 21:44:53 DEBUG "pod": "am-767f6b67b5-9zw22" 21:44:53 DEBUG { 21:44:53 DEBUG }, 21:44:53 DEBUG "metric": { 21:44:53 DEBUG "metric": { 21:44:53 DEBUG "resultType": "vector", 21:44:53 DEBUG 21:44:53 DEBUG }, 21:44:53 DEBUG "metric": { 21:44:53 DEBUG "value": [ 21:44:53 DEBUG { 21:44:53 DEBUG 21:44:53 DEBUG }, 21:44:53 DEBUG "resultType": "vector", 21:44:53 DEBUG "result": [ 21:44:53 DEBUG }, 21:44:53 DEBUG "instance": "10.142.0.107:9100" 21:44:53 DEBUG "metric": { 21:44:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:44:53 DEBUG "metric": { 21:44:53 DEBUG "pod": "ds-idrepo-0" 21:44:53 DEBUG "metric": { 21:44:53 DEBUG }, 21:44:53 DEBUG }, 21:44:53 DEBUG "metric": { 21:44:53 DEBUG "value": [ 21:44:53 DEBUG "pod": "am-767f6b67b5-9zw22" 21:44:53 DEBUG }, 21:44:53 DEBUG "result": [ 21:44:53 DEBUG 21:44:53 DEBUG "value": [ 21:44:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:44:53 DEBUG 21:44:53 DEBUG "metric": { 21:44:53 DEBUG "pod": "am-767f6b67b5-9zw22" 21:44:53 DEBUG "value": [ 21:44:53 DEBUG "result": [ 21:44:53 DEBUG { 21:44:53 DEBUG "value": [ 21:44:53 DEBUG }, 21:44:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:44:53 DEBUG }, 21:44:53 DEBUG "instance": "10.142.0.107:9100" 21:44:53 DEBUG }, 21:44:53 DEBUG "pod": "ds-cts-0" 21:44:53 DEBUG "value": [ 21:44:53 DEBUG "value": [ 21:44:53 DEBUG "pod": "ds-cts-0" 21:44:53 DEBUG 21:44:53 DEBUG "value": [ 21:44:53 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 21:44:53 DEBUG }, 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG }, 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG }, 21:44:53 DEBUG 21:44:53 DEBUG { 21:44:53 DEBUG "metric": { 21:44:53 DEBUG 21:44:53 DEBUG "value": [ 21:44:53 DEBUG }, 21:44:53 DEBUG "value": [ 21:44:53 DEBUG }, 21:44:53 DEBUG "value": [ 21:44:53 DEBUG "value": [ 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG }, 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG { 21:44:53 DEBUG "value": [ 21:44:53 DEBUG [...] 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG [...] 21:44:53 DEBUG 21:44:53 DEBUG "value": [ 21:44:53 DEBUG 21:44:53 DEBUG "metric": { 21:44:53 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG }, 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG "metric": { 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG "value": [ 21:44:53 DEBUG 21:44:53 DEBUG [...] 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG }, 21:44:53 DEBUG "pod": "ds-idrepo-0" 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG "value": [ 21:44:53 DEBUG 21:44:53 DEBUG "value": [ 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG [...] 21:44:53 DEBUG "value": [ 21:44:53 DEBUG [...] 21:44:53 DEBUG 21:44:53 DEBUG "pod": "am-767f6b67b5-9zw22" 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG [...] 21:44:53 DEBUG "value": [ 21:44:53 DEBUG }, 21:44:53 DEBUG [...] 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG [...] 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG [...] 21:44:53 DEBUG 21:44:53 DEBUG }, 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG [...] 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG "value": [ 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG [...] 21:44:53 DEBUG [...] 21:44:53 DEBUG 21:44:53 DEBUG [...] 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG "value": [ 21:44:53 DEBUG "pod": "am-767f6b67b5-vbltz" 21:44:53 DEBUG 21:44:53 DEBUG "pod": "am-767f6b67b5-vbltz" 21:44:53 DEBUG "pod": "ds-idrepo-2" 21:44:53 DEBUG 21:44:53 DEBUG [...] 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG [...] 21:44:53 DEBUG "pod": "am-767f6b67b5-vbltz" 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG [...] 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG }, 21:44:53 DEBUG [...] 21:44:53 DEBUG }, 21:44:53 DEBUG }, 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG }, 21:44:53 DEBUG [...] 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG [...] 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG "instance": "10.142.0.43:9100" 21:44:53 DEBUG [...] 21:44:53 DEBUG "pod": "ds-cts-2" 21:44:53 DEBUG 21:44:53 DEBUG "pod": "am-767f6b67b5-vbltz" 21:44:53 DEBUG 21:44:53 DEBUG "value": [ 21:44:53 DEBUG 21:44:53 DEBUG "value": [ 21:44:53 DEBUG "value": [ 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG [...] 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG "value": [ 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG "pod": "am-767f6b67b5-vbltz" 21:44:53 DEBUG }, 21:44:53 DEBUG 21:44:53 DEBUG }, 21:44:53 DEBUG 21:44:53 DEBUG }, 21:44:53 DEBUG 21:44:53 DEBUG 1713645892, 21:44:53 DEBUG 21:44:53 DEBUG 1713645892, 21:44:53 DEBUG 1713645892, 21:44:53 DEBUG "instance": "10.142.0.43:9100" 21:44:53 DEBUG "instance": "10.142.0.43:9100" 21:44:53 DEBUG "pod": "am-767f6b67b5-vbltz" 21:44:53 DEBUG 21:44:53 DEBUG [...] 21:44:53 DEBUG 21:44:53 DEBUG 1713645892, 21:44:53 DEBUG 21:44:53 DEBUG "pod": "overseer-0-967d957f7-zk66v" 21:44:53 DEBUG "instance": "10.142.0.43:9100" 21:44:53 DEBUG 21:44:53 DEBUG "pod": "ds-idrepo-2" 21:44:53 DEBUG }, 21:44:53 DEBUG "value": [ 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG "value": [ 21:44:53 DEBUG 1713645892, 21:44:53 DEBUG [...] 21:44:53 DEBUG 21:44:53 DEBUG "pod": "overseer-0-967d957f7-zk66v" 21:44:53 DEBUG }, 21:44:53 DEBUG "0" 21:44:53 DEBUG }, 21:44:53 DEBUG }, 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG "pod": "overseer-0-967d957f7-zk66v" 21:44:53 DEBUG "49.377" 21:44:53 DEBUG 21:44:53 DEBUG }, 21:44:53 DEBUG }, 21:44:53 DEBUG 21:44:53 DEBUG }, 21:44:53 DEBUG "value": [ 21:44:53 DEBUG 1713645892, 21:44:53 DEBUG "value": [ 21:44:53 DEBUG 1713645892, 21:44:53 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 21:44:53 DEBUG "3325.5333333333338" 21:44:53 DEBUG "2342.833333333334" 21:44:53 DEBUG 21:44:53 DEBUG }, 21:44:53 DEBUG "value": [ 21:44:53 DEBUG "3329.1666666666665" 21:44:53 DEBUG ] 21:44:53 DEBUG } 21:44:53 DEBUG "value": [ 21:44:53 DEBUG 21:44:53 DEBUG 21:44:53 DEBUG }, 21:44:53 DEBUG "value": [ 21:44:53 DEBUG "instance": "10.142.0.43:9100" 21:44:53 DEBUG "value": [ 21:44:53 DEBUG "value": [ 21:44:53 DEBUG 1713645892, 21:44:53 DEBUG "pod": "ds-cts-2" 21:44:53 DEBUG 1713645892, 21:44:53 DEBUG "3954946538496" 21:44:53 DEBUG "0" 21:44:53 DEBUG "pod": "ds-cts-2" 21:44:53 DEBUG }, 21:44:53 DEBUG ] 21:44:53 DEBUG ] 21:44:53 DEBUG 21:44:53 DEBUG "value": [ 21:44:53 DEBUG 1713645892, 21:44:53 DEBUG ] 21:44:53 DEBUG ] 21:44:53 DEBUG "value": [ 21:44:53 DEBUG 1713645892, 21:44:53 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 21:44:53 DEBUG 21:44:53 DEBUG 1713645892, 21:44:53 DEBUG ] 21:44:53 DEBUG }, 21:44:53 DEBUG 1713645892, 21:44:53 DEBUG "0.03933333333333925" 21:44:53 DEBUG "value": [ 21:44:53 DEBUG }, 21:44:53 DEBUG "0" 21:44:53 DEBUG ] 21:44:53 DEBUG ] 21:44:53 DEBUG }, 21:44:53 DEBUG "value": [ 21:44:53 DEBUG } 21:44:53 DEBUG } 21:44:53 DEBUG ] 21:44:53 DEBUG 1713645892, 21:44:53 DEBUG "11.75086384904657" 21:44:53 DEBUG } 21:44:53 DEBUG } 21:44:53 DEBUG 1713645892, 21:44:53 DEBUG "3324.833333333334" 21:44:53 DEBUG }, 21:44:53 DEBUG "pod": "ds-idrepo-2" 21:44:53 DEBUG "0" 21:44:53 DEBUG } 21:44:53 DEBUG "value": [ 21:44:53 DEBUG "0" 21:44:53 DEBUG ] 21:44:53 DEBUG 1713645892, 21:44:53 DEBUG "value": [ 21:44:53 DEBUG ] 21:44:53 DEBUG } 21:44:53 DEBUG ] 21:44:53 DEBUG } 21:44:53 DEBUG } 21:44:53 DEBUG ] 21:44:53 DEBUG } 21:44:53 DEBUG } 21:44:53 DEBUG 21:44:53 DEBUG ] 21:44:53 DEBUG ] 21:44:53 DEBUG } 21:44:53 DEBUG "0.06540000000004416" 21:44:53 DEBUG ] 21:44:53 DEBUG "value": [ 21:44:53 DEBUG }, 21:44:53 DEBUG ] 21:44:53 DEBUG ] 21:44:53 DEBUG 1713645892, 21:44:53 DEBUG ] 21:44:53 DEBUG } 21:44:53 DEBUG "0" 21:44:53 DEBUG ] 21:44:53 DEBUG } 21:44:53 DEBUG "value": [ 21:44:53 DEBUG 1713645892, 21:44:53 DEBUG } 21:44:53 DEBUG } 21:44:53 DEBUG } 21:44:53 DEBUG "pod": "am-767f6b67b5-vbltz" 21:44:53 DEBUG "86522450.76666667" 21:44:53 DEBUG } 21:44:53 DEBUG } 21:44:53 DEBUG } 21:44:53 DEBUG 1713645892, 21:44:53 DEBUG "value": [ 21:44:53 DEBUG } 21:44:53 DEBUG } 21:44:53 DEBUG "86526485.36666667" 21:44:53 DEBUG } 21:44:53 DEBUG ] 21:44:53 DEBUG } 21:44:53 DEBUG 1713645892, 21:44:53 DEBUG ] 21:44:53 DEBUG 1713645892, 21:44:53 DEBUG "762855424" 21:44:53 DEBUG ] 21:44:53 DEBUG }, 21:44:53 DEBUG } 21:44:53 DEBUG ] 21:44:53 DEBUG ] 21:44:53 DEBUG ] 21:44:53 DEBUG "11.801237376478491" 21:44:53 DEBUG 1713645892, 21:44:53 DEBUG ] 21:44:53 DEBUG } 21:44:53 DEBUG ] 21:44:53 DEBUG ] 21:44:53 DEBUG } 21:44:53 DEBUG ] 21:44:53 DEBUG "0.000033333333333333335" 21:44:53 DEBUG } 21:44:53 DEBUG "0" 21:44:53 DEBUG ] 21:44:53 DEBUG } 21:44:53 DEBUG "value": [ 21:44:53 DEBUG ] 21:44:53 DEBUG } 21:44:53 DEBUG ] 21:44:53 DEBUG } 21:44:53 DEBUG ] 21:44:53 DEBUG "0" 21:44:53 DEBUG ] 21:44:53 DEBUG } 21:44:53 DEBUG } 21:44:53 DEBUG } 21:44:53 DEBUG } 21:44:53 DEBUG ] 21:44:53 DEBUG } 21:44:53 DEBUG } 21:44:53 DEBUG ] 21:44:53 DEBUG } 21:44:53 DEBUG 1713645892, 21:44:53 DEBUG "585235" 21:44:53 DEBUG } 21:44:53 DEBUG } 21:44:53 DEBUG ] 21:44:53 DEBUG } 21:44:53 DEBUG ] 21:44:53 DEBUG } 21:44:53 DEBUG } 21:44:53 DEBUG } 21:44:53 DEBUG } 21:44:53 DEBUG } 21:44:53 DEBUG ] 21:44:53 DEBUG ] 21:44:53 DEBUG } 21:44:53 DEBUG ] 21:44:53 DEBUG ] 21:44:53 DEBUG } 21:44:53 DEBUG } 21:44:53 DEBUG } 21:44:53 DEBUG ] 21:44:53 DEBUG } 21:44:53 DEBUG ] 21:44:53 DEBUG } 21:44:53 DEBUG } 21:44:53 DEBUG } 21:44:53 DEBUG } 21:44:53 DEBUG } 21:44:53 DEBUG } 21:44:53 DEBUG } 21:44:53 DEBUG } 21:44:53 DEBUG } 21:44:53 DEBUG } 21:44:53 DEBUG } 21:44:53 DEBUG } 21:44:53 DEBUG } 21:44:53 DEBUG } 21:44:53 DEBUG ] 21:44:53 DEBUG } 21:44:53 DEBUG } 21:44:53 DEBUG } 21:44:53 DEBUG } 21:45:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713645952" 21:45:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645952" 21:45:52 ERROR [http_cmd]: ERROR 21:45:52 ERROR --- status code --- 21:45:52 ERROR http status code is 400 (expected [200, 401, 403, 404]) 21:45:52 INFO [http_cmd]: http status code OK 21:45:52 ERROR --- http response --- 21:45:52 DEBUG --- status code --- 21:45:52 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 21:45:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:45:52 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 21:45:52 DEBUG --- http response --- 21:45:52 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 21:45:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713645952" 21:45:52 INFO 21:45:52 INFO [loop_until]: kubectl --namespace=xlou top pods 21:45:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:45:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645952" 21:45:52 INFO [http_cmd]: http status code OK 21:45:52 DEBUG --- status code --- 21:45:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:45:52 DEBUG --- http response --- 21:45:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:45:52 DEBUG ----- output ----- 21:45:52 DEBUG { 21:45:52 DEBUG "status": "success", 21:45:52 DEBUG "data": { 21:45:52 DEBUG "resultType": "vector", 21:45:52 DEBUG "result": [ 21:45:52 DEBUG { 21:45:52 DEBUG "metric": { 21:45:52 DEBUG "pod": "ds-idrepo-0" 21:45:52 DEBUG }, 21:45:52 DEBUG "value": [ 21:45:52 DEBUG 21:45:52 DEBUG 21:45:52 DEBUG 21:45:52 DEBUG [...] 21:45:52 DEBUG 21:45:52 DEBUG 21:45:52 DEBUG 21:45:52 DEBUG "pod": "ds-cts-2" 21:45:52 DEBUG }, 21:45:52 DEBUG "value": [ 21:45:52 DEBUG 1713645952, 21:45:52 DEBUG "0" 21:45:52 DEBUG ] 21:45:52 DEBUG } 21:45:52 DEBUG ] 21:45:52 DEBUG } 21:45:52 DEBUG } 21:45:52 INFO [http_cmd]: http status code OK 21:45:52 DEBUG --- status code --- 21:45:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:45:52 DEBUG --- http response --- 21:45:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:45:52 DEBUG ----- output ----- 21:45:52 DEBUG { 21:45:52 DEBUG "status": "success", 21:45:52 DEBUG "data": { 21:45:52 DEBUG "resultType": "vector", 21:45:52 DEBUG "result": [ 21:45:52 DEBUG { 21:45:52 DEBUG "metric": { 21:45:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:45:52 DEBUG }, 21:45:52 DEBUG "value": [ 21:45:52 DEBUG 21:45:52 DEBUG 21:45:52 DEBUG 21:45:52 DEBUG [...] 21:45:52 DEBUG 21:45:52 DEBUG 21:45:52 DEBUG 21:45:52 DEBUG "pod": "overseer-0-967d957f7-zk66v" 21:45:52 DEBUG }, 21:45:52 DEBUG "value": [ 21:45:52 DEBUG 1713645952, 21:45:52 DEBUG "0" 21:45:52 DEBUG ] 21:45:52 DEBUG } 21:45:52 DEBUG ] 21:45:52 DEBUG } 21:45:52 DEBUG } 21:45:52 INFO [loop_until]: OK (rc = 0) 21:45:52 DEBUG --- stdout --- 21:45:52 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 22112m 8756Mi am-767f6b67b5-qcqz2 21999m 8877Mi am-767f6b67b5-vbltz 23476m 8940Mi ds-cts-0 20326m 11493Mi ds-cts-1 20072m 11547Mi ds-cts-2 21090m 11758Mi ds-idrepo-0 9546m 3968Mi ds-idrepo-1 9596m 3980Mi ds-idrepo-2 9805m 3870Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-7dd48676f6-tdxl9 16m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-967d957f7-zk66v 11846m 792Mi 21:45:52 DEBUG --- stderr --- 21:45:52 DEBUG 21:45:53 INFO 21:45:53 INFO [loop_until]: kubectl --namespace=xlou top node 21:45:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:45:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645953" 21:45:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713645953" 21:45:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645953" 21:45:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645953" 21:45:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645953" 21:45:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713645953" 21:45:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645953" 21:45:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713645953" 21:45:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645953" 21:45:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645953" 21:45:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645953" 21:45:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645953" 21:45:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713645953" 21:45:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645953" 21:45:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713645953" 21:45:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645953" 21:45:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645953" 21:45:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645953" 21:45:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713645953" 21:45:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645953" 21:45:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713645953" 21:45:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713645953" 21:45:53 INFO [http_cmd]: http status code OK 21:45:53 DEBUG --- status code --- 21:45:53 INFO [http_cmd]: http status code OK 21:45:53 INFO [http_cmd]: http status code OK 21:45:53 INFO [loop_until]: OK (rc = 0) 21:45:53 INFO [http_cmd]: http status code OK 21:45:53 INFO [http_cmd]: http status code OK 21:45:53 DEBUG --- status code --- 21:45:53 INFO [http_cmd]: http status code OK 21:45:53 INFO [http_cmd]: http status code OK 21:45:53 INFO [http_cmd]: http status code OK 21:45:53 DEBUG --- status code --- 21:45:53 INFO [http_cmd]: http status code OK 21:45:53 INFO [http_cmd]: http status code OK 21:45:53 INFO [http_cmd]: http status code OK 21:45:53 INFO [http_cmd]: http status code OK 21:45:53 INFO [http_cmd]: http status code OK 21:45:53 INFO [http_cmd]: http status code OK 21:45:53 INFO [http_cmd]: http status code OK 21:45:53 INFO [http_cmd]: http status code OK 21:45:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713645953" 21:45:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:45:53 DEBUG --- status code --- 21:45:53 DEBUG --- status code --- 21:45:53 INFO [http_cmd]: http status code OK 21:45:53 INFO [http_cmd]: http status code OK 21:45:53 DEBUG --- stdout --- 21:45:53 DEBUG --- status code --- 21:45:53 INFO [http_cmd]: http status code OK 21:45:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:45:53 INFO [http_cmd]: http status code OK 21:45:53 DEBUG --- status code --- 21:45:53 DEBUG --- status code --- 21:45:53 INFO [http_cmd]: http status code OK 21:45:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:45:53 INFO [http_cmd]: http status code OK 21:45:53 DEBUG --- status code --- 21:45:53 DEBUG --- status code --- 21:45:53 DEBUG --- status code --- 21:45:53 DEBUG --- status code --- 21:45:53 DEBUG --- status code --- 21:45:53 DEBUG --- status code --- 21:45:53 DEBUG --- status code --- 21:45:53 DEBUG --- status code --- 21:45:53 DEBUG --- http response --- 21:45:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:45:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:45:53 DEBUG --- status code --- 21:45:53 DEBUG --- status code --- 21:45:53 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 60m 0% 6218Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 9945m 33% 7718Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 21739m 72% 11475Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 85m 0% 3398Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 20185m 67% 15060Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 159m 0% 5208Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 22277m 74% 11766Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19741m 66% 3074Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 24646m 82% 11179Mi 10% gke-xlou-cdm-ds-0f601751-j488 10350m 34% 7275Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 20229m 67% 16626Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 10131m 33% 7953Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20763m 69% 14450Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 12070m 75% 8744Mi 14% 21:45:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:45:53 DEBUG --- status code --- 21:45:53 DEBUG --- http response --- 21:45:53 DEBUG --- status code --- 21:45:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:45:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:45:53 DEBUG --- status code --- 21:45:53 DEBUG --- http response --- 21:45:53 DEBUG --- status code --- 21:45:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:45:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:45:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:45:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:45:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:45:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:45:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:45:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:45:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:45:53 DEBUG --- http response --- 21:45:53 DEBUG --- http response --- 21:45:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:45:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:45:53 DEBUG --- stderr --- 21:45:53 DEBUG --- http response --- 21:45:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:45:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:45:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:45:53 DEBUG --- http response --- 21:45:53 DEBUG --- http response --- 21:45:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:45:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:45:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:45:53 DEBUG --- http response --- 21:45:53 DEBUG --- http response --- 21:45:53 DEBUG --- http response --- 21:45:53 DEBUG --- http response --- 21:45:53 DEBUG --- http response --- 21:45:53 DEBUG --- http response --- 21:45:53 DEBUG --- http response --- 21:45:53 DEBUG --- http response --- 21:45:53 DEBUG ----- output ----- 21:45:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:45:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:45:53 DEBUG --- http response --- 21:45:53 DEBUG --- http response --- 21:45:53 DEBUG 21:45:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:45:53 DEBUG --- http response --- 21:45:53 DEBUG --- http response --- 21:45:53 DEBUG ----- output ----- 21:45:53 DEBUG --- http response --- 21:45:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:45:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:45:53 DEBUG --- http response --- 21:45:53 DEBUG ----- output ----- 21:45:53 INFO [http_cmd]: http status code OK 21:45:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 21:45:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:45:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:45:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:45:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:45:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:45:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:45:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 21:45:53 DEBUG { 21:45:53 DEBUG ----- output ----- 21:45:53 DEBUG ----- output ----- 21:45:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:45:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:45:53 DEBUG ----- output ----- 21:45:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:45:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:45:53 DEBUG { 21:45:53 DEBUG ----- output ----- 21:45:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:45:53 DEBUG ----- output ----- 21:45:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:45:53 DEBUG { 21:45:53 DEBUG --- status code --- 21:45:53 DEBUG ----- output ----- 21:45:53 DEBUG ----- output ----- 21:45:53 DEBUG ----- output ----- 21:45:53 DEBUG ----- output ----- 21:45:53 DEBUG ----- output ----- 21:45:53 DEBUG ----- output ----- 21:45:53 DEBUG ----- output ----- 21:45:53 DEBUG ----- output ----- 21:45:53 DEBUG "status": "success", 21:45:53 DEBUG { 21:45:53 DEBUG { 21:45:53 DEBUG ----- output ----- 21:45:53 DEBUG ----- output ----- 21:45:53 DEBUG { 21:45:53 DEBUG ----- output ----- 21:45:53 DEBUG ----- output ----- 21:45:53 DEBUG "status": "success", 21:45:53 DEBUG { 21:45:53 DEBUG ----- output ----- 21:45:53 DEBUG { 21:45:53 DEBUG ----- output ----- 21:45:53 DEBUG "status": "success", 21:45:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:45:53 DEBUG { 21:45:53 DEBUG { 21:45:53 DEBUG { 21:45:53 DEBUG { 21:45:53 DEBUG { 21:45:53 DEBUG { 21:45:53 DEBUG "status": "success", 21:45:53 DEBUG { 21:45:53 DEBUG "data": { 21:45:53 DEBUG "status": "success", 21:45:53 DEBUG "status": "success", 21:45:53 DEBUG { 21:45:53 DEBUG { 21:45:53 DEBUG "status": "success", 21:45:53 DEBUG { 21:45:53 DEBUG { 21:45:53 DEBUG "data": { 21:45:53 DEBUG "status": "success", 21:45:53 DEBUG "data": { 21:45:53 DEBUG { 21:45:53 DEBUG "data": { 21:45:53 DEBUG --- http response --- 21:45:53 DEBUG { 21:45:53 DEBUG "status": "success", 21:45:53 DEBUG "status": "success", 21:45:53 DEBUG "status": "success", 21:45:53 DEBUG "status": "success", 21:45:53 DEBUG "status": "success", 21:45:53 DEBUG "data": { 21:45:53 DEBUG { 21:45:53 DEBUG "status": "success", 21:45:53 DEBUG "resultType": "vector", 21:45:53 DEBUG "data": { 21:45:53 DEBUG "data": { 21:45:53 DEBUG "status": "success", 21:45:53 DEBUG "status": "success", 21:45:53 DEBUG "data": { 21:45:53 DEBUG "status": "success", 21:45:53 DEBUG "status": "success", 21:45:53 DEBUG "resultType": "vector", 21:45:53 DEBUG "resultType": "vector", 21:45:53 DEBUG "status": "success", 21:45:53 DEBUG "status": "success", 21:45:53 DEBUG "resultType": "vector", 21:45:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:45:53 DEBUG "status": "success", 21:45:53 DEBUG "data": { 21:45:53 DEBUG "data": { 21:45:53 DEBUG "data": { 21:45:53 DEBUG "data": { 21:45:53 DEBUG "data": { 21:45:53 DEBUG "resultType": "vector", 21:45:53 DEBUG "status": "success", 21:45:53 DEBUG "data": { 21:45:53 DEBUG "result": [ 21:45:53 DEBUG "resultType": "vector", 21:45:53 DEBUG "resultType": "vector", 21:45:53 DEBUG "data": { 21:45:53 DEBUG "data": { 21:45:53 DEBUG "resultType": "vector", 21:45:53 DEBUG "data": { 21:45:53 DEBUG "data": { 21:45:53 DEBUG "result": [ 21:45:53 DEBUG "result": [ 21:45:53 DEBUG { 21:45:53 DEBUG "data": { 21:45:53 DEBUG "result": [ 21:45:53 DEBUG ----- output ----- 21:45:53 DEBUG "data": { 21:45:53 DEBUG "resultType": "vector", 21:45:53 DEBUG "resultType": "vector", 21:45:53 DEBUG "resultType": "vector", 21:45:53 DEBUG "resultType": "vector", 21:45:53 DEBUG "result": [ 21:45:53 DEBUG "result": [ 21:45:53 DEBUG "data": { 21:45:53 DEBUG "resultType": "vector", 21:45:53 DEBUG { 21:45:53 DEBUG "result": [ 21:45:53 DEBUG "result": [ 21:45:53 DEBUG { 21:45:53 DEBUG "resultType": "vector", 21:45:53 DEBUG "result": [ 21:45:53 DEBUG "resultType": "vector", 21:45:53 DEBUG "resultType": "vector", 21:45:53 DEBUG { 21:45:53 DEBUG "metric": { 21:45:53 DEBUG "data": { 21:45:53 DEBUG "resultType": "vector", 21:45:53 DEBUG { 21:45:53 DEBUG "metric": { 21:45:53 DEBUG "resultType": "vector", 21:45:53 DEBUG "result": [ 21:45:53 DEBUG "result": [ 21:45:53 DEBUG "result": [ 21:45:53 DEBUG { 21:45:53 DEBUG "resultType": "vector", 21:45:53 DEBUG { 21:45:53 DEBUG "resultType": "vector", 21:45:53 DEBUG "result": [ 21:45:53 DEBUG "metric": { 21:45:53 DEBUG { 21:45:53 DEBUG "metric": { 21:45:53 DEBUG "resultType": "vector", 21:45:53 DEBUG { 21:45:53 DEBUG "metric": { 21:45:53 DEBUG "result": [ 21:45:53 DEBUG "metric": { 21:45:53 DEBUG "pod": "ds-cts-0" 21:45:53 DEBUG "result": [ 21:45:53 DEBUG "result": [ 21:45:53 DEBUG "pod": "am-767f6b67b5-9zw22" 21:45:53 DEBUG "resultType": "vector", 21:45:53 DEBUG { 21:45:53 DEBUG "result": [ 21:45:53 DEBUG { 21:45:53 DEBUG { 21:45:53 DEBUG "metric": { 21:45:53 DEBUG "metric": { 21:45:53 DEBUG "result": [ 21:45:53 DEBUG "metric": { 21:45:53 DEBUG "result": [ 21:45:53 DEBUG { 21:45:53 DEBUG "pod": "ds-idrepo-0" 21:45:53 DEBUG "metric": { 21:45:53 DEBUG "pod": "am-767f6b67b5-9zw22" 21:45:53 DEBUG "result": [ 21:45:53 DEBUG "instance": "10.142.0.107:9100" 21:45:53 DEBUG }, 21:45:53 DEBUG { 21:45:53 DEBUG "metric": { 21:45:53 DEBUG }, 21:45:53 DEBUG { 21:45:53 DEBUG { 21:45:53 DEBUG }, 21:45:53 DEBUG "result": [ 21:45:53 DEBUG "status": "success", 21:45:53 DEBUG { 21:45:53 DEBUG "metric": { 21:45:53 DEBUG "pod": "am-767f6b67b5-9zw22" 21:45:53 DEBUG { 21:45:53 DEBUG "instance": "10.142.0.107:9100" 21:45:53 DEBUG { 21:45:53 DEBUG "pod": "ds-idrepo-0" 21:45:53 DEBUG { 21:45:53 DEBUG "metric": { 21:45:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:45:53 DEBUG "instance": "10.142.0.107:9100" 21:45:53 DEBUG }, 21:45:53 DEBUG { 21:45:53 DEBUG "value": [ 21:45:53 DEBUG "result": [ 21:45:53 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 21:45:53 DEBUG "pod": "am-767f6b67b5-9zw22" 21:45:53 DEBUG "value": [ 21:45:53 DEBUG "metric": { 21:45:53 DEBUG "pod": "am-767f6b67b5-9zw22" 21:45:53 DEBUG "value": [ 21:45:53 DEBUG { 21:45:53 DEBUG "data": { 21:45:53 DEBUG "metric": { 21:45:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:45:53 DEBUG }, 21:45:53 DEBUG "metric": { 21:45:53 DEBUG }, 21:45:53 DEBUG "metric": { 21:45:53 DEBUG }, 21:45:53 DEBUG "metric": { 21:45:53 DEBUG }, 21:45:53 DEBUG }, 21:45:53 DEBUG }, 21:45:53 DEBUG "value": [ 21:45:53 DEBUG "metric": { 21:45:53 DEBUG 21:45:53 DEBUG { 21:45:53 DEBUG }, 21:45:53 DEBUG }, 21:45:53 DEBUG 21:45:53 DEBUG }, 21:45:53 DEBUG "metric": { 21:45:53 DEBUG 21:45:53 DEBUG "metric": { 21:45:53 DEBUG "resultType": "vector", 21:45:53 DEBUG "pod": "ds-idrepo-0" 21:45:53 DEBUG }, 21:45:53 DEBUG "value": [ 21:45:53 DEBUG "pod": "am-767f6b67b5-9zw22" 21:45:53 DEBUG "value": [ 21:45:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:45:53 DEBUG "value": [ 21:45:53 DEBUG 21:45:53 DEBUG "pod": "ds-cts-0" 21:45:53 DEBUG "value": [ 21:45:53 DEBUG "value": [ 21:45:53 DEBUG 21:45:53 DEBUG "pod": "am-767f6b67b5-9zw22" 21:45:53 DEBUG 21:45:53 DEBUG "metric": { 21:45:53 DEBUG "value": [ 21:45:53 DEBUG "value": [ 21:45:53 DEBUG 21:45:53 DEBUG "value": [ 21:45:53 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 21:45:53 DEBUG 21:45:53 DEBUG "instance": "10.142.0.107:9100" 21:45:53 DEBUG "result": [ 21:45:53 DEBUG }, 21:45:53 DEBUG "value": [ 21:45:53 DEBUG 21:45:53 DEBUG }, 21:45:53 DEBUG 21:45:53 DEBUG }, 21:45:53 DEBUG "value": [ 21:45:53 DEBUG 21:45:53 DEBUG }, 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG }, 21:45:53 DEBUG 21:45:53 DEBUG "instance": "10.142.0.107:9100" 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG }, 21:45:53 DEBUG 21:45:53 DEBUG }, 21:45:53 DEBUG { 21:45:53 DEBUG "value": [ 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG "value": [ 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG "value": [ 21:45:53 DEBUG 21:45:53 DEBUG "value": [ 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG [...] 21:45:53 DEBUG [...] 21:45:53 DEBUG }, 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG [...] 21:45:53 DEBUG 21:45:53 DEBUG "value": [ 21:45:53 DEBUG [...] 21:45:53 DEBUG "value": [ 21:45:53 DEBUG "metric": { 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG [...] 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG "value": [ 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG "value": [ 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG "pod": "ds-cts-0" 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG [...] 21:45:53 DEBUG 21:45:53 DEBUG [...] 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG [...] 21:45:53 DEBUG 21:45:53 DEBUG [...] 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG [...] 21:45:53 DEBUG [...] 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG [...] 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG }, 21:45:53 DEBUG 21:45:53 DEBUG [...] 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG [...] 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG [...] 21:45:53 DEBUG 21:45:53 DEBUG [...] 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG "pod": "am-767f6b67b5-vbltz" 21:45:53 DEBUG "value": [ 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG "pod": "am-767f6b67b5-vbltz" 21:45:53 DEBUG "instance": "10.142.0.43:9100" 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG "pod": "ds-cts-2" 21:45:53 DEBUG 21:45:53 DEBUG [...] 21:45:53 DEBUG 21:45:53 DEBUG }, 21:45:53 DEBUG [...] 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG "pod": "idm-b85f9d978-wnhxb" 21:45:53 DEBUG "pod": "overseer-0-967d957f7-zk66v" 21:45:53 DEBUG [...] 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG }, 21:45:53 DEBUG }, 21:45:53 DEBUG [...] 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG [...] 21:45:53 DEBUG }, 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG [...] 21:45:53 DEBUG "value": [ 21:45:53 DEBUG 21:45:53 DEBUG [...] 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG }, 21:45:53 DEBUG }, 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG "pod": "ds-idrepo-2" 21:45:53 DEBUG 21:45:53 DEBUG "pod": "ds-idrepo-2" 21:45:53 DEBUG 21:45:53 DEBUG "value": [ 21:45:53 DEBUG "value": [ 21:45:53 DEBUG 21:45:53 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 21:45:53 DEBUG "pod": "am-767f6b67b5-vbltz" 21:45:53 DEBUG }, 21:45:53 DEBUG "value": [ 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG 1713645953, 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG "pod": "am-767f6b67b5-vbltz" 21:45:53 DEBUG "value": [ 21:45:53 DEBUG 21:45:53 DEBUG "instance": "10.142.0.43:9100" 21:45:53 DEBUG }, 21:45:53 DEBUG "pod": "ds-cts-2" 21:45:53 DEBUG }, 21:45:53 DEBUG "instance": "10.142.0.43:9100" 21:45:53 DEBUG 1713645953, 21:45:53 DEBUG 1713645953, 21:45:53 DEBUG 21:45:53 DEBUG }, 21:45:53 DEBUG "value": [ 21:45:53 DEBUG 1713645953, 21:45:53 DEBUG "pod": "am-767f6b67b5-vbltz" 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG "49.377" 21:45:53 DEBUG 21:45:53 DEBUG 21:45:53 DEBUG "pod": "am-767f6b67b5-vbltz" 21:45:53 DEBUG }, 21:45:53 DEBUG "value": [ 21:45:53 DEBUG 1713645953, 21:45:53 DEBUG 21:45:53 DEBUG }, 21:45:53 DEBUG "value": [ 21:45:53 DEBUG }, 21:45:53 DEBUG "value": [ 21:45:53 DEBUG "value": [ 21:45:53 DEBUG "3321.6" 21:45:53 DEBUG "100335262.83333333" 21:45:53 DEBUG 21:45:53 DEBUG "value": [ 21:45:53 DEBUG 1713645953, 21:45:53 DEBUG "3331.1" 21:45:53 DEBUG }, 21:45:53 DEBUG 21:45:53 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 21:45:53 DEBUG 21:45:53 DEBUG ] 21:45:53 DEBUG "pod": "ds-idrepo-2" 21:45:53 DEBUG 21:45:53 DEBUG }, 21:45:53 DEBUG "value": [ 21:45:53 DEBUG 1713645953, 21:45:53 DEBUG "0" 21:45:53 DEBUG "pod": "idm-b85f9d978-wnhxb" 21:45:53 DEBUG 1713645953, 21:45:53 DEBUG "value": [ 21:45:53 DEBUG 1713645953, 21:45:53 DEBUG "0" 21:45:53 DEBUG 1713645953, 21:45:53 DEBUG ] 21:45:53 DEBUG ] 21:45:53 DEBUG "instance": "10.142.0.43:9100" 21:45:53 DEBUG 1713645953, 21:45:53 DEBUG "0" 21:45:53 DEBUG ] 21:45:53 DEBUG "value": [ 21:45:53 DEBUG "pod": "am-767f6b67b5-vbltz" 21:45:53 DEBUG }, 21:45:53 DEBUG "instance": "10.142.0.43:9100" 21:45:53 DEBUG } 21:45:53 DEBUG }, 21:45:53 DEBUG "value": [ 21:45:53 DEBUG "value": [ 21:45:53 DEBUG 1713645953, 21:45:53 DEBUG "0" 21:45:53 DEBUG ] 21:45:53 DEBUG }, 21:45:53 DEBUG "100331679.96666667" 21:45:53 DEBUG 1713645953, 21:45:53 DEBUG ] 21:45:53 DEBUG }, 21:45:53 DEBUG "0" 21:45:53 DEBUG } 21:45:53 DEBUG } 21:45:53 DEBUG }, 21:45:53 DEBUG "926306304" 21:45:53 DEBUG ] 21:45:53 DEBUG } 21:45:53 DEBUG 1713645953, 21:45:53 DEBUG }, 21:45:53 DEBUG "value": [ 21:45:53 DEBUG }, 21:45:53 DEBUG ] 21:45:53 DEBUG 1713645953, 21:45:53 DEBUG "pod": "ds-cts-2" 21:45:53 DEBUG 1713645953, 21:45:53 DEBUG "3315.2666666666664" 21:45:53 DEBUG ] 21:45:53 DEBUG } 21:45:53 DEBUG "value": [ 21:45:53 DEBUG ] 21:45:53 DEBUG "0" 21:45:53 DEBUG } 21:45:53 DEBUG "value": [ 21:45:53 DEBUG ] 21:45:53 DEBUG ] 21:45:53 DEBUG ] 21:45:53 DEBUG "value": [ 21:45:53 DEBUG ] 21:45:53 DEBUG } 21:45:53 DEBUG ] 21:45:53 DEBUG } 21:45:53 DEBUG "value": [ 21:45:53 DEBUG 1713645953, 21:45:53 DEBUG "value": [ 21:45:53 DEBUG } 21:45:53 DEBUG "0" 21:45:53 DEBUG }, 21:45:53 DEBUG "0" 21:45:53 DEBUG ] 21:45:53 DEBUG } 21:45:53 DEBUG ] 21:45:53 DEBUG 1713645953, 21:45:53 DEBUG } 21:45:53 DEBUG ] 21:45:53 DEBUG ] 21:45:53 DEBUG 1713645953, 21:45:53 DEBUG } 21:45:53 DEBUG } 21:45:53 DEBUG } 21:45:53 DEBUG 1713645953, 21:45:53 DEBUG } 21:45:53 DEBUG ] 21:45:53 DEBUG } 21:45:53 DEBUG 1713645953, 21:45:53 DEBUG "3317.5" 21:45:53 DEBUG "11.839784437699345" 21:45:53 DEBUG 1713645953, 21:45:53 DEBUG } 21:45:53 DEBUG ] 21:45:53 DEBUG "value": [ 21:45:53 DEBUG ] 21:45:53 DEBUG } 21:45:53 DEBUG ] 21:45:53 DEBUG } 21:45:53 DEBUG "787.5405618874054" 21:45:53 DEBUG ] 21:45:53 DEBUG } 21:45:53 DEBUG } 21:45:53 DEBUG "3958971632640" 21:45:53 DEBUG ] 21:45:53 DEBUG } 21:45:53 DEBUG } 21:45:53 DEBUG "0.07400000000000091" 21:45:53 DEBUG ] 21:45:53 DEBUG } 21:45:53 DEBUG "599648" 21:45:53 DEBUG ] 21:45:53 DEBUG ] 21:45:53 DEBUG "0.2722666666666347" 21:45:53 DEBUG } 21:45:53 DEBUG ] 21:45:53 DEBUG } 21:45:53 DEBUG ] 21:45:53 DEBUG } 21:45:53 DEBUG } 21:45:53 DEBUG ] 21:45:53 DEBUG } 21:45:53 DEBUG ] 21:45:53 DEBUG } 21:45:53 DEBUG ] 21:45:53 DEBUG } 21:45:53 DEBUG ] 21:45:53 DEBUG } 21:45:53 DEBUG } 21:45:53 DEBUG ] 21:45:53 DEBUG } 21:45:53 DEBUG } 21:45:53 DEBUG ] 21:45:53 DEBUG 1713645953, 21:45:53 DEBUG } 21:45:53 DEBUG ] 21:45:53 DEBUG } 21:45:53 DEBUG } 21:45:53 DEBUG } 21:45:53 DEBUG } 21:45:53 DEBUG } 21:45:53 DEBUG } 21:45:53 DEBUG } 21:45:53 DEBUG } 21:45:53 DEBUG } 21:45:53 DEBUG } 21:45:53 DEBUG ] 21:45:53 DEBUG ] 21:45:53 DEBUG } 21:45:53 DEBUG "0.38686666666666664" 21:45:53 DEBUG } 21:45:53 DEBUG } 21:45:53 DEBUG } 21:45:53 DEBUG ] 21:45:53 DEBUG } 21:45:53 DEBUG ] 21:45:53 DEBUG ] 21:45:53 DEBUG ] 21:45:53 DEBUG } 21:45:53 DEBUG } 21:45:53 DEBUG ] 21:45:53 DEBUG ] 21:45:53 DEBUG } 21:45:53 DEBUG } 21:45:53 DEBUG } 21:45:53 DEBUG } 21:45:53 DEBUG } 21:45:53 DEBUG } 21:45:53 DEBUG } 21:45:53 DEBUG } 21:45:53 DEBUG } 21:45:53 DEBUG } 21:45:53 DEBUG } 21:45:53 DEBUG } 21:45:53 DEBUG } 21:45:53 DEBUG ] 21:45:53 DEBUG } 21:45:53 DEBUG } 21:45:53 DEBUG } 21:46:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713646012" 21:46:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646012" 21:46:52 ERROR [http_cmd]: ERROR 21:46:52 ERROR --- status code --- 21:46:52 ERROR http status code is 400 (expected [200, 401, 403, 404]) 21:46:52 INFO [http_cmd]: http status code OK 21:46:52 ERROR --- http response --- 21:46:52 DEBUG --- status code --- 21:46:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:46:52 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 21:46:52 DEBUG --- http response --- 21:46:52 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 21:46:52 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 21:46:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713646012" 21:46:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646012" 21:46:52 INFO [http_cmd]: http status code OK 21:46:52 DEBUG --- status code --- 21:46:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:46:52 DEBUG --- http response --- 21:46:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:46:52 DEBUG ----- output ----- 21:46:52 DEBUG { 21:46:52 DEBUG "status": "success", 21:46:52 DEBUG "data": { 21:46:52 DEBUG "resultType": "vector", 21:46:52 DEBUG "result": [ 21:46:52 DEBUG { 21:46:52 DEBUG "metric": { 21:46:52 DEBUG "pod": "ds-idrepo-0" 21:46:52 DEBUG }, 21:46:52 DEBUG "value": [ 21:46:52 DEBUG 21:46:52 DEBUG 21:46:52 DEBUG 21:46:52 DEBUG [...] 21:46:52 DEBUG 21:46:52 DEBUG 21:46:52 DEBUG 21:46:52 DEBUG "pod": "ds-cts-2" 21:46:52 DEBUG }, 21:46:52 DEBUG "value": [ 21:46:52 DEBUG 1713646012, 21:46:52 DEBUG "0" 21:46:52 DEBUG ] 21:46:52 DEBUG } 21:46:52 DEBUG ] 21:46:52 DEBUG } 21:46:52 DEBUG } 21:46:52 INFO [http_cmd]: http status code OK 21:46:52 DEBUG --- status code --- 21:46:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:46:52 DEBUG --- http response --- 21:46:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 21:46:52 DEBUG ----- output ----- 21:46:52 DEBUG { 21:46:52 DEBUG "status": "success", 21:46:52 DEBUG "data": { 21:46:52 DEBUG "resultType": "vector", 21:46:52 DEBUG "result": [ 21:46:52 DEBUG { 21:46:52 DEBUG "metric": { 21:46:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:46:52 DEBUG }, 21:46:52 DEBUG "value": [ 21:46:52 DEBUG 21:46:52 DEBUG 21:46:52 DEBUG 21:46:52 DEBUG [...] 21:46:52 DEBUG 21:46:52 DEBUG 21:46:52 DEBUG 21:46:52 DEBUG "pod": "am-767f6b67b5-vbltz" 21:46:52 DEBUG }, 21:46:52 DEBUG "value": [ 21:46:52 DEBUG 1713646012, 21:46:52 DEBUG "252494008.7537013" 21:46:52 DEBUG ] 21:46:52 DEBUG } 21:46:52 DEBUG ] 21:46:52 DEBUG } 21:46:52 DEBUG } 21:46:52 INFO 21:46:52 INFO [loop_until]: kubectl --namespace=xlou top pods 21:46:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:46:53 INFO [loop_until]: OK (rc = 0) 21:46:53 DEBUG --- stdout --- 21:46:53 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 21845m 8761Mi am-767f6b67b5-qcqz2 22248m 8878Mi am-767f6b67b5-vbltz 24018m 8934Mi ds-cts-0 20440m 11854Mi ds-cts-1 19875m 11890Mi ds-cts-2 20671m 12045Mi ds-idrepo-0 9703m 3859Mi ds-idrepo-1 9486m 3981Mi ds-idrepo-2 9623m 3870Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1178Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-7dd48676f6-tdxl9 14m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-967d957f7-zk66v 11781m 796Mi 21:46:53 DEBUG --- stderr --- 21:46:53 DEBUG 21:46:53 INFO 21:46:53 INFO [loop_until]: kubectl --namespace=xlou top node 21:46:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:46:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646013" 21:46:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713646013" 21:46:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646013" 21:46:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646013" 21:46:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646013" 21:46:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646013" 21:46:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646013" 21:46:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646013" 21:46:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713646013" 21:46:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646013" 21:46:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646013" 21:46:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646013" 21:46:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646013" 21:46:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646013" 21:46:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646013" 21:46:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646013" 21:46:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646013" 21:46:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646013" 21:46:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646013" 21:46:53 INFO [http_cmd]: http status code OK 21:46:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646013" 21:46:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713646013" 21:46:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646013" 21:46:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646013" 21:46:53 INFO [http_cmd]: http status code OK 21:46:53 INFO [http_cmd]: http status code OK 21:46:53 INFO [http_cmd]: http status code OK 21:46:53 INFO [http_cmd]: http status code OK 21:46:53 INFO [loop_until]: OK (rc = 0) 21:46:53 INFO [http_cmd]: http status code OK 21:46:53 INFO [http_cmd]: http status code OK 21:46:53 INFO [http_cmd]: http status code OK 21:46:53 INFO [http_cmd]: http status code OK 21:46:53 INFO [http_cmd]: http status code OK 21:46:53 INFO [http_cmd]: http status code OK 21:46:53 DEBUG --- status code --- 21:46:53 INFO [http_cmd]: http status code OK 21:46:53 INFO [http_cmd]: http status code OK 21:46:53 INFO [http_cmd]: http status code OK 21:46:53 INFO [http_cmd]: http status code OK 21:46:53 DEBUG --- status code --- 21:46:53 INFO [http_cmd]: http status code OK 21:46:53 INFO [http_cmd]: http status code OK 21:46:53 INFO [http_cmd]: http status code OK 21:46:53 DEBUG --- status code --- 21:46:53 DEBUG --- status code --- 21:46:53 INFO [http_cmd]: http status code OK 21:46:53 DEBUG --- status code --- 21:46:53 DEBUG --- stdout --- 21:46:53 DEBUG --- status code --- 21:46:53 INFO [http_cmd]: http status code OK 21:46:53 DEBUG --- status code --- 21:46:53 INFO [http_cmd]: http status code OK 21:46:53 DEBUG --- status code --- 21:46:53 DEBUG --- status code --- 21:46:53 DEBUG --- status code --- 21:46:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:46:53 DEBUG --- status code --- 21:46:53 DEBUG --- status code --- 21:46:53 DEBUG --- status code --- 21:46:53 DEBUG --- status code --- 21:46:53 DEBUG --- status code --- 21:46:53 INFO [http_cmd]: http status code OK 21:46:53 INFO [http_cmd]: http status code OK 21:46:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:46:53 DEBUG --- status code --- 21:46:53 DEBUG --- status code --- 21:46:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:46:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:46:53 DEBUG --- status code --- 21:46:53 DEBUG --- status code --- 21:46:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:46:53 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 60m 0% 6215Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 10115m 33% 7599Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 21760m 72% 11473Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 85m 0% 3384Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 19705m 66% 15042Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 157m 0% 5217Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 22173m 74% 11769Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19679m 65% 3063Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 24568m 82% 11185Mi 10% gke-xlou-cdm-ds-0f601751-j488 10335m 34% 7289Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 20243m 67% 16952Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 10193m 34% 7967Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20802m 69% 14716Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 12048m 75% 8743Mi 14% 21:46:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:46:53 DEBUG --- status code --- 21:46:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:46:53 DEBUG --- status code --- 21:46:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:46:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:46:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:46:53 DEBUG --- http response --- 21:46:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:46:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:46:53 DEBUG --- http response --- 21:46:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:46:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:46:53 DEBUG --- status code --- 21:46:53 DEBUG --- status code --- 21:46:53 DEBUG --- http response --- 21:46:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:46:53 DEBUG --- http response --- 21:46:53 DEBUG --- http response --- 21:46:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:46:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:46:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:46:53 DEBUG --- http response --- 21:46:53 DEBUG --- stderr --- 21:46:53 DEBUG 21:46:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:46:53 DEBUG --- http response --- 21:46:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:46:53 DEBUG --- http response --- 21:46:53 DEBUG --- http response --- 21:46:53 DEBUG --- http response --- 21:46:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:46:53 DEBUG --- http response --- 21:46:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:46:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:46:53 DEBUG --- http response --- 21:46:53 DEBUG --- http response --- 21:46:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:46:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:46:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:46:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:46:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:46:53 DEBUG ----- output ----- 21:46:53 DEBUG --- http response --- 21:46:53 DEBUG --- http response --- 21:46:53 DEBUG --- http response --- 21:46:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:46:53 DEBUG ----- output ----- 21:46:53 DEBUG --- http response --- 21:46:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:46:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:46:53 DEBUG ----- output ----- 21:46:53 DEBUG { 21:46:53 DEBUG "status": "success", 21:46:53 DEBUG ----- output ----- 21:46:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:46:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:46:53 DEBUG ----- output ----- 21:46:53 DEBUG { 21:46:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 21:46:53 DEBUG --- http response --- 21:46:53 DEBUG --- http response --- 21:46:53 DEBUG ----- output ----- 21:46:53 DEBUG ----- output ----- 21:46:53 DEBUG --- http response --- 21:46:53 DEBUG { 21:46:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 21:46:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:46:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:46:53 DEBUG { 21:46:53 DEBUG "status": "success", 21:46:53 DEBUG "data": { 21:46:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:46:53 DEBUG "data": { 21:46:53 DEBUG --- http response --- 21:46:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:46:53 DEBUG --- http response --- 21:46:53 DEBUG { 21:46:53 DEBUG ----- output ----- 21:46:53 DEBUG { 21:46:53 DEBUG ----- output ----- 21:46:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 21:46:53 DEBUG ----- output ----- 21:46:53 DEBUG { 21:46:53 DEBUG { 21:46:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:46:53 DEBUG { 21:46:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:46:53 DEBUG "status": "success", 21:46:53 DEBUG ----- output ----- 21:46:53 DEBUG ----- output ----- 21:46:53 DEBUG ----- output ----- 21:46:53 DEBUG --- http response --- 21:46:53 DEBUG "resultType": "vector", 21:46:53 DEBUG "result": [ 21:46:53 DEBUG ----- output ----- 21:46:53 DEBUG "resultType": "vector", 21:46:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:46:53 DEBUG ----- output ----- 21:46:53 DEBUG "status": "success", 21:46:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:46:53 DEBUG "status": "success", 21:46:53 DEBUG "data": { 21:46:53 DEBUG "resultType": "vector", 21:46:53 DEBUG ----- output ----- 21:46:53 DEBUG { 21:46:53 DEBUG "status": "success", 21:46:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:46:53 DEBUG ----- output ----- 21:46:53 DEBUG "status": "success", 21:46:53 DEBUG ----- output ----- 21:46:53 DEBUG "data": { 21:46:53 DEBUG { 21:46:53 DEBUG "status": "success", 21:46:53 DEBUG { 21:46:53 DEBUG "status": "success", 21:46:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:46:53 DEBUG { 21:46:53 DEBUG { 21:46:53 DEBUG "result": [ 21:46:53 DEBUG ----- output ----- 21:46:53 DEBUG { 21:46:53 DEBUG "status": "success", 21:46:53 DEBUG ----- output ----- 21:46:53 DEBUG "result": [ 21:46:53 DEBUG { 21:46:53 DEBUG "metric": { 21:46:53 DEBUG "status": "success", 21:46:53 DEBUG "status": "success", 21:46:53 DEBUG "status": "success", 21:46:53 DEBUG ----- output ----- 21:46:53 DEBUG { 21:46:53 DEBUG "data": { 21:46:53 DEBUG { 21:46:53 DEBUG "resultType": "vector", 21:46:53 DEBUG "data": { 21:46:53 DEBUG "data": { 21:46:53 DEBUG ----- output ----- 21:46:53 DEBUG { 21:46:53 DEBUG ----- output ----- 21:46:53 DEBUG { 21:46:53 DEBUG "status": "success", 21:46:53 DEBUG { 21:46:53 DEBUG { 21:46:53 DEBUG "data": { 21:46:53 DEBUG "data": { 21:46:53 DEBUG "resultType": "vector", 21:46:53 DEBUG "data": { 21:46:53 DEBUG "pod": "am-767f6b67b5-9zw22" 21:46:53 DEBUG { 21:46:53 DEBUG "data": { 21:46:53 DEBUG "resultType": "vector", 21:46:53 DEBUG "data": { 21:46:53 DEBUG "resultType": "vector", 21:46:53 DEBUG "status": "success", 21:46:53 DEBUG "resultType": "vector", 21:46:53 DEBUG "status": "success", 21:46:53 DEBUG "result": [ 21:46:53 DEBUG "resultType": "vector", 21:46:53 DEBUG "resultType": "vector", 21:46:53 DEBUG { 21:46:53 DEBUG "status": "success", 21:46:53 DEBUG "status": "success", 21:46:53 DEBUG "data": { 21:46:53 DEBUG "metric": { 21:46:53 DEBUG "status": "success", 21:46:53 DEBUG "metric": { 21:46:53 DEBUG "resultType": "vector", 21:46:53 DEBUG { 21:46:53 DEBUG "status": "success", 21:46:53 DEBUG "resultType": "vector", 21:46:53 DEBUG }, 21:46:53 DEBUG "status": "success", 21:46:53 DEBUG "result": [ 21:46:53 DEBUG "data": { 21:46:53 DEBUG { 21:46:53 DEBUG "result": [ 21:46:53 DEBUG "data": { 21:46:53 DEBUG "result": [ 21:46:53 DEBUG "data": { 21:46:53 DEBUG { 21:46:53 DEBUG "result": [ 21:46:53 DEBUG "result": [ 21:46:53 DEBUG "status": "success", 21:46:53 DEBUG "data": { 21:46:53 DEBUG "resultType": "vector", 21:46:53 DEBUG "result": [ 21:46:53 DEBUG "resultType": "vector", 21:46:53 DEBUG "data": { 21:46:53 DEBUG "instance": "10.142.0.107:9100" 21:46:53 DEBUG "result": [ 21:46:53 DEBUG "data": { 21:46:53 DEBUG "resultType": "vector", 21:46:53 DEBUG "result": [ 21:46:53 DEBUG "value": [ 21:46:53 DEBUG 21:46:53 DEBUG { 21:46:53 DEBUG "resultType": "vector", 21:46:53 DEBUG "status": "success", 21:46:53 DEBUG { 21:46:53 DEBUG "resultType": "vector", 21:46:53 DEBUG { 21:46:53 DEBUG "resultType": "vector", 21:46:53 DEBUG "metric": { 21:46:53 DEBUG { 21:46:53 DEBUG { 21:46:53 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 21:46:53 DEBUG { 21:46:53 DEBUG "data": { 21:46:53 DEBUG "result": [ 21:46:53 DEBUG "data": { 21:46:53 DEBUG "resultType": "vector", 21:46:53 DEBUG }, 21:46:53 DEBUG { 21:46:53 DEBUG "result": [ 21:46:53 DEBUG { 21:46:53 DEBUG "result": [ 21:46:53 DEBUG 21:46:53 DEBUG "metric": { 21:46:53 DEBUG "data": { 21:46:53 DEBUG "result": [ 21:46:53 DEBUG "data": { 21:46:53 DEBUG "resultType": "vector", 21:46:53 DEBUG "result": [ 21:46:53 DEBUG "metric": { 21:46:53 DEBUG "result": [ 21:46:53 DEBUG "pod": "am-767f6b67b5-9zw22" 21:46:53 DEBUG "metric": { 21:46:53 DEBUG "metric": { 21:46:53 DEBUG }, 21:46:53 DEBUG "value": [ 21:46:53 DEBUG 21:46:53 DEBUG { 21:46:53 DEBUG "resultType": "vector", 21:46:53 DEBUG "result": [ 21:46:53 DEBUG "value": [ 21:46:53 DEBUG "metric": { 21:46:53 DEBUG { 21:46:53 DEBUG "metric": { 21:46:53 DEBUG { 21:46:53 DEBUG 21:46:53 DEBUG [...] 21:46:53 DEBUG "resultType": "vector", 21:46:53 DEBUG { 21:46:53 DEBUG "result": [ 21:46:53 DEBUG "metric": { 21:46:53 DEBUG { 21:46:53 DEBUG "pod": "ds-cts-0" 21:46:53 DEBUG { 21:46:53 DEBUG }, 21:46:53 DEBUG "pod": "am-767f6b67b5-9zw22" 21:46:53 DEBUG }, 21:46:53 DEBUG "metric": { 21:46:53 DEBUG 21:46:53 DEBUG "resultType": "vector", 21:46:53 DEBUG "metric": { 21:46:53 DEBUG "result": [ 21:46:53 DEBUG { 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG "metric": { 21:46:53 DEBUG "pod": "ds-idrepo-0" 21:46:53 DEBUG "metric": { 21:46:53 DEBUG "pod": "am-767f6b67b5-9zw22" 21:46:53 DEBUG 21:46:53 DEBUG "result": [ 21:46:53 DEBUG "metric": { 21:46:53 DEBUG { 21:46:53 DEBUG "metric": { 21:46:53 DEBUG "metric": { 21:46:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:46:53 DEBUG "metric": { 21:46:53 DEBUG "instance": "10.142.0.107:9100" 21:46:53 DEBUG "value": [ 21:46:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:46:53 DEBUG "pod": "ds-cts-0" 21:46:53 DEBUG 21:46:53 DEBUG "result": [ 21:46:53 DEBUG "pod": "ds-idrepo-0" 21:46:53 DEBUG { 21:46:53 DEBUG "metric": { 21:46:53 DEBUG "pod": "ds-cts-0" 21:46:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:46:53 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 21:46:53 DEBUG }, 21:46:53 DEBUG "instance": "10.142.0.107:9100" 21:46:53 DEBUG }, 21:46:53 DEBUG 21:46:53 DEBUG { 21:46:53 DEBUG "pod": "am-767f6b67b5-9zw22" 21:46:53 DEBUG "instance": "10.142.0.107:9100" 21:46:53 DEBUG }, 21:46:53 DEBUG "instance": "10.142.0.107:9100" 21:46:53 DEBUG }, 21:46:53 DEBUG }, 21:46:53 DEBUG 21:46:53 DEBUG }, 21:46:53 DEBUG "value": [ 21:46:53 DEBUG }, 21:46:53 DEBUG [...] 21:46:53 DEBUG { 21:46:53 DEBUG }, 21:46:53 DEBUG "metric": { 21:46:53 DEBUG }, 21:46:53 DEBUG }, 21:46:53 DEBUG "value": [ 21:46:53 DEBUG 21:46:53 DEBUG }, 21:46:53 DEBUG }, 21:46:53 DEBUG "value": [ 21:46:53 DEBUG 21:46:53 DEBUG "metric": { 21:46:53 DEBUG }, 21:46:53 DEBUG }, 21:46:53 DEBUG "value": [ 21:46:53 DEBUG }, 21:46:53 DEBUG "value": [ 21:46:53 DEBUG "value": [ 21:46:53 DEBUG 21:46:53 DEBUG "value": [ 21:46:53 DEBUG 21:46:53 DEBUG "value": [ 21:46:53 DEBUG 21:46:53 DEBUG "metric": { 21:46:53 DEBUG "value": [ 21:46:53 DEBUG "pod": "am-767f6b67b5-9zw22" 21:46:53 DEBUG "value": [ 21:46:53 DEBUG "value": [ 21:46:53 DEBUG 21:46:53 DEBUG [...] 21:46:53 DEBUG "value": [ 21:46:53 DEBUG "value": [ 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG "pod": "ds-idrepo-0" 21:46:53 DEBUG "value": [ 21:46:53 DEBUG "value": [ 21:46:53 DEBUG 21:46:53 DEBUG "value": [ 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG "pod": "am-767f6b67b5-9zw22" 21:46:53 DEBUG 21:46:53 DEBUG }, 21:46:53 DEBUG "value": [ 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG "pod": "am-767f6b67b5-vbltz" 21:46:53 DEBUG }, 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG [...] 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG }, 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG [...] 21:46:53 DEBUG }, 21:46:53 DEBUG "value": [ 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG [...] 21:46:53 DEBUG 21:46:53 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 21:46:53 DEBUG "value": [ 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG [...] 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG "value": [ 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG [...] 21:46:53 DEBUG 21:46:53 DEBUG [...] 21:46:53 DEBUG [...] 21:46:53 DEBUG 21:46:53 DEBUG [...] 21:46:53 DEBUG 21:46:53 DEBUG [...] 21:46:53 DEBUG }, 21:46:53 DEBUG 21:46:53 DEBUG [...] 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG [...] 21:46:53 DEBUG 21:46:53 DEBUG "instance": "10.142.0.43:9100" 21:46:53 DEBUG [...] 21:46:53 DEBUG [...] 21:46:53 DEBUG 21:46:53 DEBUG 1713646013, 21:46:53 DEBUG 21:46:53 DEBUG [...] 21:46:53 DEBUG [...] 21:46:53 DEBUG 21:46:53 DEBUG [...] 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG "value": [ 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG [...] 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG }, 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG "3329" 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 1713646013, 21:46:53 DEBUG [...] 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG [...] 21:46:53 DEBUG "value": [ 21:46:53 DEBUG 21:46:53 DEBUG "pod": "am-767f6b67b5-vbltz" 21:46:53 DEBUG 21:46:53 DEBUG ] 21:46:53 DEBUG [...] 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG "pod": "am-767f6b67b5-vbltz" 21:46:53 DEBUG "instance": "10.142.0.43:9100" 21:46:53 DEBUG 21:46:53 DEBUG "pod": "am-767f6b67b5-vbltz" 21:46:53 DEBUG 21:46:53 DEBUG "11.808506753901316" 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 21:46:53 DEBUG 21:46:53 DEBUG 1713646013, 21:46:53 DEBUG "100068520.9" 21:46:53 DEBUG }, 21:46:53 DEBUG 21:46:53 DEBUG } 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG "instance": "10.142.0.43:9100" 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG "pod": "am-767f6b67b5-vbltz" 21:46:53 DEBUG }, 21:46:53 DEBUG }, 21:46:53 DEBUG "pod": "am-767f6b67b5-vbltz" 21:46:53 DEBUG }, 21:46:53 DEBUG "pod": "ds-cts-2" 21:46:53 DEBUG }, 21:46:53 DEBUG 21:46:53 DEBUG 21:46:53 DEBUG "pod": "ds-idrepo-2" 21:46:53 DEBUG "pod": "am-767f6b67b5-vbltz" 21:46:53 DEBUG }, 21:46:53 DEBUG 21:46:53 DEBUG ] 21:46:53 DEBUG "instance": "10.142.0.43:9100" 21:46:53 DEBUG "value": [ 21:46:53 DEBUG 21:46:53 DEBUG ] 21:46:53 DEBUG 21:46:53 DEBUG "pod": "am-767f6b67b5-vbltz" 21:46:53 DEBUG }, 21:46:53 DEBUG "pod": "ds-cts-2" 21:46:53 DEBUG "instance": "10.142.0.43:9100" 21:46:53 DEBUG }, 21:46:53 DEBUG "value": [ 21:46:53 DEBUG "value": [ 21:46:53 DEBUG }, 21:46:53 DEBUG "value": [ 21:46:53 DEBUG "value": [ 21:46:53 DEBUG ] 21:46:53 DEBUG } 21:46:53 DEBUG "pod": "am-767f6b67b5-vbltz" 21:46:53 DEBUG }, 21:46:53 DEBUG }, 21:46:53 DEBUG "value": [ 21:46:53 DEBUG 21:46:53 DEBUG } 21:46:53 DEBUG }, 21:46:53 DEBUG 1713646013, 21:46:53 DEBUG "pod": "ds-idrepo-2" 21:46:53 DEBUG } 21:46:53 DEBUG 21:46:53 DEBUG }, 21:46:53 DEBUG "value": [ 21:46:53 DEBUG }, 21:46:53 DEBUG }, 21:46:53 DEBUG "value": [ 21:46:53 DEBUG 1713646013, 21:46:53 DEBUG 1713646013, 21:46:53 DEBUG "3963066375168" 21:46:53 DEBUG ] 21:46:53 DEBUG 1713646013, 21:46:53 DEBUG 1713646013, 21:46:53 DEBUG 21:46:53 DEBUG }, 21:46:53 DEBUG "value": [ 21:46:53 DEBUG "value": [ 21:46:53 DEBUG 1713646013, 21:46:53 DEBUG "pod": "ds-cts-2" 21:46:53 DEBUG ] 21:46:53 DEBUG "value": [ 21:46:53 DEBUG "3332.6" 21:46:53 DEBUG }, 21:46:53 DEBUG "value": [ 21:46:53 DEBUG "pod": "ds-idrepo-2" 21:46:53 DEBUG "value": [ 21:46:53 DEBUG 1713646013, 21:46:53 DEBUG "value": [ 21:46:53 DEBUG "value": [ 21:46:53 DEBUG 1713646013, 21:46:53 DEBUG "49.377" 21:46:53 DEBUG } 21:46:53 DEBUG "value": [ 21:46:53 DEBUG "0.3981333333333334" 21:46:53 DEBUG "0" 21:46:53 DEBUG ] 21:46:53 DEBUG "pod": "am-767f6b67b5-vbltz" 21:46:53 DEBUG "value": [ 21:46:53 DEBUG 1713646013, 21:46:53 DEBUG 1713646013, 21:46:53 DEBUG "929574912" 21:46:53 DEBUG }, 21:46:53 DEBUG } 21:46:53 DEBUG } 21:46:53 DEBUG ] 21:46:53 DEBUG 1713646013, 21:46:53 DEBUG }, 21:46:53 DEBUG } 21:46:53 DEBUG 1713646013, 21:46:53 DEBUG "0.04700000000000462" 21:46:53 DEBUG 1713646013, 21:46:53 DEBUG 1713646013, 21:46:53 DEBUG "431.34858262930015" 21:46:53 DEBUG ] 21:46:53 DEBUG ] 21:46:53 DEBUG 1713646013, 21:46:53 DEBUG ] 21:46:53 DEBUG ] 21:46:53 DEBUG } 21:46:53 DEBUG }, 21:46:53 DEBUG 1713646013, 21:46:53 DEBUG "3330.7666666666664" 21:46:53 DEBUG "0" 21:46:53 DEBUG ] 21:46:53 DEBUG "value": [ 21:46:53 DEBUG 1713646013, 21:46:53 DEBUG } 21:46:53 DEBUG "0" 21:46:53 DEBUG "value": [ 21:46:53 DEBUG "3328" 21:46:53 DEBUG ] 21:46:53 DEBUG } 21:46:53 DEBUG ] 21:46:53 DEBUG ] 21:46:53 DEBUG } 21:46:53 DEBUG } 21:46:53 DEBUG "17.081772271819982" 21:46:53 DEBUG } 21:46:53 DEBUG } 21:46:53 DEBUG } 21:46:53 DEBUG "value": [ 21:46:53 DEBUG ] 21:46:53 DEBUG ] 21:46:53 DEBUG "0" 21:46:53 DEBUG ] 21:46:53 DEBUG } 21:46:53 DEBUG "100063277.69999999" 21:46:53 DEBUG ] 21:46:53 DEBUG ] 21:46:53 DEBUG 1713646013, 21:46:53 DEBUG ] 21:46:53 DEBUG } 21:46:53 DEBUG "0.15199999999997543" 21:46:53 DEBUG ] 21:46:53 DEBUG } 21:46:53 DEBUG "0" 21:46:53 DEBUG } 21:46:53 DEBUG ] 21:46:53 DEBUG ] 21:46:53 DEBUG } 21:46:53 DEBUG 1713646013, 21:46:53 DEBUG } 21:46:53 DEBUG } 21:46:53 DEBUG 1713646013, 21:46:53 DEBUG } 21:46:53 DEBUG ] 21:46:53 DEBUG ] 21:46:53 DEBUG } 21:46:53 DEBUG } 21:46:53 DEBUG "0" 21:46:53 DEBUG } 21:46:53 DEBUG } 21:46:53 DEBUG ] 21:46:53 DEBUG } 21:46:53 DEBUG ] 21:46:53 DEBUG ] 21:46:53 DEBUG } 21:46:53 DEBUG } 21:46:53 DEBUG "599524" 21:46:53 DEBUG ] 21:46:53 DEBUG ] 21:46:53 DEBUG ] 21:46:53 DEBUG "0" 21:46:53 DEBUG ] 21:46:53 DEBUG } 21:46:53 DEBUG } 21:46:53 DEBUG ] 21:46:53 DEBUG } 21:46:53 DEBUG ] 21:46:53 DEBUG } 21:46:53 DEBUG } 21:46:53 DEBUG } 21:46:53 DEBUG } 21:46:53 DEBUG } 21:46:53 DEBUG ] 21:46:53 DEBUG } 21:46:53 DEBUG } 21:46:53 DEBUG } 21:46:53 DEBUG ] 21:46:53 DEBUG } 21:46:53 DEBUG } 21:46:53 DEBUG ] 21:46:53 DEBUG } 21:46:53 DEBUG ] 21:46:53 DEBUG } 21:46:53 DEBUG ] 21:46:53 DEBUG ] 21:46:53 DEBUG ] 21:46:53 DEBUG } 21:46:53 DEBUG ] 21:46:53 DEBUG } 21:46:53 DEBUG } 21:46:53 DEBUG ] 21:46:53 DEBUG } 21:46:53 DEBUG } 21:46:53 DEBUG } 21:46:53 DEBUG } 21:46:53 DEBUG } 21:46:53 DEBUG } 21:46:53 DEBUG } 21:46:53 DEBUG } 21:46:53 DEBUG } 21:46:53 DEBUG } 21:46:53 DEBUG } 21:46:53 DEBUG ] 21:46:53 DEBUG } 21:46:53 DEBUG } 21:46:53 DEBUG } 21:46:53 DEBUG } 21:46:53 DEBUG } 21:46:53 DEBUG } 21:46:53 DEBUG } 21:46:53 DEBUG } 21:46:53 DEBUG } 21:47:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713646072" 21:47:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646072" 21:47:52 ERROR [http_cmd]: ERROR 21:47:52 INFO [http_cmd]: http status code OK 21:47:52 DEBUG --- status code --- 21:47:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:47:52 DEBUG --- http response --- 21:47:52 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 21:47:52 ERROR --- status code --- 21:47:52 ERROR http status code is 400 (expected [200, 401, 403, 404]) 21:47:52 ERROR --- http response --- 21:47:52 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 21:47:52 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 21:47:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713646073" 21:47:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646073" 21:47:53 INFO [http_cmd]: http status code OK 21:47:53 DEBUG --- status code --- 21:47:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:47:53 DEBUG --- http response --- 21:47:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:47:53 DEBUG ----- output ----- 21:47:53 DEBUG { 21:47:53 DEBUG "status": "success", 21:47:53 DEBUG "data": { 21:47:53 DEBUG "resultType": "vector", 21:47:53 DEBUG "result": [ 21:47:53 DEBUG { 21:47:53 DEBUG "metric": { 21:47:53 DEBUG "pod": "ds-idrepo-0" 21:47:53 DEBUG }, 21:47:53 DEBUG "value": [ 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG [...] 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG "pod": "ds-cts-2" 21:47:53 DEBUG }, 21:47:53 DEBUG "value": [ 21:47:53 DEBUG 1713646073, 21:47:53 DEBUG "0" 21:47:53 DEBUG ] 21:47:53 DEBUG } 21:47:53 DEBUG ] 21:47:53 DEBUG } 21:47:53 DEBUG } 21:47:53 INFO [http_cmd]: http status code OK 21:47:53 DEBUG --- status code --- 21:47:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:47:53 DEBUG --- http response --- 21:47:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 21:47:53 DEBUG ----- output ----- 21:47:53 DEBUG { 21:47:53 DEBUG "status": "success", 21:47:53 DEBUG "data": { 21:47:53 DEBUG "resultType": "vector", 21:47:53 DEBUG "result": [ 21:47:53 DEBUG { 21:47:53 DEBUG "metric": { 21:47:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:47:53 DEBUG }, 21:47:53 DEBUG "value": [ 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG [...] 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG "pod": "idm-b85f9d978-txpwv" 21:47:53 DEBUG }, 21:47:53 DEBUG "value": [ 21:47:53 DEBUG 1713646073, 21:47:53 DEBUG "37225541.83912183" 21:47:53 DEBUG ] 21:47:53 DEBUG } 21:47:53 DEBUG ] 21:47:53 DEBUG } 21:47:53 DEBUG } 21:47:53 INFO 21:47:53 INFO [loop_until]: kubectl --namespace=xlou top pods 21:47:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:47:53 INFO [loop_until]: OK (rc = 0) 21:47:53 DEBUG --- stdout --- 21:47:53 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 22028m 8764Mi am-767f6b67b5-qcqz2 22095m 8880Mi am-767f6b67b5-vbltz 23507m 8947Mi ds-cts-0 19987m 11999Mi ds-cts-1 20353m 11898Mi ds-cts-2 20874m 12136Mi ds-idrepo-0 9519m 3875Mi ds-idrepo-1 9582m 3975Mi ds-idrepo-2 9888m 3871Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 4m 1239Mi lodemon-7dd48676f6-tdxl9 18m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-967d957f7-zk66v 11670m 868Mi 21:47:53 DEBUG --- stderr --- 21:47:53 DEBUG 21:47:53 INFO 21:47:53 INFO [loop_until]: kubectl --namespace=xlou top node 21:47:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:47:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646073" 21:47:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646073" 21:47:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713646073" 21:47:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646073" 21:47:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646073" 21:47:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646073" 21:47:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646073" 21:47:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713646073" 21:47:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646073" 21:47:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713646073" 21:47:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646073" 21:47:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646073" 21:47:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646073" 21:47:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646073" 21:47:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646073" 21:47:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646073" 21:47:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646073" 21:47:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646073" 21:47:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646073" 21:47:53 INFO [http_cmd]: http status code OK 21:47:53 INFO [loop_until]: OK (rc = 0) 21:47:53 DEBUG --- stdout --- 21:47:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646073" 21:47:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646073" 21:47:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646073" 21:47:53 INFO [http_cmd]: http status code OK 21:47:53 INFO [http_cmd]: http status code OK 21:47:53 INFO [http_cmd]: http status code OK 21:47:53 INFO [http_cmd]: http status code OK 21:47:53 INFO [http_cmd]: http status code OK 21:47:53 INFO [http_cmd]: http status code OK 21:47:53 INFO [http_cmd]: http status code OK 21:47:53 INFO [http_cmd]: http status code OK 21:47:53 INFO [http_cmd]: http status code OK 21:47:53 INFO [http_cmd]: http status code OK 21:47:53 INFO [http_cmd]: http status code OK 21:47:53 INFO [http_cmd]: http status code OK 21:47:53 DEBUG --- status code --- 21:47:53 INFO [http_cmd]: http status code OK 21:47:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646073" 21:47:53 INFO [http_cmd]: http status code OK 21:47:53 INFO [http_cmd]: http status code OK 21:47:53 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6208Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 9986m 33% 7637Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 21730m 72% 11495Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 88m 0% 3385Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 20318m 68% 15220Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 155m 0% 5238Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 22207m 74% 11787Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19044m 63% 3063Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 24788m 83% 11204Mi 10% gke-xlou-cdm-ds-0f601751-j488 10375m 34% 7283Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 20306m 68% 16955Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 10230m 34% 7947Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20709m 69% 14828Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 12046m 75% 8813Mi 15% 21:47:53 INFO [http_cmd]: http status code OK 21:47:53 INFO [http_cmd]: http status code OK 21:47:53 DEBUG --- status code --- 21:47:53 DEBUG --- status code --- 21:47:53 DEBUG --- status code --- 21:47:53 DEBUG --- status code --- 21:47:53 DEBUG --- status code --- 21:47:53 INFO [http_cmd]: http status code OK 21:47:53 DEBUG --- status code --- 21:47:53 DEBUG --- status code --- 21:47:53 DEBUG --- status code --- 21:47:53 DEBUG --- status code --- 21:47:53 DEBUG --- status code --- 21:47:53 DEBUG --- status code --- 21:47:53 DEBUG --- status code --- 21:47:53 INFO [http_cmd]: http status code OK 21:47:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:47:53 DEBUG --- status code --- 21:47:53 DEBUG --- status code --- 21:47:53 DEBUG --- status code --- 21:47:53 DEBUG --- stderr --- 21:47:53 DEBUG --- status code --- 21:47:53 DEBUG --- status code --- 21:47:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:47:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:47:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:47:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:47:53 INFO [http_cmd]: http status code OK 21:47:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:47:53 INFO [http_cmd]: http status code OK 21:47:53 DEBUG --- status code --- 21:47:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:47:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:47:53 DEBUG --- http response --- 21:47:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:47:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:47:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:47:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:47:53 DEBUG --- status code --- 21:47:53 DEBUG --- http response --- 21:47:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:47:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:47:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:47:53 DEBUG 21:47:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:47:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:47:53 DEBUG --- http response --- 21:47:53 DEBUG --- http response --- 21:47:53 DEBUG --- http response --- 21:47:53 DEBUG --- http response --- 21:47:53 DEBUG --- status code --- 21:47:53 DEBUG --- http response --- 21:47:53 DEBUG --- status code --- 21:47:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:47:53 DEBUG --- http response --- 21:47:53 INFO [http_cmd]: http status code OK 21:47:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:47:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:47:53 DEBUG --- http response --- 21:47:53 DEBUG --- http response --- 21:47:53 DEBUG --- http response --- 21:47:53 DEBUG --- http response --- 21:47:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:47:53 DEBUG --- http response --- 21:47:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:47:53 DEBUG --- http response --- 21:47:53 DEBUG --- http response --- 21:47:53 DEBUG --- http response --- 21:47:53 DEBUG --- http response --- 21:47:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:47:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:47:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:47:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:47:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:47:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:47:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:47:53 DEBUG --- http response --- 21:47:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:47:53 DEBUG --- status code --- 21:47:53 DEBUG --- http response --- 21:47:53 DEBUG ----- output ----- 21:47:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:47:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:47:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:47:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:47:53 DEBUG --- http response --- 21:47:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:47:53 DEBUG ----- output ----- 21:47:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:47:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:47:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:47:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:47:53 DEBUG ----- output ----- 21:47:53 DEBUG ----- output ----- 21:47:53 DEBUG ----- output ----- 21:47:53 DEBUG ----- output ----- 21:47:53 DEBUG --- http response --- 21:47:53 DEBUG ----- output ----- 21:47:53 DEBUG --- http response --- 21:47:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:47:53 DEBUG ----- output ----- 21:47:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:47:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:47:53 DEBUG { 21:47:53 DEBUG ----- output ----- 21:47:53 DEBUG ----- output ----- 21:47:53 DEBUG ----- output ----- 21:47:53 DEBUG ----- output ----- 21:47:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 21:47:53 DEBUG ----- output ----- 21:47:53 DEBUG { 21:47:53 DEBUG ----- output ----- 21:47:53 DEBUG ----- output ----- 21:47:53 DEBUG ----- output ----- 21:47:53 DEBUG ----- output ----- 21:47:53 DEBUG { 21:47:53 DEBUG { 21:47:53 DEBUG { 21:47:53 DEBUG { 21:47:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:47:53 DEBUG { 21:47:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:47:53 DEBUG ----- output ----- 21:47:53 DEBUG { 21:47:53 DEBUG --- http response --- 21:47:53 DEBUG ----- output ----- 21:47:53 DEBUG "status": "success", 21:47:53 DEBUG { 21:47:53 DEBUG { 21:47:53 DEBUG { 21:47:53 DEBUG { 21:47:53 DEBUG ----- output ----- 21:47:53 DEBUG { 21:47:53 DEBUG "status": "success", 21:47:53 DEBUG { 21:47:53 DEBUG { 21:47:53 DEBUG { 21:47:53 DEBUG { 21:47:53 DEBUG "status": "success", 21:47:53 DEBUG "status": "success", 21:47:53 DEBUG "status": "success", 21:47:53 DEBUG "status": "success", 21:47:53 DEBUG ----- output ----- 21:47:53 DEBUG "status": "success", 21:47:53 DEBUG ----- output ----- 21:47:53 DEBUG { 21:47:53 DEBUG "status": "success", 21:47:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:47:53 DEBUG { 21:47:53 DEBUG "data": { 21:47:53 DEBUG "status": "success", 21:47:53 DEBUG "status": "success", 21:47:53 DEBUG "status": "success", 21:47:53 DEBUG "status": "success", 21:47:53 DEBUG { 21:47:53 DEBUG "status": "success", 21:47:53 DEBUG "data": { 21:47:53 DEBUG "status": "success", 21:47:53 DEBUG "data": { 21:47:53 DEBUG "status": "success", 21:47:53 DEBUG "status": "success", 21:47:53 DEBUG "data": { 21:47:53 DEBUG "data": { 21:47:53 DEBUG "data": { 21:47:53 DEBUG "data": { 21:47:53 DEBUG { 21:47:53 DEBUG "data": { 21:47:53 DEBUG { 21:47:53 DEBUG "status": "success", 21:47:53 DEBUG "data": { 21:47:53 DEBUG ----- output ----- 21:47:53 DEBUG "status": "success", 21:47:53 DEBUG "resultType": "vector", 21:47:53 DEBUG "data": { 21:47:53 DEBUG "data": { 21:47:53 DEBUG "data": { 21:47:53 DEBUG "data": { 21:47:53 DEBUG "status": "success", 21:47:53 DEBUG "data": { 21:47:53 DEBUG "resultType": "vector", 21:47:53 DEBUG "resultType": "vector", 21:47:53 DEBUG "status": "success", 21:47:53 DEBUG "data": { 21:47:53 DEBUG "data": { 21:47:53 DEBUG "resultType": "vector", 21:47:53 DEBUG "resultType": "vector", 21:47:53 DEBUG "resultType": "vector", 21:47:53 DEBUG "resultType": "vector", 21:47:53 DEBUG "status": "success", 21:47:53 DEBUG "resultType": "vector", 21:47:53 DEBUG "status": "success", 21:47:53 DEBUG "data": { 21:47:53 DEBUG "resultType": "vector", 21:47:53 DEBUG { 21:47:53 DEBUG "data": { 21:47:53 DEBUG "result": [ 21:47:53 DEBUG "resultType": "vector", 21:47:53 DEBUG "resultType": "vector", 21:47:53 DEBUG "resultType": "vector", 21:47:53 DEBUG "resultType": "vector", 21:47:53 DEBUG "data": { 21:47:53 DEBUG "resultType": "vector", 21:47:53 DEBUG "result": [ 21:47:53 DEBUG "result": [ 21:47:53 DEBUG "data": { 21:47:53 DEBUG "resultType": "vector", 21:47:53 DEBUG "resultType": "vector", 21:47:53 DEBUG "result": [ 21:47:53 DEBUG "result": [ 21:47:53 DEBUG "result": [ 21:47:53 DEBUG "result": [ 21:47:53 DEBUG "data": { 21:47:53 DEBUG "result": [ 21:47:53 DEBUG "data": { 21:47:53 DEBUG "resultType": "vector", 21:47:53 DEBUG "result": [ 21:47:53 DEBUG "status": "success", 21:47:53 DEBUG "resultType": "vector", 21:47:53 DEBUG { 21:47:53 DEBUG "result": [ 21:47:53 DEBUG "result": [ 21:47:53 DEBUG "result": [ 21:47:53 DEBUG "result": [ 21:47:53 DEBUG "resultType": "vector", 21:47:53 DEBUG "result": [ 21:47:53 DEBUG { 21:47:53 DEBUG { 21:47:53 DEBUG "resultType": "vector", 21:47:53 DEBUG "result": [ 21:47:53 DEBUG "result": [ 21:47:53 DEBUG { 21:47:53 DEBUG { 21:47:53 DEBUG { 21:47:53 DEBUG { 21:47:53 DEBUG "resultType": "vector", 21:47:53 DEBUG { 21:47:53 DEBUG "resultType": "vector", 21:47:53 DEBUG "result": [ 21:47:53 DEBUG { 21:47:53 DEBUG "data": { 21:47:53 DEBUG "result": [ 21:47:53 DEBUG "metric": { 21:47:53 DEBUG { 21:47:53 DEBUG { 21:47:53 DEBUG { 21:47:53 DEBUG { 21:47:53 DEBUG "result": [ 21:47:53 DEBUG { 21:47:53 DEBUG "metric": { 21:47:53 DEBUG "metric": { 21:47:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:47:53 DEBUG }, 21:47:53 DEBUG { 21:47:53 DEBUG "metric": { 21:47:53 DEBUG "metric": { 21:47:53 DEBUG "metric": { 21:47:53 DEBUG "metric": { 21:47:53 DEBUG "result": [ 21:47:53 DEBUG "metric": { 21:47:53 DEBUG "result": [ 21:47:53 DEBUG { 21:47:53 DEBUG "metric": { 21:47:53 DEBUG "resultType": "vector", 21:47:53 DEBUG { 21:47:53 DEBUG "pod": "am-767f6b67b5-9zw22" 21:47:53 DEBUG "metric": { 21:47:53 DEBUG "metric": { 21:47:53 DEBUG "metric": { 21:47:53 DEBUG "metric": { 21:47:53 DEBUG { 21:47:53 DEBUG "metric": { 21:47:53 DEBUG "instance": "10.142.0.107:9100" 21:47:53 DEBUG { 21:47:53 DEBUG "value": [ 21:47:53 DEBUG 21:47:53 DEBUG "metric": { 21:47:53 DEBUG "instance": "10.142.0.107:9100" 21:47:53 DEBUG "pod": "am-767f6b67b5-9zw22" 21:47:53 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 21:47:53 DEBUG "pod": "ds-cts-0" 21:47:53 DEBUG { 21:47:53 DEBUG "pod": "am-767f6b67b5-9zw22" 21:47:53 DEBUG { 21:47:53 DEBUG "metric": { 21:47:53 DEBUG "instance": "10.142.0.107:9100" 21:47:53 DEBUG "result": [ 21:47:53 DEBUG "metric": { 21:47:53 DEBUG }, 21:47:53 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 21:47:53 DEBUG }, 21:47:53 DEBUG "pod": "ds-cts-0" 21:47:53 DEBUG }, 21:47:53 DEBUG "metric": { 21:47:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:47:53 DEBUG }, 21:47:53 DEBUG "metric": { 21:47:53 DEBUG 21:47:53 DEBUG "pod": "ds-idrepo-0" 21:47:53 DEBUG }, 21:47:53 DEBUG }, 21:47:53 DEBUG "result": [ 21:47:53 DEBUG }, 21:47:53 DEBUG }, 21:47:53 DEBUG "metric": { 21:47:53 DEBUG }, 21:47:53 DEBUG "metric": { 21:47:53 DEBUG "pod": "ds-idrepo-0" 21:47:53 DEBUG }, 21:47:53 DEBUG { 21:47:53 DEBUG "pod": "am-767f6b67b5-9zw22" 21:47:53 DEBUG "value": [ 21:47:53 DEBUG "pod": "am-767f6b67b5-9zw22" 21:47:53 DEBUG "value": [ 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:47:53 DEBUG }, 21:47:53 DEBUG "value": [ 21:47:53 DEBUG "pod": "am-767f6b67b5-9zw22" 21:47:53 DEBUG 21:47:53 DEBUG }, 21:47:53 DEBUG "value": [ 21:47:53 DEBUG "value": [ 21:47:53 DEBUG { 21:47:53 DEBUG "value": [ 21:47:53 DEBUG "value": [ 21:47:53 DEBUG "pod": "ds-idrepo-0" 21:47:53 DEBUG "value": [ 21:47:53 DEBUG "instance": "10.142.0.107:9100" 21:47:53 DEBUG }, 21:47:53 DEBUG "value": [ 21:47:53 DEBUG "metric": { 21:47:53 DEBUG }, 21:47:53 DEBUG 21:47:53 DEBUG }, 21:47:53 DEBUG 21:47:53 DEBUG [...] 21:47:53 DEBUG }, 21:47:53 DEBUG "value": [ 21:47:53 DEBUG "value": [ 21:47:53 DEBUG 21:47:53 DEBUG }, 21:47:53 DEBUG [...] 21:47:53 DEBUG "value": [ 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG "metric": { 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG }, 21:47:53 DEBUG 21:47:53 DEBUG }, 21:47:53 DEBUG "value": [ 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG "value": [ 21:47:53 DEBUG 21:47:53 DEBUG "value": [ 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG "value": [ 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG "instance": "10.142.0.107:9100" 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG "value": [ 21:47:53 DEBUG 21:47:53 DEBUG "value": [ 21:47:53 DEBUG 21:47:53 DEBUG "pod": "ds-cts-0" 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 21:47:53 DEBUG "pod": "am-767f6b67b5-9zw22" 21:47:53 DEBUG "value": [ 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG }, 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG }, 21:47:53 DEBUG "value": [ 21:47:53 DEBUG 21:47:53 DEBUG [...] 21:47:53 DEBUG 21:47:53 DEBUG }, 21:47:53 DEBUG }, 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG [...] 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG [...] 21:47:53 DEBUG [...] 21:47:53 DEBUG 21:47:53 DEBUG "value": [ 21:47:53 DEBUG [...] 21:47:53 DEBUG [...] 21:47:53 DEBUG 21:47:53 DEBUG [...] 21:47:53 DEBUG 21:47:53 DEBUG [...] 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG [...] 21:47:53 DEBUG 21:47:53 DEBUG "value": [ 21:47:53 DEBUG "value": [ 21:47:53 DEBUG 21:47:53 DEBUG [...] 21:47:53 DEBUG [...] 21:47:53 DEBUG 21:47:53 DEBUG [...] 21:47:53 DEBUG [...] 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG [...] 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG [...] 21:47:53 DEBUG 1713646073, 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG "pod": "idm-b85f9d978-txpwv" 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG [...] 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG [...] 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG "1006628864" 21:47:53 DEBUG ] 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG "instance": "10.142.0.43:9100" 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG [...] 21:47:53 DEBUG 21:47:53 DEBUG }, 21:47:53 DEBUG 21:47:53 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG "pod": "ds-idrepo-2" 21:47:53 DEBUG [...] 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG "pod": "am-767f6b67b5-vbltz" 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG [...] 21:47:53 DEBUG } 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG "pod": "idm-b85f9d978-txpwv" 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG "pod": "am-767f6b67b5-vbltz" 21:47:53 DEBUG "value": [ 21:47:53 DEBUG [...] 21:47:53 DEBUG }, 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG }, 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG }, 21:47:53 DEBUG "instance": "10.142.0.43:9100" 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG ] 21:47:53 DEBUG 21:47:53 DEBUG }, 21:47:53 DEBUG }, 21:47:53 DEBUG 21:47:53 DEBUG "pod": "am-767f6b67b5-vbltz" 21:47:53 DEBUG 21:47:53 DEBUG "instance": "10.142.0.43:9100" 21:47:53 DEBUG }, 21:47:53 DEBUG 1713646073, 21:47:53 DEBUG 21:47:53 DEBUG "value": [ 21:47:53 DEBUG "pod": "ds-cts-2" 21:47:53 DEBUG 21:47:53 DEBUG "pod": "am-767f6b67b5-vbltz" 21:47:53 DEBUG "value": [ 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG "value": [ 21:47:53 DEBUG }, 21:47:53 DEBUG "value": [ 21:47:53 DEBUG 21:47:53 DEBUG 21:47:53 DEBUG "pod": "am-767f6b67b5-vbltz" 21:47:53 DEBUG "pod": "ds-cts-2" 21:47:53 DEBUG "value": [ 21:47:53 DEBUG 1713646073, 21:47:53 DEBUG }, 21:47:53 DEBUG 21:47:53 DEBUG "pod": "idm-b85f9d978-txpwv" 21:47:53 DEBUG }, 21:47:53 DEBUG "491.39223801811534" 21:47:53 DEBUG 21:47:53 DEBUG 1713646073, 21:47:53 DEBUG }, 21:47:53 DEBUG "pod": "ds-idrepo-2" 21:47:53 DEBUG }, 21:47:53 DEBUG 1713646073, 21:47:53 DEBUG 21:47:53 DEBUG "pod": "am-767f6b67b5-vbltz" 21:47:53 DEBUG "pod": "ds-cts-2" 21:47:53 DEBUG 1713646073, 21:47:53 DEBUG 1713646073, 21:47:53 DEBUG "value": [ 21:47:53 DEBUG }, 21:47:53 DEBUG } 21:47:53 DEBUG "pod": "am-767f6b67b5-vbltz" 21:47:53 DEBUG }, 21:47:53 DEBUG "pod": "ds-idrepo-2" 21:47:53 DEBUG "100134209.4" 21:47:53 DEBUG "value": [ 21:47:53 DEBUG }, 21:47:53 DEBUG "value": [ 21:47:53 DEBUG "value": [ 21:47:53 DEBUG ] 21:47:53 DEBUG 21:47:53 DEBUG "11.779649107788373" 21:47:53 DEBUG "value": [ 21:47:53 DEBUG }, 21:47:53 DEBUG "value": [ 21:47:53 DEBUG "0" 21:47:53 DEBUG "instance": "10.142.0.43:9100" 21:47:53 DEBUG }, 21:47:53 DEBUG }, 21:47:53 DEBUG "3337.6" 21:47:53 DEBUG "0.059333333333326736" 21:47:53 DEBUG 1713646073, 21:47:53 DEBUG "value": [ 21:47:53 DEBUG } 21:47:53 DEBUG }, 21:47:53 DEBUG "value": [ 21:47:53 DEBUG }, 21:47:53 DEBUG ] 21:47:53 DEBUG } 21:47:53 DEBUG "value": [ 21:47:53 DEBUG 1713646073, 21:47:53 DEBUG 1713646073, 21:47:53 DEBUG } 21:47:53 DEBUG "instance": "10.142.0.43:9100" 21:47:53 DEBUG ] 21:47:53 DEBUG 1713646073, 21:47:53 DEBUG "value": [ 21:47:53 DEBUG 1713646073, 21:47:53 DEBUG ] 21:47:53 DEBUG }, 21:47:53 DEBUG "value": [ 21:47:53 DEBUG "value": [ 21:47:53 DEBUG ] 21:47:53 DEBUG ] 21:47:53 DEBUG "0" 21:47:53 DEBUG ] 21:47:53 DEBUG "value": [ 21:47:53 DEBUG 1713646073, 21:47:53 DEBUG "value": [ 21:47:53 DEBUG 1713646073, 21:47:53 DEBUG ] 21:47:53 DEBUG 1713646073, 21:47:53 DEBUG "0.0031262071861355998" 21:47:53 DEBUG "3967289736192" 21:47:53 DEBUG ] 21:47:53 DEBUG }, 21:47:53 DEBUG } 21:47:53 DEBUG "0.39779999999999993" 21:47:53 DEBUG 1713646073, 21:47:53 DEBUG "49.377" 21:47:53 DEBUG } 21:47:53 DEBUG "value": [ 21:47:53 DEBUG 1713646073, 21:47:53 DEBUG 1713646073, 21:47:53 DEBUG } 21:47:53 DEBUG } 21:47:53 DEBUG } 21:47:53 DEBUG 1713646073, 21:47:53 DEBUG 1713646073, 21:47:53 DEBUG "598998.5" 21:47:53 DEBUG 1713646073, 21:47:53 DEBUG "0" 21:47:53 DEBUG } 21:47:53 DEBUG "3337.2999999999997" 21:47:53 DEBUG ] 21:47:53 DEBUG ] 21:47:53 DEBUG } 21:47:53 DEBUG "value": [ 21:47:53 DEBUG ] 21:47:53 DEBUG ] 21:47:53 DEBUG "0" 21:47:53 DEBUG ] 21:47:53 DEBUG } 21:47:53 DEBUG 1713646073, 21:47:53 DEBUG "3334.2" 21:47:53 DEBUG "0" 21:47:53 DEBUG ] 21:47:53 DEBUG ] 21:47:53 DEBUG ] 21:47:53 DEBUG "0" 21:47:53 DEBUG ] 21:47:53 DEBUG "0" 21:47:53 DEBUG ] 21:47:53 DEBUG "3320" 21:47:53 DEBUG ] 21:47:53 DEBUG ] 21:47:53 DEBUG } 21:47:53 DEBUG } 21:47:53 DEBUG } 21:47:53 DEBUG 1713646073, 21:47:53 DEBUG } 21:47:53 DEBUG } 21:47:53 DEBUG ] 21:47:53 DEBUG ] 21:47:53 DEBUG ] 21:47:53 DEBUG } 21:47:53 DEBUG } 21:47:53 DEBUG ] 21:47:53 DEBUG } 21:47:53 DEBUG } 21:47:53 DEBUG } 21:47:53 DEBUG ] 21:47:53 DEBUG } 21:47:53 DEBUG ] 21:47:53 DEBUG } 21:47:53 DEBUG } 21:47:53 DEBUG } 21:47:53 DEBUG } 21:47:53 DEBUG ] 21:47:53 DEBUG ] 21:47:53 DEBUG "0.19886666666661007" 21:47:53 DEBUG } 21:47:53 DEBUG ] 21:47:53 DEBUG } 21:47:53 DEBUG } 21:47:53 DEBUG ] 21:47:53 DEBUG } 21:47:53 DEBUG } 21:47:53 DEBUG "100130342.9" 21:47:53 DEBUG } 21:47:53 DEBUG } 21:47:53 DEBUG } 21:47:53 DEBUG ] 21:47:53 DEBUG } 21:47:53 DEBUG ] 21:47:53 DEBUG ] 21:47:53 DEBUG } 21:47:53 DEBUG } 21:47:53 DEBUG } 21:47:53 DEBUG ] 21:47:53 DEBUG } 21:47:53 DEBUG ] 21:47:53 DEBUG } 21:47:53 DEBUG } 21:47:53 DEBUG ] 21:47:53 DEBUG ] 21:47:53 DEBUG ] 21:47:53 DEBUG } 21:47:53 DEBUG } 21:47:53 DEBUG } 21:47:53 DEBUG ] 21:47:53 DEBUG } 21:47:53 DEBUG } 21:47:53 DEBUG } 21:47:53 DEBUG } 21:47:53 DEBUG } 21:47:53 DEBUG } 21:47:53 DEBUG } 21:47:53 DEBUG ] 21:47:53 DEBUG } 21:47:53 DEBUG } 21:47:53 DEBUG ] 21:47:53 DEBUG } 21:47:53 DEBUG } 21:47:53 DEBUG ] 21:47:53 DEBUG } 21:47:53 DEBUG } 21:47:53 DEBUG ] 21:47:53 DEBUG } 21:47:53 DEBUG } 21:47:53 DEBUG } 21:47:53 DEBUG } 21:47:53 DEBUG } 21:47:53 DEBUG } 21:47:53 DEBUG } 21:47:53 DEBUG } 21:47:53 DEBUG } 21:47:53 DEBUG } 21:48:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713646132" 21:48:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646132" 21:48:52 INFO [http_cmd]: http status code OK 21:48:52 ERROR [http_cmd]: ERROR 21:48:52 DEBUG --- status code --- 21:48:52 ERROR --- status code --- 21:48:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:48:52 ERROR http status code is 400 (expected [200, 401, 403, 404]) 21:48:52 DEBUG --- http response --- 21:48:52 ERROR --- http response --- 21:48:52 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 21:48:52 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 21:48:52 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 21:48:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713646133" 21:48:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646133" 21:48:53 INFO [http_cmd]: http status code OK 21:48:53 DEBUG --- status code --- 21:48:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:48:53 DEBUG --- http response --- 21:48:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:48:53 DEBUG ----- output ----- 21:48:53 DEBUG { 21:48:53 DEBUG "status": "success", 21:48:53 DEBUG "data": { 21:48:53 DEBUG "resultType": "vector", 21:48:53 DEBUG "result": [ 21:48:53 DEBUG { 21:48:53 DEBUG "metric": { 21:48:53 DEBUG "pod": "ds-idrepo-0" 21:48:53 DEBUG }, 21:48:53 DEBUG "value": [ 21:48:53 DEBUG 21:48:53 DEBUG 21:48:53 DEBUG 21:48:53 DEBUG [...] 21:48:53 DEBUG 21:48:53 DEBUG 21:48:53 DEBUG 21:48:53 DEBUG "pod": "ds-cts-2" 21:48:53 DEBUG }, 21:48:53 DEBUG "value": [ 21:48:53 DEBUG 1713646133, 21:48:53 DEBUG "0" 21:48:53 DEBUG ] 21:48:53 DEBUG } 21:48:53 DEBUG ] 21:48:53 DEBUG } 21:48:53 DEBUG } 21:48:53 INFO [http_cmd]: http status code OK 21:48:53 DEBUG --- status code --- 21:48:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:48:53 DEBUG --- http response --- 21:48:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:48:53 DEBUG ----- output ----- 21:48:53 DEBUG { 21:48:53 DEBUG "status": "success", 21:48:53 DEBUG "data": { 21:48:53 DEBUG "resultType": "vector", 21:48:53 DEBUG "result": [ 21:48:53 DEBUG { 21:48:53 DEBUG "metric": { 21:48:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:48:53 DEBUG }, 21:48:53 DEBUG "value": [ 21:48:53 DEBUG 21:48:53 DEBUG 21:48:53 DEBUG 21:48:53 DEBUG [...] 21:48:53 DEBUG 21:48:53 DEBUG 21:48:53 DEBUG 21:48:53 DEBUG "pod": "overseer-0-967d957f7-zk66v" 21:48:53 DEBUG }, 21:48:53 DEBUG "value": [ 21:48:53 DEBUG 1713646133, 21:48:53 DEBUG "72867.82726764065" 21:48:53 DEBUG ] 21:48:53 DEBUG } 21:48:53 DEBUG ] 21:48:53 DEBUG } 21:48:53 DEBUG } 21:48:53 INFO 21:48:53 INFO [loop_until]: kubectl --namespace=xlou top pods 21:48:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:48:53 INFO [loop_until]: OK (rc = 0) 21:48:53 DEBUG --- stdout --- 21:48:53 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 22010m 8762Mi am-767f6b67b5-qcqz2 21949m 8880Mi am-767f6b67b5-vbltz 24094m 8938Mi ds-cts-0 18974m 12022Mi ds-cts-1 20005m 11902Mi ds-cts-2 20439m 12128Mi ds-idrepo-0 9278m 3893Mi ds-idrepo-1 9707m 3974Mi ds-idrepo-2 9569m 3874Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 4m 1239Mi lodemon-7dd48676f6-tdxl9 19m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-967d957f7-zk66v 11777m 871Mi 21:48:53 DEBUG --- stderr --- 21:48:53 DEBUG 21:48:53 INFO 21:48:53 INFO [loop_until]: kubectl --namespace=xlou top node 21:48:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:48:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713646133" 21:48:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646133" 21:48:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646133" 21:48:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646133" 21:48:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713646133" 21:48:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646133" 21:48:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646133" 21:48:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646133" 21:48:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646133" 21:48:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713646133" 21:48:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646133" 21:48:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646133" 21:48:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646133" 21:48:53 INFO [loop_until]: OK (rc = 0) 21:48:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646133" 21:48:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646133" 21:48:53 INFO [http_cmd]: http status code OK 21:48:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646133" 21:48:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646133" 21:48:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646133" 21:48:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646133" 21:48:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646133" 21:48:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646133" 21:48:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646133" 21:48:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646133" 21:48:53 INFO [http_cmd]: http status code OK 21:48:53 INFO [http_cmd]: http status code OK 21:48:53 INFO [http_cmd]: http status code OK 21:48:53 INFO [http_cmd]: http status code OK 21:48:53 INFO [http_cmd]: http status code OK 21:48:53 INFO [http_cmd]: http status code OK 21:48:53 DEBUG --- stdout --- 21:48:53 INFO [http_cmd]: http status code OK 21:48:53 INFO [http_cmd]: http status code OK 21:48:53 INFO [http_cmd]: http status code OK 21:48:53 INFO [http_cmd]: http status code OK 21:48:53 DEBUG --- status code --- 21:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:48:54 DEBUG --- http response --- 21:48:53 INFO [http_cmd]: http status code OK 21:48:53 INFO [http_cmd]: http status code OK 21:48:53 INFO [http_cmd]: http status code OK 21:48:54 INFO [http_cmd]: http status code OK 21:48:54 INFO [http_cmd]: http status code OK 21:48:54 DEBUG --- status code --- 21:48:54 DEBUG --- status code --- 21:48:54 INFO [http_cmd]: http status code OK 21:48:54 INFO [http_cmd]: http status code OK 21:48:54 DEBUG --- status code --- 21:48:54 INFO [http_cmd]: http status code OK 21:48:54 DEBUG --- status code --- 21:48:54 INFO [http_cmd]: http status code OK 21:48:54 DEBUG --- status code --- 21:48:54 DEBUG --- status code --- 21:48:54 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6205Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 9962m 33% 7649Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 21045m 70% 11487Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 84m 0% 3379Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 19812m 66% 15250Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 158m 0% 5221Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 22194m 74% 11774Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19593m 65% 3055Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 24634m 82% 11193Mi 10% gke-xlou-cdm-ds-0f601751-j488 10393m 34% 7287Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19868m 66% 16972Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 10174m 34% 7951Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20278m 67% 14829Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 12030m 75% 8815Mi 15% 21:48:54 DEBUG --- status code --- 21:48:54 DEBUG --- status code --- 21:48:54 DEBUG --- status code --- 21:48:54 DEBUG --- status code --- 21:48:53 INFO [http_cmd]: http status code OK 21:48:53 INFO [http_cmd]: http status code OK 21:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:48:54 DEBUG --- status code --- 21:48:54 DEBUG --- status code --- 21:48:54 DEBUG --- status code --- 21:48:54 DEBUG --- status code --- 21:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:48:54 DEBUG --- status code --- 21:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:48:54 INFO [http_cmd]: http status code OK 21:48:54 DEBUG --- status code --- 21:48:54 DEBUG --- status code --- 21:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:48:54 DEBUG --- status code --- 21:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:48:54 DEBUG --- status code --- 21:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:48:54 DEBUG --- stderr --- 21:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:48:54 DEBUG --- http response --- 21:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:48:54 DEBUG --- status code --- 21:48:54 DEBUG --- status code --- 21:48:54 DEBUG ----- output ----- 21:48:54 DEBUG { 21:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:48:54 DEBUG --- http response --- 21:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:48:54 DEBUG --- http response --- 21:48:54 DEBUG --- status code --- 21:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:48:54 DEBUG --- http response --- 21:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:48:54 DEBUG --- http response --- 21:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:48:54 DEBUG --- http response --- 21:48:54 DEBUG --- http response --- 21:48:54 DEBUG 21:48:54 DEBUG --- http response --- 21:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:48:54 DEBUG --- http response --- 21:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:48:54 DEBUG "status": "success", 21:48:54 DEBUG --- http response --- 21:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:48:54 DEBUG --- http response --- 21:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:48:54 DEBUG --- http response --- 21:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:48:54 DEBUG --- http response --- 21:48:54 DEBUG --- http response --- 21:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:48:54 DEBUG --- http response --- 21:48:54 DEBUG --- http response --- 21:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:48:54 DEBUG ----- output ----- 21:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:48:54 DEBUG --- http response --- 21:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:48:54 DEBUG ----- output ----- 21:48:54 DEBUG --- http response --- 21:48:54 DEBUG --- http response --- 21:48:54 DEBUG "data": { 21:48:54 DEBUG ----- output ----- 21:48:54 DEBUG --- http response --- 21:48:54 DEBUG --- http response --- 21:48:54 DEBUG ----- output ----- 21:48:54 DEBUG { 21:48:54 DEBUG "status": "success", 21:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:48:54 DEBUG --- http response --- 21:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:48:54 DEBUG ----- output ----- 21:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:48:54 DEBUG ----- output ----- 21:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:48:54 DEBUG { 21:48:54 DEBUG "status": "success", 21:48:54 DEBUG "data": { 21:48:54 DEBUG "resultType": "vector", 21:48:54 DEBUG ----- output ----- 21:48:54 DEBUG { 21:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:48:54 DEBUG "resultType": "vector", 21:48:54 DEBUG { 21:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:48:54 DEBUG "data": { 21:48:54 DEBUG ----- output ----- 21:48:54 DEBUG ----- output ----- 21:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:48:54 DEBUG ----- output ----- 21:48:54 DEBUG { 21:48:54 DEBUG "status": "success", 21:48:54 DEBUG ----- output ----- 21:48:54 DEBUG ----- output ----- 21:48:54 DEBUG ----- output ----- 21:48:54 DEBUG "result": [ 21:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:48:54 DEBUG ----- output ----- 21:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:48:54 DEBUG "status": "success", 21:48:54 DEBUG ----- output ----- 21:48:54 DEBUG ----- output ----- 21:48:54 DEBUG { 21:48:54 DEBUG "result": [ 21:48:54 DEBUG "status": "success", 21:48:54 DEBUG "data": { 21:48:54 DEBUG ----- output ----- 21:48:54 DEBUG ----- output ----- 21:48:54 DEBUG "resultType": "vector", 21:48:54 DEBUG { 21:48:54 DEBUG { 21:48:54 DEBUG ----- output ----- 21:48:54 DEBUG { 21:48:54 DEBUG { 21:48:54 DEBUG "data": { 21:48:54 DEBUG { 21:48:54 DEBUG { 21:48:54 DEBUG { 21:48:54 DEBUG { 21:48:54 DEBUG ----- output ----- 21:48:54 DEBUG { 21:48:54 DEBUG ----- output ----- 21:48:54 DEBUG "data": { 21:48:54 DEBUG "resultType": "vector", 21:48:54 DEBUG "result": [ 21:48:54 DEBUG "status": "success", 21:48:54 DEBUG { 21:48:54 DEBUG "resultType": "vector", 21:48:54 DEBUG ----- output ----- 21:48:54 DEBUG { 21:48:54 DEBUG { 21:48:54 DEBUG "result": [ 21:48:54 DEBUG "status": "success", 21:48:54 DEBUG "status": "success", 21:48:54 DEBUG { 21:48:54 DEBUG "status": "success", 21:48:54 DEBUG "data": { 21:48:54 DEBUG "resultType": "vector", 21:48:54 DEBUG "status": "success", 21:48:54 DEBUG "status": "success", 21:48:54 DEBUG "status": "success", 21:48:54 DEBUG "metric": { 21:48:54 DEBUG { 21:48:54 DEBUG "status": "success", 21:48:54 DEBUG "data": { 21:48:54 DEBUG { 21:48:54 DEBUG { 21:48:54 DEBUG { 21:48:54 DEBUG "data": { 21:48:54 DEBUG "metric": { 21:48:54 DEBUG "result": [ 21:48:54 DEBUG { 21:48:54 DEBUG "status": "success", 21:48:54 DEBUG "status": "success", 21:48:54 DEBUG { 21:48:54 DEBUG "data": { 21:48:54 DEBUG "data": { 21:48:54 DEBUG "status": "success", 21:48:54 DEBUG "result": [ 21:48:54 DEBUG "status": "success", 21:48:54 DEBUG "data": { 21:48:54 DEBUG "data": { 21:48:54 DEBUG "data": { 21:48:54 DEBUG "pod": "am-767f6b67b5-9zw22" 21:48:54 DEBUG }, 21:48:54 DEBUG "status": "success", 21:48:54 DEBUG { 21:48:54 DEBUG "resultType": "vector", 21:48:54 DEBUG "status": "success", 21:48:54 DEBUG "metric": { 21:48:54 DEBUG "status": "success", 21:48:54 DEBUG "resultType": "vector", 21:48:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:48:54 DEBUG { 21:48:54 DEBUG "metric": { 21:48:54 DEBUG "data": { 21:48:54 DEBUG "resultType": "vector", 21:48:54 DEBUG "metric": { 21:48:54 DEBUG "resultType": "vector", 21:48:54 DEBUG "resultType": "vector", 21:48:54 DEBUG "data": { 21:48:54 DEBUG { 21:48:54 DEBUG "data": { 21:48:54 DEBUG "resultType": "vector", 21:48:54 DEBUG "resultType": "vector", 21:48:54 DEBUG "result": [ 21:48:54 DEBUG "value": [ 21:48:54 DEBUG "resultType": "vector", 21:48:54 DEBUG "data": { 21:48:54 DEBUG "resultType": "vector", 21:48:54 DEBUG "result": [ 21:48:54 DEBUG "data": { 21:48:54 DEBUG "pod": "am-767f6b67b5-9zw22" 21:48:54 DEBUG "data": { 21:48:54 DEBUG "result": [ 21:48:54 DEBUG }, 21:48:54 DEBUG "pod": "ds-cts-0" 21:48:54 DEBUG "result": [ 21:48:54 DEBUG "data": { 21:48:54 DEBUG "pod": "ds-cts-0" 21:48:54 DEBUG "status": "success", 21:48:54 DEBUG "result": [ 21:48:54 DEBUG "result": [ 21:48:54 DEBUG "resultType": "vector", 21:48:54 DEBUG "metric": { 21:48:54 DEBUG "instance": "10.142.0.107:9100" 21:48:54 DEBUG "result": [ 21:48:54 DEBUG { 21:48:54 DEBUG 21:48:54 DEBUG "result": [ 21:48:54 DEBUG "resultType": "vector", 21:48:54 DEBUG "result": [ 21:48:54 DEBUG { 21:48:54 DEBUG { 21:48:54 DEBUG "resultType": "vector", 21:48:54 DEBUG }, 21:48:54 DEBUG "value": [ 21:48:54 DEBUG { 21:48:54 DEBUG "value": [ 21:48:54 DEBUG 21:48:54 DEBUG { 21:48:54 DEBUG "metric": { 21:48:54 DEBUG }, 21:48:54 DEBUG "data": { 21:48:54 DEBUG { 21:48:54 DEBUG { 21:48:54 DEBUG "result": [ 21:48:54 DEBUG "resultType": "vector", 21:48:54 DEBUG }, 21:48:54 DEBUG "value": [ 21:48:54 DEBUG "metric": { 21:48:54 DEBUG 21:48:54 DEBUG { 21:48:54 DEBUG "result": [ 21:48:54 DEBUG "metric": { 21:48:54 DEBUG "status": "success", 21:48:54 DEBUG "metric": { 21:48:54 DEBUG "result": [ 21:48:54 DEBUG 21:48:54 DEBUG "resultType": "vector", 21:48:54 DEBUG "metric": { 21:48:54 DEBUG }, 21:48:54 DEBUG 21:48:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 21:48:54 DEBUG "resultType": "vector", 21:48:54 DEBUG "value": [ 21:48:54 DEBUG "resultType": "vector", 21:48:54 DEBUG "metric": { 21:48:54 DEBUG "metric": { 21:48:54 DEBUG { 21:48:54 DEBUG "result": [ 21:48:54 DEBUG 21:48:54 DEBUG { 21:48:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 21:48:54 DEBUG }, 21:48:54 DEBUG "metric": { 21:48:54 DEBUG { 21:48:54 DEBUG "instance": "10.142.0.107:9100" 21:48:54 DEBUG "data": { 21:48:54 DEBUG "pod": "am-767f6b67b5-9zw22" 21:48:54 DEBUG { 21:48:54 DEBUG 21:48:54 DEBUG "result": [ 21:48:54 DEBUG "instance": "10.142.0.107:9100" 21:48:54 DEBUG "value": [ 21:48:54 DEBUG 21:48:54 DEBUG }, 21:48:54 DEBUG "result": [ 21:48:54 DEBUG 21:48:54 DEBUG "result": [ 21:48:54 DEBUG "pod": "am-767f6b67b5-9zw22" 21:48:54 DEBUG "instance": "10.142.0.107:9100" 21:48:54 DEBUG "metric": { 21:48:54 DEBUG { 21:48:54 DEBUG 21:48:54 DEBUG "metric": { 21:48:54 DEBUG "value": [ 21:48:54 DEBUG "pod": "ds-idrepo-0" 21:48:54 DEBUG "metric": { 21:48:54 DEBUG }, 21:48:54 DEBUG "resultType": "vector", 21:48:54 DEBUG 21:48:54 DEBUG }, 21:48:54 DEBUG "metric": { 21:48:54 DEBUG 21:48:54 DEBUG { 21:48:54 DEBUG }, 21:48:54 DEBUG 21:48:54 DEBUG [...] 21:48:54 DEBUG "value": [ 21:48:54 DEBUG { 21:48:54 DEBUG 21:48:54 DEBUG { 21:48:54 DEBUG }, 21:48:54 DEBUG "value": [ 21:48:54 DEBUG "pod": "ds-cts-0" 21:48:54 DEBUG "metric": { 21:48:54 DEBUG 21:48:54 DEBUG "pod": "am-767f6b67b5-9zw22" 21:48:54 DEBUG 21:48:54 DEBUG }, 21:48:54 DEBUG "pod": "am-767f6b67b5-9zw22" 21:48:54 DEBUG "value": [ 21:48:54 DEBUG "result": [ 21:48:54 DEBUG [...] 21:48:54 DEBUG "value": [ 21:48:54 DEBUG "pod": "am-767f6b67b5-9zw22" 21:48:54 DEBUG [...] 21:48:54 DEBUG "metric": { 21:48:54 DEBUG "value": [ 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG "metric": { 21:48:54 DEBUG 21:48:54 DEBUG "metric": { 21:48:54 DEBUG 21:48:54 DEBUG }, 21:48:54 DEBUG }, 21:48:54 DEBUG "value": [ 21:48:54 DEBUG [...] 21:48:54 DEBUG }, 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG }, 21:48:54 DEBUG "value": [ 21:48:54 DEBUG { 21:48:54 DEBUG "metric": { 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG "pod": "ds-idrepo-0" 21:48:54 DEBUG [...] 21:48:54 DEBUG "pod": "ds-idrepo-0" 21:48:54 DEBUG 21:48:54 DEBUG "value": [ 21:48:54 DEBUG 21:48:54 DEBUG "instance": "10.142.0.107:9100" 21:48:54 DEBUG 21:48:54 DEBUG "value": [ 21:48:54 DEBUG [...] 21:48:54 DEBUG "value": [ 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:48:54 DEBUG }, 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG }, 21:48:54 DEBUG [...] 21:48:54 DEBUG "pod": "overseer-0-967d957f7-zk66v" 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG [...] 21:48:54 DEBUG }, 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG }, 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG "value": [ 21:48:54 DEBUG [...] 21:48:54 DEBUG 21:48:54 DEBUG "pod": "am-767f6b67b5-vbltz" 21:48:54 DEBUG 21:48:54 DEBUG "value": [ 21:48:54 DEBUG 21:48:54 DEBUG }, 21:48:54 DEBUG 21:48:54 DEBUG "value": [ 21:48:54 DEBUG 21:48:54 DEBUG [...] 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG }, 21:48:54 DEBUG "value": [ 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG [...] 21:48:54 DEBUG }, 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG }, 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG "value": [ 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG [...] 21:48:54 DEBUG "value": [ 21:48:54 DEBUG 21:48:54 DEBUG "instance": "10.142.0.43:9100" 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG "value": [ 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG [...] 21:48:54 DEBUG "value": [ 21:48:54 DEBUG "pod": "am-767f6b67b5-vbltz" 21:48:54 DEBUG 21:48:54 DEBUG 1713646133, 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG "pod": "ds-cts-2" 21:48:54 DEBUG 21:48:54 DEBUG [...] 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG }, 21:48:54 DEBUG [...] 21:48:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 21:48:54 DEBUG [...] 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG [...] 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG 1713646133, 21:48:54 DEBUG }, 21:48:54 DEBUG [...] 21:48:54 DEBUG "0" 21:48:54 DEBUG 21:48:54 DEBUG "pod": "ds-cts-2" 21:48:54 DEBUG 21:48:54 DEBUG }, 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG "value": [ 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG [...] 21:48:54 DEBUG [...] 21:48:54 DEBUG "pod": "am-767f6b67b5-vbltz" 21:48:54 DEBUG 21:48:54 DEBUG "598985.5" 21:48:54 DEBUG "value": [ 21:48:54 DEBUG 21:48:54 DEBUG ] 21:48:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 21:48:54 DEBUG }, 21:48:54 DEBUG [...] 21:48:54 DEBUG "value": [ 21:48:54 DEBUG 21:48:54 DEBUG "pod": "am-767f6b67b5-vbltz" 21:48:54 DEBUG }, 21:48:54 DEBUG "value": [ 21:48:54 DEBUG [...] 21:48:54 DEBUG 1713646133, 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG "pod": "ds-cts-2" 21:48:54 DEBUG "instance": "10.142.0.43:9100" 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG }, 21:48:54 DEBUG 21:48:54 DEBUG ] 21:48:54 DEBUG 1713646133, 21:48:54 DEBUG 21:48:54 DEBUG } 21:48:54 DEBUG ] 21:48:54 DEBUG "value": [ 21:48:54 DEBUG }, 21:48:54 DEBUG 1713646133, 21:48:54 DEBUG 1713646133, 21:48:54 DEBUG 21:48:54 DEBUG [...] 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG "pod": "am-767f6b67b5-vbltz" 21:48:54 DEBUG }, 21:48:54 DEBUG }, 21:48:54 DEBUG 21:48:54 DEBUG }, 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG "value": [ 21:48:54 DEBUG "instance": "10.142.0.43:9100" 21:48:54 DEBUG } 21:48:54 DEBUG ] 21:48:54 DEBUG 21:48:54 DEBUG "pod": "overseer-0-967d957f7-zk66v" 21:48:54 DEBUG 21:48:54 DEBUG 1713646133, 21:48:54 DEBUG "value": [ 21:48:54 DEBUG 1713646133, 21:48:54 DEBUG "0" 21:48:54 DEBUG "0" 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG }, 21:48:54 DEBUG "value": [ 21:48:54 DEBUG "value": [ 21:48:54 DEBUG 21:48:54 DEBUG "pod": "overseer-0-967d957f7-zk66v" 21:48:54 DEBUG }, 21:48:54 DEBUG "value": [ 21:48:54 DEBUG 21:48:54 DEBUG "pod": "am-767f6b67b5-vbltz" 21:48:54 DEBUG }, 21:48:54 DEBUG "value": [ 21:48:54 DEBUG "3330.5" 21:48:54 DEBUG ] 21:48:54 DEBUG } 21:48:54 DEBUG 21:48:54 DEBUG "1009512448" 21:48:54 DEBUG "0.5513333333333332" 21:48:54 DEBUG ] 21:48:54 DEBUG } 21:48:54 DEBUG "pod": "ds-idrepo-2" 21:48:54 DEBUG }, 21:48:54 DEBUG "0.39193333333328534" 21:48:54 DEBUG "pod": "am-767f6b67b5-vbltz" 21:48:54 DEBUG }, 21:48:54 DEBUG 1713646133, 21:48:54 DEBUG "value": [ 21:48:54 DEBUG 1713646133, 21:48:54 DEBUG "0" 21:48:54 DEBUG ] 21:48:54 DEBUG } 21:48:54 DEBUG 1713646133, 21:48:54 DEBUG 1713646133, 21:48:54 DEBUG }, 21:48:54 DEBUG "value": [ 21:48:54 DEBUG 1713646133, 21:48:54 DEBUG "0.07333333333333107" 21:48:54 DEBUG ] 21:48:54 DEBUG } 21:48:54 DEBUG ] 21:48:54 DEBUG ] 21:48:54 DEBUG ] 21:48:54 DEBUG ] 21:48:54 DEBUG "value": [ 21:48:54 DEBUG 1713646133, 21:48:54 DEBUG ] 21:48:54 DEBUG "value": [ 21:48:54 DEBUG 1713646133, 21:48:54 DEBUG "3302.2" 21:48:54 DEBUG ] 21:48:54 DEBUG "value": [ 21:48:54 DEBUG 1713646133, 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG "3332.0666666666666" 21:48:54 DEBUG "49.377" 21:48:54 DEBUG } 21:48:54 DEBUG } 21:48:54 DEBUG } 21:48:54 DEBUG }, 21:48:54 DEBUG } 21:48:54 DEBUG 21:48:54 DEBUG } 21:48:54 DEBUG ] 21:48:54 DEBUG } 21:48:54 DEBUG } 21:48:54 DEBUG "0" 21:48:54 DEBUG ] 21:48:54 DEBUG } 21:48:54 DEBUG } 21:48:54 DEBUG "3972078572544" 21:48:54 DEBUG ] 21:48:54 DEBUG } 21:48:54 DEBUG "11.790107685276736" 21:48:54 DEBUG ] 21:48:54 DEBUG 21:48:54 DEBUG "instance": "10.142.0.43:9100" 21:48:54 DEBUG ] 21:48:54 DEBUG } 21:48:54 DEBUG ] 21:48:54 DEBUG } 21:48:54 DEBUG "value": [ 21:48:54 DEBUG "pod": "ds-idrepo-2" 21:48:54 DEBUG }, 21:48:54 DEBUG } 21:48:54 DEBUG ] 21:48:54 DEBUG ] 21:48:54 DEBUG ] 21:48:54 DEBUG 1713646133, 21:48:54 DEBUG 21:48:54 DEBUG 21:48:54 DEBUG ] 21:48:54 DEBUG ] 21:48:54 DEBUG } 21:48:54 DEBUG } 21:48:54 DEBUG 1713646133, 21:48:54 DEBUG }, 21:48:54 DEBUG "pod": "ds-idrepo-2" 21:48:54 DEBUG } 21:48:54 DEBUG } 21:48:54 DEBUG ] 21:48:54 DEBUG 1713646133, 21:48:54 DEBUG "6.511606288325545" 21:48:54 DEBUG ] 21:48:54 DEBUG } 21:48:54 DEBUG ] 21:48:54 DEBUG } 21:48:54 DEBUG } 21:48:54 DEBUG "3331.5333333333333" 21:48:54 DEBUG } 21:48:54 DEBUG } 21:48:54 DEBUG } 21:48:54 DEBUG } 21:48:54 DEBUG } 21:48:54 DEBUG ] 21:48:54 DEBUG } 21:48:54 DEBUG "value": [ 21:48:54 DEBUG 1713646133, 21:48:54 DEBUG ] 21:48:54 DEBUG } 21:48:54 DEBUG } 21:48:54 DEBUG } 21:48:54 DEBUG } 21:48:54 DEBUG "value": [ 21:48:54 DEBUG "instance": "10.142.0.43:9100" 21:48:54 DEBUG ] 21:48:54 DEBUG } 21:48:54 DEBUG ] 21:48:54 DEBUG } 21:48:54 DEBUG } 21:48:54 DEBUG "0" 21:48:54 DEBUG } 21:48:54 DEBUG "100378400.76666667" 21:48:54 DEBUG ] 21:48:54 DEBUG } 21:48:54 DEBUG ] 21:48:54 DEBUG } 21:48:54 DEBUG } 21:48:54 DEBUG 1713646133, 21:48:54 DEBUG "0" 21:48:54 DEBUG } 21:48:54 DEBUG } 21:48:54 DEBUG ] 21:48:54 DEBUG }, 21:48:54 DEBUG } 21:48:54 DEBUG ] 21:48:54 DEBUG } 21:48:54 DEBUG } 21:48:54 DEBUG ] 21:48:54 DEBUG } 21:48:54 DEBUG ] 21:48:54 DEBUG ] 21:48:54 DEBUG } 21:48:54 DEBUG "value": [ 21:48:54 DEBUG ] 21:48:54 DEBUG } 21:48:54 DEBUG ] 21:48:54 DEBUG }, 21:48:54 DEBUG "value": [ 21:48:54 DEBUG } 21:48:54 DEBUG } 21:48:54 DEBUG } 21:48:54 DEBUG 1713646133, 21:48:54 DEBUG } 21:48:54 DEBUG } 21:48:54 DEBUG } 21:48:54 DEBUG 1713646133, 21:48:54 DEBUG } 21:48:54 DEBUG ] 21:48:54 DEBUG } 21:48:54 DEBUG "0" 21:48:54 DEBUG ] 21:48:54 DEBUG } 21:48:54 DEBUG "100374343.93333334" 21:48:54 DEBUG ] 21:48:54 DEBUG } 21:48:54 DEBUG } 21:48:54 DEBUG } 21:48:54 DEBUG } 21:48:54 DEBUG ] 21:48:54 DEBUG ] 21:48:54 DEBUG } 21:48:54 DEBUG } 21:48:54 DEBUG } 21:48:54 DEBUG } 21:48:54 DEBUG } 21:49:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713646193" 21:49:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646193" 21:49:53 ERROR [http_cmd]: ERROR 21:49:53 ERROR --- status code --- 21:49:53 ERROR http status code is 400 (expected [200, 401, 403, 404]) 21:49:53 ERROR --- http response --- 21:49:53 INFO [http_cmd]: http status code OK 21:49:53 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 21:49:53 DEBUG --- status code --- 21:49:53 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 21:49:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:49:53 DEBUG --- http response --- 21:49:53 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 21:49:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646193" 21:49:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713646193" 21:49:53 INFO [http_cmd]: http status code OK 21:49:53 DEBUG --- status code --- 21:49:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:49:53 DEBUG --- http response --- 21:49:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:49:53 DEBUG ----- output ----- 21:49:53 DEBUG { 21:49:53 DEBUG "status": "success", 21:49:53 DEBUG "data": { 21:49:53 DEBUG "resultType": "vector", 21:49:53 DEBUG "result": [ 21:49:53 DEBUG { 21:49:53 DEBUG "metric": { 21:49:53 INFO [http_cmd]: http status code OK 21:49:53 DEBUG "pod": "ds-idrepo-0" 21:49:53 DEBUG --- status code --- 21:49:53 DEBUG }, 21:49:53 DEBUG "value": [ 21:49:53 DEBUG 21:49:53 DEBUG 21:49:53 DEBUG 21:49:53 DEBUG [...] 21:49:53 DEBUG 21:49:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:49:53 DEBUG 21:49:53 DEBUG --- http response --- 21:49:53 DEBUG 21:49:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:49:53 DEBUG "pod": "ds-cts-2" 21:49:53 DEBUG ----- output ----- 21:49:53 DEBUG }, 21:49:53 DEBUG { 21:49:53 DEBUG "value": [ 21:49:53 DEBUG "status": "success", 21:49:53 DEBUG 1713646193, 21:49:53 DEBUG "data": { 21:49:53 DEBUG "0" 21:49:53 DEBUG "resultType": "vector", 21:49:53 DEBUG ] 21:49:53 DEBUG "result": [ 21:49:53 DEBUG } 21:49:53 DEBUG { 21:49:53 DEBUG ] 21:49:53 DEBUG "metric": { 21:49:53 DEBUG } 21:49:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:49:53 DEBUG } 21:49:53 DEBUG }, 21:49:53 DEBUG "value": [ 21:49:53 DEBUG 21:49:53 DEBUG 21:49:53 DEBUG 21:49:53 DEBUG [...] 21:49:53 DEBUG 21:49:53 DEBUG 21:49:53 DEBUG 21:49:53 DEBUG "pod": "overseer-0-967d957f7-zk66v" 21:49:53 DEBUG }, 21:49:53 DEBUG "value": [ 21:49:53 DEBUG 1713646193, 21:49:53 DEBUG "64547271.53375138" 21:49:53 DEBUG ] 21:49:53 DEBUG } 21:49:53 DEBUG ] 21:49:53 DEBUG } 21:49:53 DEBUG } 21:49:53 INFO 21:49:53 INFO [loop_until]: kubectl --namespace=xlou top pods 21:49:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:49:53 INFO [loop_until]: OK (rc = 0) 21:49:53 DEBUG --- stdout --- 21:49:53 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 21863m 8761Mi am-767f6b67b5-qcqz2 22275m 8883Mi am-767f6b67b5-vbltz 24385m 8943Mi ds-cts-0 19398m 12027Mi ds-cts-1 19356m 11914Mi ds-cts-2 20245m 12194Mi ds-idrepo-0 9402m 3888Mi ds-idrepo-1 9454m 3975Mi ds-idrepo-2 9618m 3874Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-7dd48676f6-tdxl9 17m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-967d957f7-zk66v 11833m 872Mi 21:49:53 DEBUG --- stderr --- 21:49:53 DEBUG 21:49:54 INFO 21:49:54 INFO [loop_until]: kubectl --namespace=xlou top node 21:49:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646194" 21:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646194" 21:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646194" 21:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646194" 21:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646194" 21:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646194" 21:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646194" 21:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646194" 21:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646194" 21:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646194" 21:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646194" 21:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713646194" 21:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646194" 21:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713646194" 21:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646194" 21:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646194" 21:49:54 INFO [http_cmd]: http status code OK 21:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646194" 21:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713646194" 21:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646194" 21:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646194" 21:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646194" 21:49:54 INFO [http_cmd]: http status code OK 21:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646194" 21:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646194" 21:49:54 INFO [http_cmd]: http status code OK 21:49:54 INFO [loop_until]: OK (rc = 0) 21:49:54 INFO [http_cmd]: http status code OK 21:49:54 INFO [http_cmd]: http status code OK 21:49:54 INFO [http_cmd]: http status code OK 21:49:54 INFO [http_cmd]: http status code OK 21:49:54 INFO [http_cmd]: http status code OK 21:49:54 INFO [http_cmd]: http status code OK 21:49:54 DEBUG --- status code --- 21:49:54 INFO [http_cmd]: http status code OK 21:49:54 INFO [http_cmd]: http status code OK 21:49:54 INFO [http_cmd]: http status code OK 21:49:54 INFO [http_cmd]: http status code OK 21:49:54 INFO [http_cmd]: http status code OK 21:49:54 DEBUG --- status code --- 21:49:54 INFO [http_cmd]: http status code OK 21:49:54 INFO [http_cmd]: http status code OK 21:49:54 INFO [http_cmd]: http status code OK 21:49:54 DEBUG --- status code --- 21:49:54 INFO [http_cmd]: http status code OK 21:49:54 DEBUG --- stdout --- 21:49:54 DEBUG --- status code --- 21:49:54 DEBUG --- status code --- 21:49:54 DEBUG --- status code --- 21:49:54 INFO [http_cmd]: http status code OK 21:49:54 DEBUG --- status code --- 21:49:54 INFO [http_cmd]: http status code OK 21:49:54 INFO [http_cmd]: http status code OK 21:49:54 DEBUG --- status code --- 21:49:54 DEBUG --- status code --- 21:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:49:54 DEBUG --- status code --- 21:49:54 DEBUG --- status code --- 21:49:54 INFO [http_cmd]: http status code OK 21:49:54 DEBUG --- status code --- 21:49:54 DEBUG --- status code --- 21:49:54 DEBUG --- status code --- 21:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:49:54 DEBUG --- status code --- 21:49:54 DEBUG --- status code --- 21:49:54 DEBUG --- status code --- 21:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:49:54 DEBUG --- status code --- 21:49:54 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6204Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 10021m 33% 7646Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 21710m 72% 11487Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 3380Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 19829m 66% 15245Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 157m 0% 5206Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 22283m 74% 11777Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19687m 65% 3060Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 23866m 79% 11198Mi 10% gke-xlou-cdm-ds-0f601751-j488 10326m 34% 7286Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19827m 66% 16982Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 10133m 33% 7962Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20585m 68% 14897Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 12075m 75% 8814Mi 15% 21:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:49:54 DEBUG --- status code --- 21:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:49:54 DEBUG --- status code --- 21:49:54 DEBUG --- status code --- 21:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:49:54 INFO [http_cmd]: http status code OK 21:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:49:54 DEBUG --- http response --- 21:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:49:54 DEBUG --- status code --- 21:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:49:54 DEBUG --- http response --- 21:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:49:54 DEBUG --- http response --- 21:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:49:54 DEBUG --- http response --- 21:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:49:54 DEBUG --- http response --- 21:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:49:54 DEBUG --- stderr --- 21:49:54 DEBUG --- http response --- 21:49:54 DEBUG --- http response --- 21:49:54 DEBUG --- http response --- 21:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:49:54 DEBUG --- http response --- 21:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:49:54 DEBUG --- http response --- 21:49:54 DEBUG --- status code --- 21:49:54 DEBUG --- http response --- 21:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:49:54 DEBUG --- http response --- 21:49:54 DEBUG --- http response --- 21:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:49:54 DEBUG --- http response --- 21:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:49:54 DEBUG ----- output ----- 21:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:49:54 DEBUG --- http response --- 21:49:54 DEBUG 21:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:49:54 DEBUG --- http response --- 21:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:49:54 DEBUG ----- output ----- 21:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:49:54 DEBUG --- http response --- 21:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:49:54 DEBUG ----- output ----- 21:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:49:54 DEBUG --- http response --- 21:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:49:54 DEBUG ----- output ----- 21:49:54 DEBUG --- http response --- 21:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:49:54 DEBUG { 21:49:54 DEBUG --- http response --- 21:49:54 DEBUG --- http response --- 21:49:54 DEBUG ----- output ----- 21:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:49:54 DEBUG ----- output ----- 21:49:54 DEBUG ----- output ----- 21:49:54 DEBUG ----- output ----- 21:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:49:54 DEBUG ----- output ----- 21:49:54 DEBUG { 21:49:54 DEBUG ----- output ----- 21:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:49:54 DEBUG --- http response --- 21:49:54 DEBUG { 21:49:54 DEBUG ----- output ----- 21:49:54 DEBUG ----- output ----- 21:49:54 DEBUG ----- output ----- 21:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:49:54 DEBUG ----- output ----- 21:49:54 DEBUG { 21:49:54 DEBUG ----- output ----- 21:49:54 DEBUG ----- output ----- 21:49:54 DEBUG "status": "success", 21:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:49:54 DEBUG { 21:49:54 DEBUG "status": "success", 21:49:54 DEBUG ----- output ----- 21:49:54 DEBUG { 21:49:54 DEBUG { 21:49:54 DEBUG { 21:49:54 DEBUG --- http response --- 21:49:54 DEBUG "status": "success", 21:49:54 DEBUG { 21:49:54 DEBUG { 21:49:54 DEBUG ----- output ----- 21:49:54 DEBUG "status": "success", 21:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:49:54 DEBUG { 21:49:54 DEBUG { 21:49:54 DEBUG { 21:49:54 DEBUG ----- output ----- 21:49:54 DEBUG { 21:49:54 DEBUG "status": "success", 21:49:54 DEBUG { 21:49:54 DEBUG { 21:49:54 DEBUG "data": { 21:49:54 DEBUG ----- output ----- 21:49:54 DEBUG "data": { 21:49:54 DEBUG { 21:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:49:54 DEBUG "status": "success", 21:49:54 DEBUG "status": "success", 21:49:54 DEBUG "status": "success", 21:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:49:54 DEBUG "data": { 21:49:54 DEBUG "status": "success", 21:49:54 DEBUG "status": "success", 21:49:54 DEBUG { 21:49:54 DEBUG "data": { 21:49:54 DEBUG ----- output ----- 21:49:54 DEBUG "status": "success", 21:49:54 DEBUG "status": "success", 21:49:54 DEBUG "status": "success", 21:49:54 DEBUG { 21:49:54 DEBUG "status": "success", 21:49:54 DEBUG "data": { 21:49:54 DEBUG "resultType": "vector", 21:49:54 DEBUG "status": "success", 21:49:54 DEBUG "resultType": "vector", 21:49:54 DEBUG { 21:49:54 DEBUG "resultType": "vector", 21:49:54 DEBUG "status": "success", 21:49:54 DEBUG ----- output ----- 21:49:54 DEBUG "data": { 21:49:54 DEBUG "data": { 21:49:54 DEBUG "data": { 21:49:54 DEBUG ----- output ----- 21:49:54 DEBUG "resultType": "vector", 21:49:54 DEBUG "data": { 21:49:54 DEBUG "data": { 21:49:54 DEBUG "status": "success", 21:49:54 DEBUG "resultType": "vector", 21:49:54 DEBUG { 21:49:54 DEBUG "data": { 21:49:54 DEBUG "data": { 21:49:54 DEBUG "data": { 21:49:54 DEBUG "status": "success", 21:49:54 DEBUG "data": { 21:49:54 DEBUG "result": [ 21:49:54 DEBUG "data": { 21:49:54 DEBUG "status": "success", 21:49:54 DEBUG "result": [ 21:49:54 DEBUG "status": "success", 21:49:54 DEBUG "result": [ 21:49:54 DEBUG "data": { 21:49:54 DEBUG "resultType": "vector", 21:49:54 DEBUG "resultType": "vector", 21:49:54 DEBUG "resultType": "vector", 21:49:54 DEBUG "resultType": "vector", 21:49:54 DEBUG { 21:49:54 DEBUG "result": [ 21:49:54 DEBUG "resultType": "vector", 21:49:54 DEBUG "resultType": "vector", 21:49:54 DEBUG "data": { 21:49:54 DEBUG "result": [ 21:49:54 DEBUG "status": "success", 21:49:54 DEBUG "resultType": "vector", 21:49:54 DEBUG "resultType": "vector", 21:49:54 DEBUG "resultType": "vector", 21:49:54 DEBUG "resultType": "vector", 21:49:54 DEBUG "data": { 21:49:54 DEBUG { 21:49:54 DEBUG "resultType": "vector", 21:49:54 DEBUG "data": { 21:49:54 DEBUG { 21:49:54 DEBUG "data": { 21:49:54 DEBUG "resultType": "vector", 21:49:54 DEBUG { 21:49:54 DEBUG "result": [ 21:49:54 DEBUG "result": [ 21:49:54 DEBUG "result": [ 21:49:54 DEBUG { 21:49:54 DEBUG "status": "success", 21:49:54 DEBUG { 21:49:54 DEBUG "result": [ 21:49:54 DEBUG "result": [ 21:49:54 DEBUG "resultType": "vector", 21:49:54 DEBUG { 21:49:54 DEBUG "data": { 21:49:54 DEBUG "result": [ 21:49:54 DEBUG "result": [ 21:49:54 DEBUG "result": [ 21:49:54 DEBUG "result": [ 21:49:54 DEBUG "resultType": "vector", 21:49:54 DEBUG "metric": { 21:49:54 DEBUG "result": [ 21:49:54 DEBUG "resultType": "vector", 21:49:54 DEBUG "metric": { 21:49:54 DEBUG "result": [ 21:49:54 DEBUG { 21:49:54 DEBUG "status": "success", 21:49:54 DEBUG "data": { 21:49:54 DEBUG { 21:49:54 DEBUG "metric": { 21:49:54 DEBUG "pod": "ds-cts-0" 21:49:54 DEBUG "data": { 21:49:54 DEBUG "metric": { 21:49:54 DEBUG { 21:49:54 DEBUG { 21:49:54 DEBUG "result": [ 21:49:54 DEBUG "metric": { 21:49:54 DEBUG "pod": "ds-idrepo-0" 21:49:54 DEBUG { 21:49:54 DEBUG { 21:49:54 DEBUG { 21:49:54 DEBUG { 21:49:54 DEBUG "result": [ 21:49:54 DEBUG "pod": "am-767f6b67b5-9zw22" 21:49:54 DEBUG { 21:49:54 DEBUG "result": [ 21:49:54 DEBUG "pod": "am-767f6b67b5-9zw22" 21:49:54 DEBUG { 21:49:54 DEBUG "metric": { 21:49:54 DEBUG "resultType": "vector", 21:49:54 DEBUG { 21:49:54 DEBUG "metric": { 21:49:54 DEBUG "result": [ 21:49:54 DEBUG }, 21:49:54 DEBUG "resultType": "vector", 21:49:54 DEBUG "instance": "10.142.0.107:9100" 21:49:54 DEBUG "metric": { 21:49:54 DEBUG "pod": "am-767f6b67b5-9zw22" 21:49:54 DEBUG { 21:49:54 DEBUG }, 21:49:54 DEBUG "resultType": "vector", 21:49:54 DEBUG "metric": { 21:49:54 DEBUG "metric": { 21:49:54 DEBUG "metric": { 21:49:54 DEBUG "metric": { 21:49:54 DEBUG { 21:49:54 DEBUG }, 21:49:54 DEBUG "metric": { 21:49:54 DEBUG { 21:49:54 DEBUG }, 21:49:54 DEBUG "metric": { 21:49:54 DEBUG "pod": "am-767f6b67b5-9zw22" 21:49:54 DEBUG "result": [ 21:49:54 DEBUG "metric": { 21:49:54 DEBUG "pod": "am-767f6b67b5-9zw22" 21:49:54 DEBUG { 21:49:54 DEBUG "value": [ 21:49:54 DEBUG "result": [ 21:49:54 DEBUG }, 21:49:54 DEBUG }, 21:49:54 DEBUG "metric": { 21:49:54 DEBUG "metric": { 21:49:54 DEBUG "value": [ 21:49:54 DEBUG "result": [ 21:49:54 DEBUG "instance": "10.142.0.107:9100" 21:49:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 21:49:54 DEBUG "pod": "am-767f6b67b5-9zw22" 21:49:54 DEBUG "pod": "ds-cts-0" 21:49:54 DEBUG "metric": { 21:49:54 DEBUG "value": [ 21:49:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:49:54 DEBUG "metric": { 21:49:54 DEBUG "value": [ 21:49:54 DEBUG "pod": "ds-cts-0" 21:49:54 DEBUG }, 21:49:54 DEBUG { 21:49:54 DEBUG "instance": "10.142.0.107:9100" 21:49:54 DEBUG }, 21:49:54 DEBUG "metric": { 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG "value": [ 21:49:54 DEBUG "value": [ 21:49:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:49:54 DEBUG "pod": "ds-idrepo-0" 21:49:54 DEBUG 21:49:54 DEBUG { 21:49:54 DEBUG }, 21:49:54 DEBUG }, 21:49:54 DEBUG }, 21:49:54 DEBUG }, 21:49:54 DEBUG "pod": "ds-idrepo-0" 21:49:54 DEBUG 21:49:54 DEBUG }, 21:49:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:49:54 DEBUG 21:49:54 DEBUG }, 21:49:54 DEBUG "value": [ 21:49:54 DEBUG "metric": { 21:49:54 DEBUG }, 21:49:54 DEBUG "value": [ 21:49:54 DEBUG "pod": "am-767f6b67b5-9zw22" 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG }, 21:49:54 DEBUG }, 21:49:54 DEBUG 21:49:54 DEBUG "metric": { 21:49:54 DEBUG "value": [ 21:49:54 DEBUG "value": [ 21:49:54 DEBUG "value": [ 21:49:54 DEBUG "value": [ 21:49:54 DEBUG }, 21:49:54 DEBUG 21:49:54 DEBUG "value": [ 21:49:54 DEBUG }, 21:49:54 DEBUG 21:49:54 DEBUG "value": [ 21:49:54 DEBUG 21:49:54 DEBUG "instance": "10.142.0.107:9100" 21:49:54 DEBUG "value": [ 21:49:54 DEBUG 21:49:54 DEBUG }, 21:49:54 DEBUG [...] 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG "value": [ 21:49:54 DEBUG "value": [ 21:49:54 DEBUG 21:49:54 DEBUG "instance": "10.142.0.107:9100" 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG "value": [ 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG "value": [ 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG { 21:49:54 DEBUG 21:49:54 DEBUG }, 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG "value": [ 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG [...] 21:49:54 DEBUG }, 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG [...] 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG [...] 21:49:54 DEBUG 21:49:54 DEBUG "metric": { 21:49:54 DEBUG 21:49:54 DEBUG "value": [ 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG [...] 21:49:54 DEBUG [...] 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG "value": [ 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 21:49:54 DEBUG [...] 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG [...] 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG [...] 21:49:54 DEBUG [...] 21:49:54 DEBUG [...] 21:49:54 DEBUG [...] 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG [...] 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG [...] 21:49:54 DEBUG }, 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG [...] 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG "pod": "ds-cts-2" 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG [...] 21:49:54 DEBUG [...] 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG [...] 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG [...] 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG "value": [ 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG [...] 21:49:54 DEBUG 21:49:54 DEBUG [...] 21:49:54 DEBUG }, 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG "pod": "ds-idrepo-2" 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG "pod": "am-767f6b67b5-vbltz" 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG "pod": "am-767f6b67b5-vbltz" 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG "value": [ 21:49:54 DEBUG "instance": "10.142.0.43:9100" 21:49:54 DEBUG "pod": "am-767f6b67b5-vbltz" 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG }, 21:49:54 DEBUG [...] 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG }, 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG }, 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG "pod": "am-767f6b67b5-vbltz" 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG "pod": "am-767f6b67b5-vbltz" 21:49:54 DEBUG 21:49:54 DEBUG 1713646194, 21:49:54 DEBUG }, 21:49:54 DEBUG }, 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG "value": [ 21:49:54 DEBUG 21:49:54 DEBUG "instance": "10.142.0.43:9100" 21:49:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 21:49:54 DEBUG "pod": "am-767f6b67b5-vbltz" 21:49:54 DEBUG "pod": "ds-cts-2" 21:49:54 DEBUG 21:49:54 DEBUG "value": [ 21:49:54 DEBUG "pod": "ds-cts-0" 21:49:54 DEBUG 21:49:54 DEBUG "value": [ 21:49:54 DEBUG "pod": "ds-cts-2" 21:49:54 DEBUG 21:49:54 DEBUG }, 21:49:54 DEBUG 21:49:54 DEBUG 21:49:54 DEBUG }, 21:49:54 DEBUG 21:49:54 DEBUG "0" 21:49:54 DEBUG "value": [ 21:49:54 DEBUG "value": [ 21:49:54 DEBUG "pod": "ds-cts-0" 21:49:54 DEBUG "pod": "ds-idrepo-2" 21:49:54 DEBUG 1713646194, 21:49:54 DEBUG 21:49:54 DEBUG }, 21:49:54 DEBUG }, 21:49:54 DEBUG }, 21:49:54 DEBUG }, 21:49:54 DEBUG "pod": "ds-idrepo-2" 21:49:54 DEBUG 1713646194, 21:49:54 DEBUG }, 21:49:54 DEBUG "pod": "ds-cts-0" 21:49:54 DEBUG 1713646194, 21:49:54 DEBUG }, 21:49:54 DEBUG [...] 21:49:54 DEBUG "value": [ 21:49:54 DEBUG "instance": "10.142.0.43:9100" 21:49:54 DEBUG "instance": "10.142.0.43:9100" 21:49:54 DEBUG "value": [ 21:49:54 DEBUG "pod": "am-767f6b67b5-vbltz" 21:49:54 DEBUG ] 21:49:54 DEBUG 1713646194, 21:49:54 DEBUG 1713646194, 21:49:54 DEBUG }, 21:49:54 DEBUG "value": [ 21:49:54 DEBUG "0" 21:49:54 DEBUG 21:49:54 DEBUG "value": [ 21:49:54 DEBUG "value": [ 21:49:54 DEBUG "value": [ 21:49:54 DEBUG "value": [ 21:49:54 DEBUG }, 21:49:54 DEBUG "49.377" 21:49:54 DEBUG "value": [ 21:49:54 DEBUG }, 21:49:54 DEBUG "3331.233333333333" 21:49:54 DEBUG "value": [ 21:49:54 DEBUG 21:49:54 DEBUG 1713646194, 21:49:54 DEBUG }, 21:49:54 DEBUG }, 21:49:54 DEBUG 1713646194, 21:49:54 DEBUG }, 21:49:54 DEBUG } 21:49:54 DEBUG "100223346.06666666" 21:49:54 DEBUG "3324.3333333333335" 21:49:54 DEBUG }, 21:49:54 DEBUG 1713646194, 21:49:54 DEBUG ] 21:49:54 DEBUG "instance": "10.142.0.43:9100" 21:49:54 DEBUG 1713646194, 21:49:54 DEBUG 1713646194, 21:49:54 DEBUG 1713646194, 21:49:54 DEBUG 1713646194, 21:49:54 DEBUG "value": [ 21:49:54 DEBUG ] 21:49:54 DEBUG 1713646194, 21:49:54 DEBUG "value": [ 21:49:54 DEBUG ] 21:49:54 DEBUG 1713646194, 21:49:54 DEBUG 21:49:54 DEBUG "599267" 21:49:54 DEBUG "value": [ 21:49:54 DEBUG "value": [ 21:49:54 DEBUG "3334.4666666666667" 21:49:54 DEBUG "value": [ 21:49:54 DEBUG ] 21:49:54 DEBUG ] 21:49:54 DEBUG ] 21:49:54 DEBUG "value": [ 21:49:54 DEBUG "9.061374076231864" 21:49:54 DEBUG } 21:49:54 DEBUG }, 21:49:54 DEBUG "0.3971333333333822" 21:49:54 DEBUG "11.809421024027493" 21:49:54 DEBUG "3330.866666666667" 21:49:54 DEBUG "0.5157333333333332" 21:49:54 DEBUG 1713646194, 21:49:54 DEBUG } 21:49:54 DEBUG "34225813.83453556" 21:49:54 DEBUG 1713646194, 21:49:54 DEBUG } 21:49:54 DEBUG "0" 21:49:54 DEBUG 21:49:54 DEBUG ] 21:49:54 DEBUG } 21:49:54 DEBUG 1713646194, 21:49:54 DEBUG ] 21:49:54 DEBUG 1713646194, 21:49:54 DEBUG } 21:49:54 DEBUG } 21:49:54 DEBUG } 21:49:54 DEBUG 1713646194, 21:49:54 DEBUG ] 21:49:54 DEBUG ] 21:49:54 DEBUG "value": [ 21:49:54 DEBUG ] 21:49:54 DEBUG ] 21:49:54 DEBUG ] 21:49:54 DEBUG ] 21:49:54 DEBUG } 21:49:54 DEBUG ] 21:49:54 DEBUG ] 21:49:54 DEBUG "25451677.138504937" 21:49:54 DEBUG ] 21:49:54 DEBUG ] 21:49:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 21:49:54 DEBUG 1713646194, 21:49:54 DEBUG ] 21:49:54 DEBUG "3977369105408" 21:49:54 DEBUG } 21:49:54 DEBUG "0" 21:49:54 DEBUG } 21:49:54 DEBUG ] 21:49:54 DEBUG ] 21:49:54 DEBUG "0" 21:49:54 DEBUG } 21:49:54 DEBUG } 21:49:54 DEBUG 1713646194, 21:49:54 DEBUG } 21:49:54 DEBUG } 21:49:54 DEBUG } 21:49:54 DEBUG ] 21:49:54 DEBUG "0" 21:49:54 DEBUG } 21:49:54 DEBUG } 21:49:54 DEBUG ] 21:49:54 DEBUG } 21:49:54 DEBUG } 21:49:54 DEBUG }, 21:49:54 DEBUG "0.06833333333332235" 21:49:54 DEBUG } 21:49:54 DEBUG ] 21:49:54 DEBUG } 21:49:54 DEBUG ] 21:49:54 DEBUG } 21:49:54 DEBUG } 21:49:54 DEBUG ] 21:49:54 DEBUG ] 21:49:54 DEBUG } 21:49:54 DEBUG "100218614.53333333" 21:49:54 DEBUG ] 21:49:54 DEBUG } 21:49:54 DEBUG ] 21:49:54 DEBUG } 21:49:54 DEBUG ] 21:49:54 DEBUG } 21:49:54 DEBUG ] 21:49:54 DEBUG } 21:49:54 DEBUG } 21:49:54 DEBUG ] 21:49:54 DEBUG "value": [ 21:49:54 DEBUG ] 21:49:54 DEBUG } 21:49:54 DEBUG ] 21:49:54 DEBUG ] 21:49:54 DEBUG } 21:49:54 DEBUG } 21:49:54 DEBUG } 21:49:54 DEBUG } 21:49:54 DEBUG } 21:49:54 DEBUG ] 21:49:54 DEBUG ] 21:49:54 DEBUG ] 21:49:54 DEBUG } 21:49:54 DEBUG } 21:49:54 DEBUG } 21:49:54 DEBUG } 21:49:54 DEBUG ] 21:49:54 DEBUG } 21:49:54 DEBUG 1713646194, 21:49:54 DEBUG } 21:49:54 DEBUG } 21:49:54 DEBUG } 21:49:54 DEBUG ] 21:49:54 DEBUG ] 21:49:54 DEBUG } 21:49:54 DEBUG } 21:49:54 DEBUG } 21:49:54 DEBUG } 21:49:54 DEBUG } 21:49:54 DEBUG ] 21:49:54 DEBUG } 21:49:54 DEBUG } 21:49:54 DEBUG } 21:49:54 DEBUG "1009582080" 21:49:54 DEBUG ] 21:49:54 DEBUG } 21:49:54 DEBUG } 21:49:54 DEBUG } 21:49:54 DEBUG } 21:49:54 DEBUG } 21:49:54 DEBUG } 21:49:54 DEBUG } 21:49:54 DEBUG } 21:49:54 DEBUG } 21:49:54 DEBUG ] 21:49:54 DEBUG } 21:49:54 DEBUG } 21:49:54 DEBUG } 21:49:54 DEBUG } 21:49:54 DEBUG } 21:49:54 DEBUG } 21:49:54 DEBUG ] 21:49:54 DEBUG } 21:49:54 DEBUG } 21:50:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713646253" 21:50:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646253" 21:50:53 INFO [http_cmd]: http status code OK 21:50:53 DEBUG --- status code --- 21:50:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:50:53 DEBUG --- http response --- 21:50:53 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 21:50:53 ERROR [http_cmd]: ERROR 21:50:53 ERROR --- status code --- 21:50:53 ERROR http status code is 400 (expected [200, 401, 403, 404]) 21:50:53 ERROR --- http response --- 21:50:53 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 21:50:53 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 21:50:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713646253" 21:50:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646253" 21:50:53 INFO [http_cmd]: http status code OK 21:50:53 DEBUG --- status code --- 21:50:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:50:53 DEBUG --- http response --- 21:50:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:50:53 INFO [http_cmd]: http status code OK 21:50:53 DEBUG ----- output ----- 21:50:53 DEBUG --- status code --- 21:50:53 DEBUG { 21:50:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:50:53 DEBUG "status": "success", 21:50:53 DEBUG --- http response --- 21:50:53 DEBUG "data": { 21:50:53 DEBUG "resultType": "vector", 21:50:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:50:53 DEBUG "result": [ 21:50:53 DEBUG ----- output ----- 21:50:53 DEBUG { 21:50:53 DEBUG { 21:50:53 DEBUG "metric": { 21:50:53 DEBUG "status": "success", 21:50:53 DEBUG "pod": "ds-idrepo-0" 21:50:53 DEBUG "data": { 21:50:53 DEBUG }, 21:50:53 DEBUG "resultType": "vector", 21:50:53 DEBUG "value": [ 21:50:53 DEBUG "result": [ 21:50:53 DEBUG 21:50:53 DEBUG { 21:50:53 DEBUG 21:50:53 DEBUG "metric": { 21:50:53 DEBUG 21:50:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:50:53 DEBUG [...] 21:50:53 DEBUG }, 21:50:53 DEBUG 21:50:53 DEBUG "value": [ 21:50:53 DEBUG 21:50:53 DEBUG 21:50:53 DEBUG 21:50:53 DEBUG 21:50:53 DEBUG "pod": "ds-cts-2" 21:50:53 DEBUG 21:50:53 DEBUG }, 21:50:53 DEBUG [...] 21:50:53 DEBUG "value": [ 21:50:53 DEBUG 21:50:53 DEBUG 1713646253, 21:50:53 DEBUG 21:50:53 DEBUG 21:50:53 DEBUG "pod": "am-767f6b67b5-vbltz" 21:50:53 DEBUG }, 21:50:53 DEBUG "value": [ 21:50:53 DEBUG 1713646253, 21:50:53 DEBUG "267834740.54260266" 21:50:53 DEBUG "0" 21:50:53 DEBUG ] 21:50:53 DEBUG ] 21:50:53 DEBUG } 21:50:53 DEBUG } 21:50:53 DEBUG ] 21:50:53 DEBUG ] 21:50:53 DEBUG } 21:50:53 DEBUG } 21:50:53 DEBUG } 21:50:53 DEBUG } 21:50:53 INFO 21:50:53 INFO [loop_until]: kubectl --namespace=xlou top pods 21:50:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:50:53 INFO [loop_until]: OK (rc = 0) 21:50:53 DEBUG --- stdout --- 21:50:53 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 21806m 8761Mi am-767f6b67b5-qcqz2 22300m 8879Mi am-767f6b67b5-vbltz 23749m 8933Mi ds-cts-0 20076m 12043Mi ds-cts-1 19933m 11919Mi ds-cts-2 20528m 12260Mi ds-idrepo-0 9440m 3888Mi ds-idrepo-1 9652m 3975Mi ds-idrepo-2 9598m 3874Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-7dd48676f6-tdxl9 2m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-967d957f7-zk66v 11960m 875Mi 21:50:53 DEBUG --- stderr --- 21:50:53 DEBUG 21:50:54 INFO 21:50:54 INFO [loop_until]: kubectl --namespace=xlou top node 21:50:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646254" 21:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646254" 21:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713646254" 21:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646254" 21:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646254" 21:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646254" 21:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646254" 21:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646254" 21:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646254" 21:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646254" 21:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646254" 21:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646254" 21:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713646254" 21:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646254" 21:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646254" 21:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646254" 21:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646254" 21:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646254" 21:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646254" 21:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713646254" 21:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646254" 21:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646254" 21:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646254" 21:50:54 INFO [http_cmd]: http status code OK 21:50:54 INFO [http_cmd]: http status code OK 21:50:54 INFO [http_cmd]: http status code OK 21:50:54 INFO [http_cmd]: http status code OK 21:50:54 INFO [http_cmd]: http status code OK 21:50:54 INFO [http_cmd]: http status code OK 21:50:54 INFO [http_cmd]: http status code OK 21:50:54 INFO [loop_until]: OK (rc = 0) 21:50:54 INFO [http_cmd]: http status code OK 21:50:54 INFO [http_cmd]: http status code OK 21:50:54 INFO [http_cmd]: http status code OK 21:50:54 INFO [http_cmd]: http status code OK 21:50:54 DEBUG --- status code --- 21:50:54 INFO [http_cmd]: http status code OK 21:50:54 INFO [http_cmd]: http status code OK 21:50:54 INFO [http_cmd]: http status code OK 21:50:54 INFO [http_cmd]: http status code OK 21:50:54 DEBUG --- status code --- 21:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:50:54 DEBUG --- status code --- 21:50:54 DEBUG --- status code --- 21:50:54 DEBUG --- status code --- 21:50:54 INFO [http_cmd]: http status code OK 21:50:54 DEBUG --- status code --- 21:50:54 DEBUG --- status code --- 21:50:54 DEBUG --- status code --- 21:50:54 DEBUG --- stdout --- 21:50:54 DEBUG --- status code --- 21:50:54 INFO [http_cmd]: http status code OK 21:50:54 DEBUG --- status code --- 21:50:54 DEBUG --- status code --- 21:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:50:54 INFO [http_cmd]: http status code OK 21:50:54 INFO [http_cmd]: http status code OK 21:50:54 DEBUG --- status code --- 21:50:54 INFO [http_cmd]: http status code OK 21:50:54 DEBUG --- status code --- 21:50:54 DEBUG --- status code --- 21:50:54 INFO [http_cmd]: http status code OK 21:50:54 DEBUG --- status code --- 21:50:54 INFO [http_cmd]: http status code OK 21:50:54 DEBUG --- http response --- 21:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:50:54 INFO [http_cmd]: http status code OK 21:50:54 DEBUG --- status code --- 21:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:50:54 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 6200Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 9989m 33% 7642Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 21755m 72% 11488Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 82m 0% 3384Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 19822m 66% 15269Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 143m 0% 5234Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 22280m 74% 11769Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19664m 65% 3053Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 24737m 82% 11183Mi 10% gke-xlou-cdm-ds-0f601751-j488 10354m 34% 7288Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19892m 66% 16974Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 10187m 34% 7957Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20412m 68% 14963Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 12066m 75% 8815Mi 15% 21:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:50:54 DEBUG --- status code --- 21:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:50:54 DEBUG --- http response --- 21:50:54 DEBUG --- status code --- 21:50:54 DEBUG --- status code --- 21:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:50:54 DEBUG --- status code --- 21:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:50:54 DEBUG --- status code --- 21:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:50:54 DEBUG --- status code --- 21:50:54 DEBUG --- http response --- 21:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:50:54 DEBUG --- http response --- 21:50:54 DEBUG --- http response --- 21:50:54 DEBUG --- status code --- 21:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:50:54 DEBUG --- http response --- 21:50:54 DEBUG --- http response --- 21:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:50:54 DEBUG --- stderr --- 21:50:54 DEBUG --- http response --- 21:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:50:54 DEBUG --- http response --- 21:50:54 DEBUG --- http response --- 21:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:50:54 DEBUG --- http response --- 21:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:50:54 DEBUG --- http response --- 21:50:54 DEBUG --- http response --- 21:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:50:54 DEBUG --- http response --- 21:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:50:54 DEBUG ----- output ----- 21:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:50:54 DEBUG --- http response --- 21:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:50:54 DEBUG ----- output ----- 21:50:54 DEBUG --- http response --- 21:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:50:54 DEBUG --- http response --- 21:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:50:54 DEBUG ----- output ----- 21:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:50:54 DEBUG --- http response --- 21:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:50:54 DEBUG --- http response --- 21:50:54 DEBUG --- http response --- 21:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:50:54 DEBUG --- http response --- 21:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:50:54 DEBUG --- http response --- 21:50:54 DEBUG ----- output ----- 21:50:54 DEBUG { 21:50:54 DEBUG ----- output ----- 21:50:54 DEBUG ----- output ----- 21:50:54 DEBUG --- http response --- 21:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:50:54 DEBUG ----- output ----- 21:50:54 DEBUG { 21:50:54 DEBUG ----- output ----- 21:50:54 DEBUG 21:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:50:54 DEBUG ----- output ----- 21:50:54 DEBUG ----- output ----- 21:50:54 DEBUG { 21:50:54 DEBUG "status": "success", 21:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:50:54 DEBUG ----- output ----- 21:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:50:54 DEBUG ----- output ----- 21:50:54 DEBUG ----- output ----- 21:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:50:54 DEBUG ----- output ----- 21:50:54 DEBUG { 21:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:50:54 DEBUG "status": "success", 21:50:54 DEBUG { 21:50:54 DEBUG { 21:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:50:54 DEBUG ----- output ----- 21:50:54 DEBUG ----- output ----- 21:50:54 DEBUG "status": "success", 21:50:54 DEBUG { 21:50:54 DEBUG "status": "success", 21:50:54 DEBUG { 21:50:54 DEBUG { 21:50:54 DEBUG "status": "success", 21:50:54 DEBUG "data": { 21:50:54 DEBUG ----- output ----- 21:50:54 DEBUG { 21:50:54 DEBUG ----- output ----- 21:50:54 DEBUG ----- output ----- 21:50:54 DEBUG { 21:50:54 DEBUG { 21:50:54 DEBUG { 21:50:54 DEBUG "status": "success", 21:50:54 DEBUG ----- output ----- 21:50:54 DEBUG "data": { 21:50:54 DEBUG "resultType": "vector", 21:50:54 DEBUG "result": [ 21:50:54 DEBUG "status": "success", 21:50:54 DEBUG { 21:50:54 DEBUG { 21:50:54 DEBUG { 21:50:54 DEBUG "data": { 21:50:54 DEBUG "data": { 21:50:54 DEBUG "status": "success", 21:50:54 DEBUG ----- output ----- 21:50:54 DEBUG "data": { 21:50:54 DEBUG ----- output ----- 21:50:54 DEBUG "resultType": "vector", 21:50:54 DEBUG { 21:50:54 DEBUG "status": "success", 21:50:54 DEBUG { 21:50:54 DEBUG { 21:50:54 DEBUG "status": "success", 21:50:54 DEBUG "data": { 21:50:54 DEBUG "status": "success", 21:50:54 DEBUG "data": { 21:50:54 DEBUG { 21:50:54 DEBUG "status": "success", 21:50:54 DEBUG { 21:50:54 DEBUG "data": { 21:50:54 DEBUG "status": "success", 21:50:54 DEBUG "status": "success", 21:50:54 DEBUG "status": "success", 21:50:54 DEBUG "data": { 21:50:54 DEBUG "resultType": "vector", 21:50:54 DEBUG "resultType": "vector", 21:50:54 DEBUG "data": { 21:50:54 DEBUG { 21:50:54 DEBUG "resultType": "vector", 21:50:54 DEBUG { 21:50:54 DEBUG "result": [ 21:50:54 DEBUG "status": "success", 21:50:54 DEBUG "data": { 21:50:54 DEBUG "resultType": "vector", 21:50:54 DEBUG "status": "success", 21:50:54 DEBUG "status": "success", 21:50:54 DEBUG "resultType": "vector", 21:50:54 DEBUG "data": { 21:50:54 DEBUG "resultType": "vector", 21:50:54 DEBUG ----- output ----- 21:50:54 DEBUG "data": { 21:50:54 DEBUG "metric": { 21:50:54 DEBUG "resultType": "vector", 21:50:54 DEBUG "data": { 21:50:54 DEBUG "data": { 21:50:54 DEBUG "resultType": "vector", 21:50:54 DEBUG "result": [ 21:50:54 DEBUG "result": [ 21:50:54 DEBUG "resultType": "vector", 21:50:54 DEBUG "status": "success", 21:50:54 DEBUG "status": "success", 21:50:54 DEBUG "result": [ 21:50:54 DEBUG "status": "success", 21:50:54 DEBUG { 21:50:54 DEBUG "data": { 21:50:54 DEBUG "result": [ 21:50:54 DEBUG "data": { 21:50:54 DEBUG "status": "success", 21:50:54 DEBUG "data": { 21:50:54 DEBUG "result": [ 21:50:54 DEBUG "resultType": "vector", 21:50:54 DEBUG "result": [ 21:50:54 DEBUG { 21:50:54 DEBUG "resultType": "vector", 21:50:54 DEBUG "result": [ 21:50:54 DEBUG "result": [ 21:50:54 DEBUG "resultType": "vector", 21:50:54 DEBUG "resultType": "vector", 21:50:54 DEBUG "result": [ 21:50:54 DEBUG { 21:50:54 DEBUG { 21:50:54 DEBUG "result": [ 21:50:54 DEBUG "data": { 21:50:54 DEBUG "data": { 21:50:54 DEBUG { 21:50:54 DEBUG "data": { 21:50:54 DEBUG "metric": { 21:50:54 DEBUG "resultType": "vector", 21:50:54 DEBUG { 21:50:54 DEBUG "resultType": "vector", 21:50:54 DEBUG "data": { 21:50:54 DEBUG "resultType": "vector", 21:50:54 DEBUG { 21:50:54 DEBUG "result": [ 21:50:54 DEBUG { 21:50:54 DEBUG "status": "success", 21:50:54 DEBUG { 21:50:54 DEBUG "pod": "ds-cts-0" 21:50:54 DEBUG { 21:50:54 DEBUG "result": [ 21:50:54 DEBUG "result": [ 21:50:54 DEBUG { 21:50:54 DEBUG "metric": { 21:50:54 DEBUG "metric": { 21:50:54 DEBUG { 21:50:54 DEBUG "resultType": "vector", 21:50:54 DEBUG "result": [ 21:50:54 DEBUG "metric": { 21:50:54 DEBUG "resultType": "vector", 21:50:54 DEBUG "result": [ 21:50:54 DEBUG { 21:50:54 DEBUG "metric": { 21:50:54 DEBUG "result": [ 21:50:54 DEBUG "resultType": "vector", 21:50:54 DEBUG "result": [ 21:50:54 DEBUG "metric": { 21:50:54 DEBUG { 21:50:54 DEBUG "metric": { 21:50:54 DEBUG "data": { 21:50:54 DEBUG "metric": { 21:50:54 DEBUG }, 21:50:54 DEBUG "value": [ 21:50:54 DEBUG { 21:50:54 DEBUG { 21:50:54 DEBUG "metric": { 21:50:54 DEBUG "pod": "am-767f6b67b5-9zw22" 21:50:54 DEBUG "pod": "am-767f6b67b5-9zw22" 21:50:54 DEBUG }, 21:50:54 DEBUG { 21:50:54 DEBUG "resultType": "vector", 21:50:54 DEBUG "result": [ 21:50:54 DEBUG { 21:50:54 DEBUG "result": [ 21:50:54 DEBUG "instance": "10.142.0.107:9100" 21:50:54 DEBUG }, 21:50:54 DEBUG { 21:50:54 DEBUG "result": [ 21:50:54 DEBUG { 21:50:54 DEBUG "instance": "10.142.0.107:9100" 21:50:54 DEBUG "metric": { 21:50:54 DEBUG "pod": "ds-cts-0" 21:50:54 DEBUG "resultType": "vector", 21:50:54 DEBUG "pod": "ds-idrepo-0" 21:50:54 DEBUG "metric": { 21:50:54 DEBUG 21:50:54 DEBUG "metric": { 21:50:54 DEBUG "metric": { 21:50:54 DEBUG "instance": "10.142.0.107:9100" 21:50:54 DEBUG }, 21:50:54 DEBUG "metric": { 21:50:54 DEBUG "value": [ 21:50:54 DEBUG 21:50:54 DEBUG "metric": { 21:50:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:50:54 DEBUG "metric": { 21:50:54 DEBUG { 21:50:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:50:54 DEBUG "value": [ 21:50:54 DEBUG "metric": { 21:50:54 DEBUG { 21:50:54 DEBUG "metric": { 21:50:54 DEBUG }, 21:50:54 DEBUG "pod": "ds-idrepo-0" 21:50:54 DEBUG }, 21:50:54 DEBUG "result": [ 21:50:54 DEBUG }, 21:50:54 DEBUG "pod": "am-767f6b67b5-9zw22" 21:50:54 DEBUG 21:50:54 DEBUG "pod": "am-767f6b67b5-9zw22" 21:50:54 DEBUG "pod": "ds-idrepo-0" 21:50:54 DEBUG }, 21:50:54 DEBUG "value": [ 21:50:54 DEBUG "pod": "am-767f6b67b5-9zw22" 21:50:54 DEBUG 21:50:54 DEBUG "metric": { 21:50:54 DEBUG "pod": "am-767f6b67b5-9zw22" 21:50:54 DEBUG }, 21:50:54 DEBUG "pod": "am-767f6b67b5-9zw22" 21:50:54 DEBUG "metric": { 21:50:54 DEBUG }, 21:50:54 DEBUG 21:50:54 DEBUG "instance": "10.142.0.107:9100" 21:50:54 DEBUG "metric": { 21:50:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 21:50:54 DEBUG "value": [ 21:50:54 DEBUG 21:50:54 DEBUG "value": [ 21:50:54 DEBUG 21:50:54 DEBUG "value": [ 21:50:54 DEBUG }, 21:50:54 DEBUG 21:50:54 DEBUG }, 21:50:54 DEBUG "value": [ 21:50:54 DEBUG "value": [ 21:50:54 DEBUG 21:50:54 DEBUG }, 21:50:54 DEBUG 21:50:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:50:54 DEBUG }, 21:50:54 DEBUG "value": [ 21:50:54 DEBUG }, 21:50:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 21:50:54 DEBUG "value": [ 21:50:54 DEBUG 21:50:54 DEBUG }, 21:50:54 DEBUG "instance": "10.142.0.107:9100" 21:50:54 DEBUG }, 21:50:54 DEBUG 21:50:54 DEBUG }, 21:50:54 DEBUG 21:50:54 DEBUG { 21:50:54 DEBUG 21:50:54 DEBUG "value": [ 21:50:54 DEBUG [...] 21:50:54 DEBUG }, 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG [...] 21:50:54 DEBUG "value": [ 21:50:54 DEBUG 21:50:54 DEBUG }, 21:50:54 DEBUG "value": [ 21:50:54 DEBUG }, 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG "value": [ 21:50:54 DEBUG }, 21:50:54 DEBUG "value": [ 21:50:54 DEBUG 21:50:54 DEBUG "value": [ 21:50:54 DEBUG 21:50:54 DEBUG "metric": { 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG "value": [ 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG "value": [ 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG "value": [ 21:50:54 DEBUG 21:50:54 DEBUG "value": [ 21:50:54 DEBUG 21:50:54 DEBUG [...] 21:50:54 DEBUG 21:50:54 DEBUG "value": [ 21:50:54 DEBUG 21:50:54 DEBUG [...] 21:50:54 DEBUG 21:50:54 DEBUG [...] 21:50:54 DEBUG "pod": "ds-cts-0" 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG [...] 21:50:54 DEBUG [...] 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG }, 21:50:54 DEBUG [...] 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG [...] 21:50:54 DEBUG 21:50:54 DEBUG [...] 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG [...] 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG "value": [ 21:50:54 DEBUG 21:50:54 DEBUG [...] 21:50:54 DEBUG "pod": "ds-cts-2" 21:50:54 DEBUG [...] 21:50:54 DEBUG [...] 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG [...] 21:50:54 DEBUG [...] 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG [...] 21:50:54 DEBUG 21:50:54 DEBUG [...] 21:50:54 DEBUG 21:50:54 DEBUG [...] 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG }, 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG "pod": "am-767f6b67b5-vbltz" 21:50:54 DEBUG }, 21:50:54 DEBUG [...] 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG [...] 21:50:54 DEBUG 21:50:54 DEBUG [...] 21:50:54 DEBUG 21:50:54 DEBUG "instance": "10.142.0.43:9100" 21:50:54 DEBUG 21:50:54 DEBUG "pod": "ds-cts-2" 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG "value": [ 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG "instance": "10.142.0.43:9100" 21:50:54 DEBUG "pod": "am-767f6b67b5-vbltz" 21:50:54 DEBUG }, 21:50:54 DEBUG "value": [ 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG "instance": "10.142.0.43:9100" 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG }, 21:50:54 DEBUG "value": [ 21:50:54 DEBUG 1713646254, 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG }, 21:50:54 DEBUG "value": [ 21:50:54 DEBUG 1713646254, 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 21:50:54 DEBUG }, 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG 21:50:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 21:50:54 DEBUG }, 21:50:54 DEBUG 21:50:54 DEBUG "0.4908666666666666" 21:50:54 DEBUG "pod": "am-767f6b67b5-vbltz" 21:50:54 DEBUG 1713646254, 21:50:54 DEBUG [...] 21:50:54 DEBUG "pod": "ds-idrepo-2" 21:50:54 DEBUG "pod": "am-767f6b67b5-vbltz" 21:50:54 DEBUG "pod": "ds-idrepo-2" 21:50:54 DEBUG "value": [ 21:50:54 DEBUG "599754" 21:50:54 DEBUG 1713646254, 21:50:54 DEBUG "pod": "am-767f6b67b5-vbltz" 21:50:54 DEBUG "pod": "am-767f6b67b5-vbltz" 21:50:54 DEBUG 21:50:54 DEBUG "pod": "am-767f6b67b5-vbltz" 21:50:54 DEBUG "pod": "am-767f6b67b5-vbltz" 21:50:54 DEBUG }, 21:50:54 DEBUG "value": [ 21:50:54 DEBUG 21:50:54 DEBUG "instance": "10.142.0.43:9100" 21:50:54 DEBUG 21:50:54 DEBUG }, 21:50:54 DEBUG "value": [ 21:50:54 DEBUG "pod": "ds-idrepo-2" 21:50:54 DEBUG ] 21:50:54 DEBUG }, 21:50:54 DEBUG "0" 21:50:54 DEBUG 21:50:54 DEBUG }, 21:50:54 DEBUG }, 21:50:54 DEBUG }, 21:50:54 DEBUG "value": [ 21:50:54 DEBUG ] 21:50:54 DEBUG "3346.5" 21:50:54 DEBUG ] 21:50:54 DEBUG }, 21:50:54 DEBUG "pod": "am-767f6b67b5-vbltz" 21:50:54 DEBUG }, 21:50:54 DEBUG }, 21:50:54 DEBUG "value": [ 21:50:54 DEBUG 1713646254, 21:50:54 DEBUG "pod": "am-767f6b67b5-vbltz" 21:50:54 DEBUG }, 21:50:54 DEBUG "instance": "10.142.0.43:9100" 21:50:54 DEBUG "value": [ 21:50:54 DEBUG 1713646254, 21:50:54 DEBUG }, 21:50:54 DEBUG } 21:50:54 DEBUG "value": [ 21:50:54 DEBUG ] 21:50:54 DEBUG 21:50:54 DEBUG "value": [ 21:50:54 DEBUG "value": [ 21:50:54 DEBUG 1713646254, 21:50:54 DEBUG 1713646254, 21:50:54 DEBUG } 21:50:54 DEBUG } 21:50:54 DEBUG }, 21:50:54 DEBUG "value": [ 21:50:54 DEBUG }, 21:50:54 DEBUG "value": [ 21:50:54 DEBUG "value": [ 21:50:54 DEBUG 1713646254, 21:50:54 DEBUG "1013215232" 21:50:54 DEBUG }, 21:50:54 DEBUG "value": [ 21:50:54 DEBUG }, 21:50:54 DEBUG 1713646254, 21:50:54 DEBUG "0.07500000000000946" 21:50:54 DEBUG "value": [ 21:50:54 DEBUG ] 21:50:54 DEBUG 1713646254, 21:50:54 DEBUG } 21:50:54 DEBUG 21:50:54 DEBUG 1713646254, 21:50:54 DEBUG 1713646254, 21:50:54 DEBUG "100149482.80000001" 21:50:54 DEBUG "0" 21:50:54 DEBUG ] 21:50:54 DEBUG ] 21:50:54 DEBUG "value": [ 21:50:54 DEBUG 1713646254, 21:50:54 DEBUG "value": [ 21:50:54 DEBUG 1713646254, 21:50:54 DEBUG 1713646254, 21:50:54 DEBUG "3982526104576" 21:50:54 DEBUG ] 21:50:54 DEBUG "value": [ 21:50:54 DEBUG 1713646254, 21:50:54 DEBUG "value": [ 21:50:54 DEBUG "11.835501249298629" 21:50:54 DEBUG ] 21:50:54 DEBUG 1713646254, 21:50:54 DEBUG } 21:50:54 DEBUG "49.377" 21:50:54 DEBUG ] 21:50:54 DEBUG } 21:50:54 DEBUG "0" 21:50:54 DEBUG "3324.4666666666667" 21:50:54 DEBUG ] 21:50:54 DEBUG ] 21:50:54 DEBUG } 21:50:54 DEBUG } 21:50:54 DEBUG } 21:50:54 DEBUG "3323.9333333333334" 21:50:54 DEBUG 1713646254, 21:50:54 DEBUG "3323.233333333333" 21:50:54 DEBUG "898.3687325201533" 21:50:54 DEBUG ] 21:50:54 DEBUG } 21:50:54 DEBUG 1713646254, 21:50:54 DEBUG "100145083.36666667" 21:50:54 DEBUG 1713646254, 21:50:54 DEBUG "0.40306666666673246" 21:50:54 DEBUG ] 21:50:54 DEBUG } 21:50:54 DEBUG } 21:50:54 DEBUG ] 21:50:54 DEBUG "pod": "ds-cts-2" 21:50:54 DEBUG } 21:50:54 DEBUG ] 21:50:54 DEBUG ] 21:50:54 DEBUG } 21:50:54 DEBUG } 21:50:54 DEBUG } 21:50:54 DEBUG 1713646254, 21:50:54 DEBUG ] 21:50:54 DEBUG "0" 21:50:54 DEBUG ] 21:50:54 DEBUG ] 21:50:54 DEBUG } 21:50:54 DEBUG ] 21:50:54 DEBUG "0" 21:50:54 DEBUG ] 21:50:54 DEBUG ] 21:50:54 DEBUG ] 21:50:54 DEBUG } 21:50:54 DEBUG } 21:50:54 DEBUG ] 21:50:54 DEBUG "0" 21:50:54 DEBUG }, 21:50:54 DEBUG } 21:50:54 DEBUG } 21:50:54 DEBUG ] 21:50:54 DEBUG ] 21:50:54 DEBUG "9.458198627347366" 21:50:54 DEBUG } 21:50:54 DEBUG ] 21:50:54 DEBUG } 21:50:54 DEBUG } 21:50:54 DEBUG ] 21:50:54 DEBUG } 21:50:54 DEBUG ] 21:50:54 DEBUG } 21:50:54 DEBUG } 21:50:54 DEBUG } 21:50:54 DEBUG } 21:50:54 DEBUG } 21:50:54 DEBUG ] 21:50:54 DEBUG "value": [ 21:50:54 DEBUG ] 21:50:54 DEBUG ] 21:50:54 DEBUG } 21:50:54 DEBUG } 21:50:54 DEBUG ] 21:50:54 DEBUG ] 21:50:54 DEBUG } 21:50:54 DEBUG ] 21:50:54 DEBUG } 21:50:54 DEBUG } 21:50:54 DEBUG } 21:50:54 DEBUG } 21:50:54 DEBUG ] 21:50:54 DEBUG ] 21:50:54 DEBUG } 21:50:54 DEBUG ] 21:50:54 DEBUG } 21:50:54 DEBUG ] 21:50:54 DEBUG } 21:50:54 DEBUG } 21:50:54 DEBUG } 21:50:54 DEBUG } 21:50:54 DEBUG } 21:50:54 DEBUG } 21:50:54 DEBUG ] 21:50:54 DEBUG } 21:50:54 DEBUG ] 21:50:54 DEBUG } 21:50:54 DEBUG ] 21:50:54 DEBUG } 21:50:54 DEBUG } 21:50:54 DEBUG } 21:50:54 DEBUG 1713646254, 21:50:54 DEBUG "0" 21:50:54 DEBUG } 21:50:54 DEBUG } 21:50:54 DEBUG ] 21:50:54 DEBUG } 21:50:54 DEBUG } 21:50:54 DEBUG } 21:50:54 DEBUG } 21:50:54 DEBUG } 21:50:54 DEBUG } 21:50:54 DEBUG } 21:50:54 DEBUG ] 21:50:54 DEBUG } 21:50:54 DEBUG } 21:50:54 DEBUG } 21:50:54 DEBUG } 21:50:54 DEBUG } 21:50:54 DEBUG } 21:50:54 DEBUG } 21:50:54 DEBUG } 21:50:54 DEBUG ] 21:50:54 DEBUG } 21:50:54 DEBUG } 21:51:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646313" 21:51:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713646313" 21:51:53 INFO [http_cmd]: http status code OK 21:51:53 DEBUG --- status code --- 21:51:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:51:53 DEBUG --- http response --- 21:51:53 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 21:51:53 ERROR [http_cmd]: ERROR 21:51:53 ERROR --- status code --- 21:51:53 ERROR http status code is 400 (expected [200, 401, 403, 404]) 21:51:53 ERROR --- http response --- 21:51:53 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 21:51:53 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 21:51:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646313" 21:51:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713646313" 21:51:53 INFO [http_cmd]: http status code OK 21:51:53 DEBUG --- status code --- 21:51:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:51:53 DEBUG --- http response --- 21:51:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 21:51:53 DEBUG ----- output ----- 21:51:53 DEBUG { 21:51:53 DEBUG "status": "success", 21:51:53 DEBUG "data": { 21:51:53 DEBUG "resultType": "vector", 21:51:53 DEBUG "result": [ 21:51:53 DEBUG { 21:51:53 DEBUG "metric": { 21:51:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:51:53 DEBUG }, 21:51:53 DEBUG "value": [ 21:51:53 DEBUG 21:51:53 DEBUG 21:51:53 DEBUG 21:51:53 DEBUG [...] 21:51:53 DEBUG 21:51:53 DEBUG 21:51:53 DEBUG 21:51:53 DEBUG "pod": "ds-cts-0" 21:51:53 DEBUG }, 21:51:53 DEBUG "value": [ 21:51:53 DEBUG 1713646313, 21:51:53 DEBUG "4337.359763043973" 21:51:53 DEBUG ] 21:51:53 INFO [http_cmd]: http status code OK 21:51:53 DEBUG } 21:51:53 DEBUG --- status code --- 21:51:53 DEBUG ] 21:51:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:51:53 DEBUG } 21:51:53 DEBUG --- http response --- 21:51:53 DEBUG } 21:51:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:51:53 DEBUG ----- output ----- 21:51:53 DEBUG { 21:51:53 DEBUG "status": "success", 21:51:53 DEBUG "data": { 21:51:53 DEBUG "resultType": "vector", 21:51:53 DEBUG "result": [ 21:51:53 DEBUG { 21:51:53 DEBUG "metric": { 21:51:53 DEBUG "pod": "ds-idrepo-0" 21:51:53 DEBUG }, 21:51:53 DEBUG "value": [ 21:51:53 DEBUG 21:51:53 DEBUG 21:51:53 DEBUG 21:51:53 DEBUG [...] 21:51:53 DEBUG 21:51:53 DEBUG 21:51:53 DEBUG 21:51:53 DEBUG "pod": "ds-cts-2" 21:51:53 DEBUG }, 21:51:53 DEBUG "value": [ 21:51:53 DEBUG 1713646313, 21:51:53 DEBUG "0" 21:51:53 DEBUG ] 21:51:53 DEBUG } 21:51:53 DEBUG ] 21:51:53 DEBUG } 21:51:53 DEBUG } 21:51:54 INFO 21:51:54 INFO [loop_until]: kubectl --namespace=xlou top pods 21:51:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:51:54 INFO [loop_until]: OK (rc = 0) 21:51:54 DEBUG --- stdout --- 21:51:54 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 22314m 8764Mi am-767f6b67b5-qcqz2 22319m 8881Mi am-767f6b67b5-vbltz 24602m 8938Mi ds-cts-0 19923m 12060Mi ds-cts-1 19629m 12068Mi ds-cts-2 20944m 12262Mi ds-idrepo-0 9509m 3891Mi ds-idrepo-1 9204m 3975Mi ds-idrepo-2 9524m 3875Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-7dd48676f6-tdxl9 12m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-967d957f7-zk66v 11927m 877Mi 21:51:54 DEBUG --- stderr --- 21:51:54 DEBUG 21:51:54 INFO 21:51:54 INFO [loop_until]: kubectl --namespace=xlou top node 21:51:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646314" 21:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646314" 21:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646314" 21:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646314" 21:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713646314" 21:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646314" 21:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646314" 21:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646314" 21:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646314" 21:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646314" 21:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646314" 21:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646314" 21:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713646314" 21:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713646314" 21:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646314" 21:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646314" 21:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646314" 21:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646314" 21:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646314" 21:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646314" 21:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646314" 21:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646314" 21:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646314" 21:51:54 INFO [http_cmd]: http status code OK 21:51:54 INFO [http_cmd]: http status code OK 21:51:54 INFO [http_cmd]: http status code OK 21:51:54 INFO [loop_until]: OK (rc = 0) 21:51:54 INFO [http_cmd]: http status code OK 21:51:54 INFO [http_cmd]: http status code OK 21:51:54 INFO [http_cmd]: http status code OK 21:51:54 INFO [http_cmd]: http status code OK 21:51:54 INFO [http_cmd]: http status code OK 21:51:54 INFO [http_cmd]: http status code OK 21:51:54 INFO [http_cmd]: http status code OK 21:51:54 INFO [http_cmd]: http status code OK 21:51:54 INFO [http_cmd]: http status code OK 21:51:54 INFO [http_cmd]: http status code OK 21:51:54 DEBUG --- status code --- 21:51:54 INFO [http_cmd]: http status code OK 21:51:54 INFO [http_cmd]: http status code OK 21:51:54 DEBUG --- status code --- 21:51:54 INFO [http_cmd]: http status code OK 21:51:54 INFO [http_cmd]: http status code OK 21:51:54 INFO [http_cmd]: http status code OK 21:51:54 DEBUG --- status code --- 21:51:54 DEBUG --- stdout --- 21:51:54 DEBUG --- status code --- 21:51:54 INFO [http_cmd]: http status code OK 21:51:54 INFO [http_cmd]: http status code OK 21:51:54 INFO [http_cmd]: http status code OK 21:51:54 INFO [http_cmd]: http status code OK 21:51:54 DEBUG --- status code --- 21:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:51:54 DEBUG --- status code --- 21:51:54 DEBUG --- status code --- 21:51:54 DEBUG --- status code --- 21:51:54 DEBUG --- status code --- 21:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:51:54 DEBUG --- http response --- 21:51:54 INFO [http_cmd]: http status code OK 21:51:54 DEBUG --- status code --- 21:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:51:54 DEBUG --- status code --- 21:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:51:54 DEBUG --- status code --- 21:51:54 DEBUG --- status code --- 21:51:54 DEBUG --- status code --- 21:51:54 DEBUG --- status code --- 21:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:51:54 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 60m 0% 6216Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 9981m 33% 7651Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 21776m 72% 11478Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 90m 0% 3379Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 19784m 66% 15261Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 160m 0% 5200Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 22240m 74% 11779Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19727m 66% 3056Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 24744m 82% 11188Mi 10% gke-xlou-cdm-ds-0f601751-j488 10300m 34% 7292Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19365m 64% 17139Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 9832m 32% 7950Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20261m 67% 14963Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 11696m 73% 8811Mi 15% 21:51:54 DEBUG --- stderr --- 21:51:54 DEBUG --- status code --- 21:51:54 DEBUG --- status code --- 21:51:54 DEBUG --- status code --- 21:51:54 DEBUG --- status code --- 21:51:54 DEBUG --- status code --- 21:51:54 DEBUG --- http response --- 21:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:51:54 DEBUG --- http response --- 21:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:51:54 DEBUG --- status code --- 21:51:54 DEBUG --- status code --- 21:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:51:54 DEBUG --- status code --- 21:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:51:54 DEBUG --- http response --- 21:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:51:54 DEBUG --- http response --- 21:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:51:54 DEBUG --- http response --- 21:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:51:54 DEBUG --- http response --- 21:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:51:54 DEBUG 21:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:51:54 DEBUG ----- output ----- 21:51:54 DEBUG { 21:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:51:54 DEBUG --- http response --- 21:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:51:54 DEBUG --- http response --- 21:51:54 DEBUG --- http response --- 21:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:51:54 DEBUG ----- output ----- 21:51:54 DEBUG --- http response --- 21:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:51:54 DEBUG --- http response --- 21:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:51:54 DEBUG --- http response --- 21:51:54 DEBUG --- http response --- 21:51:54 DEBUG --- http response --- 21:51:54 DEBUG --- http response --- 21:51:54 DEBUG --- http response --- 21:51:54 DEBUG --- http response --- 21:51:54 DEBUG ----- output ----- 21:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:51:54 DEBUG --- http response --- 21:51:54 DEBUG "status": "success", 21:51:54 DEBUG --- http response --- 21:51:54 DEBUG ----- output ----- 21:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:51:54 DEBUG --- http response --- 21:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:51:54 DEBUG { 21:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:51:54 DEBUG --- http response --- 21:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:51:54 DEBUG ----- output ----- 21:51:54 DEBUG { 21:51:54 DEBUG ----- output ----- 21:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:51:54 DEBUG ----- output ----- 21:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:51:54 DEBUG { 21:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:51:54 DEBUG --- http response --- 21:51:54 DEBUG "data": { 21:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:51:54 DEBUG { 21:51:54 DEBUG ----- output ----- 21:51:54 DEBUG ----- output ----- 21:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:51:54 DEBUG ----- output ----- 21:51:54 DEBUG "status": "success", 21:51:54 DEBUG "data": { 21:51:54 DEBUG "resultType": "vector", 21:51:54 DEBUG ----- output ----- 21:51:54 DEBUG "status": "success", 21:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:51:54 DEBUG { 21:51:54 DEBUG { 21:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:51:54 DEBUG ----- output ----- 21:51:54 DEBUG ----- output ----- 21:51:54 DEBUG ----- output ----- 21:51:54 DEBUG "status": "success", 21:51:54 DEBUG ----- output ----- 21:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:51:54 DEBUG "resultType": "vector", 21:51:54 DEBUG ----- output ----- 21:51:54 DEBUG ----- output ----- 21:51:54 DEBUG "status": "success", 21:51:54 DEBUG { 21:51:54 DEBUG { 21:51:54 DEBUG ----- output ----- 21:51:54 DEBUG { 21:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 21:51:54 DEBUG ----- output ----- 21:51:54 DEBUG "result": [ 21:51:54 DEBUG { 21:51:54 DEBUG "status": "success", 21:51:54 DEBUG ----- output ----- 21:51:54 DEBUG { 21:51:54 DEBUG "status": "success", 21:51:54 DEBUG ----- output ----- 21:51:54 DEBUG { 21:51:54 DEBUG { 21:51:54 DEBUG { 21:51:54 DEBUG "data": { 21:51:54 DEBUG { 21:51:54 DEBUG ----- output ----- 21:51:54 DEBUG "result": [ 21:51:54 DEBUG { 21:51:54 DEBUG { 21:51:54 DEBUG "data": { 21:51:54 DEBUG "status": "success", 21:51:54 DEBUG "status": "success", 21:51:54 DEBUG { 21:51:54 DEBUG "status": "success", 21:51:54 DEBUG ----- output ----- 21:51:54 DEBUG { 21:51:54 DEBUG { 21:51:54 DEBUG "data": { 21:51:54 DEBUG "data": { 21:51:54 DEBUG "status": "success", 21:51:54 DEBUG "data": { 21:51:54 DEBUG "data": { 21:51:54 DEBUG "status": "success", 21:51:54 DEBUG { 21:51:54 DEBUG "status": "success", 21:51:54 DEBUG "status": "success", 21:51:54 DEBUG "resultType": "vector", 21:51:54 DEBUG "status": "success", 21:51:54 DEBUG { 21:51:54 DEBUG { 21:51:54 DEBUG "metric": { 21:51:54 DEBUG "status": "success", 21:51:54 DEBUG "resultType": "vector", 21:51:54 DEBUG "data": { 21:51:54 DEBUG "resultType": "vector", 21:51:54 DEBUG "status": "success", 21:51:54 DEBUG "data": { 21:51:54 DEBUG { 21:51:54 DEBUG "status": "success", 21:51:54 DEBUG "metric": { 21:51:54 DEBUG "resultType": "vector", 21:51:54 DEBUG "resultType": "vector", 21:51:54 DEBUG "status": "success", 21:51:54 DEBUG "resultType": "vector", 21:51:54 DEBUG "resultType": "vector", 21:51:54 DEBUG "data": { 21:51:54 DEBUG "status": "success", 21:51:54 DEBUG "data": { 21:51:54 DEBUG "data": { 21:51:54 DEBUG "result": [ 21:51:54 DEBUG "data": { 21:51:54 DEBUG "status": "success", 21:51:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 21:51:54 DEBUG "data": { 21:51:54 DEBUG "result": [ 21:51:54 DEBUG "data": { 21:51:54 DEBUG "result": [ 21:51:54 DEBUG { 21:51:54 DEBUG "metric": { 21:51:54 DEBUG "data": { 21:51:54 DEBUG "status": "success", 21:51:54 DEBUG "data": { 21:51:54 DEBUG "pod": "am-767f6b67b5-9zw22" 21:51:54 DEBUG "result": [ 21:51:54 DEBUG "result": [ 21:51:54 DEBUG "data": { 21:51:54 DEBUG "result": [ 21:51:54 DEBUG "result": [ 21:51:54 DEBUG "resultType": "vector", 21:51:54 DEBUG "data": { 21:51:54 DEBUG "resultType": "vector", 21:51:54 DEBUG "resultType": "vector", 21:51:54 DEBUG { 21:51:54 DEBUG "resultType": "vector", 21:51:54 DEBUG "data": { 21:51:54 DEBUG }, 21:51:54 DEBUG "resultType": "vector", 21:51:54 DEBUG { 21:51:54 DEBUG "resultType": "vector", 21:51:54 DEBUG "resultType": "vector", 21:51:54 DEBUG "pod": "am-767f6b67b5-9zw22" 21:51:54 DEBUG "resultType": "vector", 21:51:54 DEBUG "data": { 21:51:54 DEBUG "resultType": "vector", 21:51:54 DEBUG "resultType": "vector", 21:51:54 DEBUG }, 21:51:54 DEBUG { 21:51:54 DEBUG { 21:51:54 DEBUG "resultType": "vector", 21:51:54 DEBUG { 21:51:54 DEBUG "metric": { 21:51:54 DEBUG "pod": "am-767f6b67b5-9zw22" 21:51:54 DEBUG "result": [ 21:51:54 DEBUG "result": [ 21:51:54 DEBUG { 21:51:54 DEBUG "metric": { 21:51:54 DEBUG "result": [ 21:51:54 DEBUG "resultType": "vector", 21:51:54 DEBUG "value": [ 21:51:54 DEBUG "result": [ 21:51:54 DEBUG { 21:51:54 DEBUG "result": [ 21:51:54 DEBUG "result": [ 21:51:54 DEBUG }, 21:51:54 DEBUG "result": [ 21:51:54 DEBUG "status": "success", 21:51:54 DEBUG "result": [ 21:51:54 DEBUG "result": [ 21:51:54 DEBUG "value": [ 21:51:54 DEBUG "metric": { 21:51:54 DEBUG "metric": { 21:51:54 DEBUG "result": [ 21:51:54 DEBUG "resultType": "vector", 21:51:54 DEBUG }, 21:51:54 DEBUG { 21:51:54 DEBUG { 21:51:54 DEBUG "metric": { 21:51:54 DEBUG "pod": "am-767f6b67b5-9zw22" 21:51:54 DEBUG }, 21:51:54 DEBUG "result": [ 21:51:54 DEBUG "result": [ 21:51:54 DEBUG 21:51:54 DEBUG "metric": { 21:51:54 DEBUG "metric": { 21:51:54 DEBUG { 21:51:54 DEBUG { 21:51:54 DEBUG "value": [ 21:51:54 DEBUG { 21:51:54 DEBUG "data": { 21:51:54 DEBUG { 21:51:54 DEBUG { 21:51:54 DEBUG 21:51:54 DEBUG "pod": "ds-cts-0" 21:51:54 DEBUG "pod": "am-767f6b67b5-9zw22" 21:51:54 DEBUG { 21:51:54 DEBUG "result": [ 21:51:54 DEBUG "value": [ 21:51:54 DEBUG "metric": { 21:51:54 DEBUG "metric": { 21:51:54 DEBUG "instance": "10.142.0.107:9100" 21:51:54 DEBUG "value": [ 21:51:54 DEBUG { 21:51:54 DEBUG { 21:51:54 DEBUG { 21:51:54 DEBUG 21:51:54 DEBUG "pod": "ds-idrepo-0" 21:51:54 DEBUG "instance": "10.142.0.107:9100" 21:51:54 DEBUG "metric": { 21:51:54 DEBUG "metric": { 21:51:54 DEBUG 21:51:54 DEBUG "metric": { 21:51:54 DEBUG "resultType": "vector", 21:51:54 DEBUG "metric": { 21:51:54 DEBUG "metric": { 21:51:54 DEBUG 21:51:54 DEBUG }, 21:51:54 DEBUG }, 21:51:54 DEBUG "metric": { 21:51:54 DEBUG { 21:51:54 DEBUG "metric": { 21:51:54 DEBUG "pod": "ds-cts-0" 21:51:54 DEBUG }, 21:51:54 DEBUG "value": [ 21:51:54 DEBUG 21:51:54 DEBUG "metric": { 21:51:54 DEBUG "metric": { 21:51:54 DEBUG "metric": { 21:51:54 DEBUG 21:51:54 DEBUG }, 21:51:54 DEBUG }, 21:51:54 DEBUG "pod": "ds-cts-0" 21:51:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 21:51:54 DEBUG 21:51:54 DEBUG "pod": "am-767f6b67b5-9zw22" 21:51:54 DEBUG "result": [ 21:51:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:51:54 DEBUG "instance": "10.142.0.107:9100" 21:51:54 DEBUG 21:51:54 DEBUG "value": [ 21:51:54 DEBUG "value": [ 21:51:54 DEBUG "pod": "am-767f6b67b5-9zw22" 21:51:54 DEBUG 21:51:54 DEBUG "pod": "ds-idrepo-0" 21:51:54 DEBUG 21:51:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:51:54 DEBUG 21:51:54 DEBUG "pod": "ds-idrepo-0" 21:51:54 DEBUG }, 21:51:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:51:54 DEBUG "instance": "10.142.0.107:9100" 21:51:54 DEBUG [...] 21:51:54 DEBUG "value": [ 21:51:54 DEBUG "value": [ 21:51:54 DEBUG }, 21:51:54 DEBUG }, 21:51:54 DEBUG 21:51:54 DEBUG }, 21:51:54 DEBUG { 21:51:54 DEBUG }, 21:51:54 DEBUG }, 21:51:54 DEBUG [...] 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG }, 21:51:54 DEBUG 21:51:54 DEBUG }, 21:51:54 DEBUG 21:51:54 DEBUG }, 21:51:54 DEBUG 21:51:54 DEBUG [...] 21:51:54 DEBUG "value": [ 21:51:54 DEBUG }, 21:51:54 DEBUG }, 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG "value": [ 21:51:54 DEBUG 21:51:54 DEBUG [...] 21:51:54 DEBUG "value": [ 21:51:54 DEBUG "metric": { 21:51:54 DEBUG "value": [ 21:51:54 DEBUG "value": [ 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG "value": [ 21:51:54 DEBUG 21:51:54 DEBUG "value": [ 21:51:54 DEBUG 21:51:54 DEBUG "value": [ 21:51:54 DEBUG 21:51:54 DEBUG }, 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG [...] 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG "value": [ 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG "instance": "10.142.0.107:9100" 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG [...] 21:51:54 DEBUG 21:51:54 DEBUG [...] 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG "value": [ 21:51:54 DEBUG "value": [ 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG "value": [ 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG }, 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG [...] 21:51:54 DEBUG [...] 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG [...] 21:51:54 DEBUG [...] 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG "pod": "am-767f6b67b5-vbltz" 21:51:54 DEBUG "value": [ 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG [...] 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG "pod": "am-767f6b67b5-vbltz" 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG [...] 21:51:54 DEBUG [...] 21:51:54 DEBUG }, 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG [...] 21:51:54 DEBUG 21:51:54 DEBUG "pod": "am-767f6b67b5-vbltz" 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG "pod": "am-767f6b67b5-vbltz" 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG [...] 21:51:54 DEBUG }, 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG [...] 21:51:54 DEBUG }, 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG "value": [ 21:51:54 DEBUG 21:51:54 DEBUG [...] 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG }, 21:51:54 DEBUG 21:51:54 DEBUG "pod": "am-767f6b67b5-vbltz" 21:51:54 DEBUG }, 21:51:54 DEBUG [...] 21:51:54 DEBUG "pod": "ds-cts-2" 21:51:54 DEBUG [...] 21:51:54 DEBUG 21:51:54 DEBUG "value": [ 21:51:54 DEBUG [...] 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG "value": [ 21:51:54 DEBUG [...] 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG 1713646314, 21:51:54 DEBUG "0" 21:51:54 DEBUG ] 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG "value": [ 21:51:54 DEBUG "pod": "ds-cts-2" 21:51:54 DEBUG }, 21:51:54 DEBUG "value": [ 21:51:54 DEBUG 21:51:54 DEBUG }, 21:51:54 DEBUG 21:51:54 DEBUG 1713646314, 21:51:54 DEBUG 21:51:54 DEBUG "instance": "10.142.0.43:9100" 21:51:54 DEBUG 1713646314, 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG "pod": "ds-idrepo-2" 21:51:54 DEBUG "instance": "10.142.0.43:9100" 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG } 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG "pod": "am-767f6b67b5-vbltz" 21:51:54 DEBUG 1713646314, 21:51:54 DEBUG }, 21:51:54 DEBUG "value": [ 21:51:54 DEBUG 1713646314, 21:51:54 DEBUG 21:51:54 DEBUG "value": [ 21:51:54 DEBUG 1713646314, 21:51:54 DEBUG "3348.233333333333" 21:51:54 DEBUG "pod": "ds-cts-0" 21:51:54 DEBUG }, 21:51:54 DEBUG "value": [ 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG }, 21:51:54 DEBUG }, 21:51:54 DEBUG "pod": "ds-cts-2" 21:51:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 21:51:54 DEBUG ] 21:51:54 DEBUG [...] 21:51:54 DEBUG 21:51:54 DEBUG "pod": "ds-cts-0" 21:51:54 DEBUG }, 21:51:54 DEBUG "599675.5" 21:51:54 DEBUG "value": [ 21:51:54 DEBUG 1713646314, 21:51:54 DEBUG "3319.5" 21:51:54 DEBUG 21:51:54 DEBUG "0" 21:51:54 DEBUG 21:51:54 DEBUG ] 21:51:54 DEBUG } 21:51:54 DEBUG 1713646314, 21:51:54 DEBUG "1015083008" 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG "value": [ 21:51:54 DEBUG "value": [ 21:51:54 DEBUG }, 21:51:54 DEBUG }, 21:51:54 DEBUG } 21:51:54 DEBUG 21:51:54 DEBUG 21:51:54 DEBUG }, 21:51:54 DEBUG "value": [ 21:51:54 DEBUG ] 21:51:54 DEBUG 1713646314, 21:51:54 DEBUG "3319.866666666667" 21:51:54 DEBUG ] 21:51:54 DEBUG "pod": "ds-idrepo-2" 21:51:54 DEBUG ] 21:51:54 DEBUG "pod": "am-767f6b67b5-vbltz" 21:51:54 DEBUG ] 21:51:54 DEBUG "0.0803333333333133" 21:51:54 DEBUG ] 21:51:54 DEBUG "instance": "10.142.0.43:9100" 21:51:54 DEBUG }, 21:51:54 DEBUG "pod": "ds-cts-0" 21:51:54 DEBUG "pod": "ds-idrepo-2" 21:51:54 DEBUG 1713646314, 21:51:54 DEBUG 1713646314, 21:51:54 DEBUG "value": [ 21:51:54 DEBUG 1713646314, 21:51:54 DEBUG } 21:51:54 DEBUG 21:51:54 DEBUG "instance": "10.142.0.43:9100" 21:51:54 DEBUG }, 21:51:54 DEBUG "value": [ 21:51:54 DEBUG } 21:51:54 DEBUG "0.5499333333333329" 21:51:54 DEBUG ] 21:51:54 DEBUG } 21:51:54 DEBUG ] 21:51:54 DEBUG } 21:51:54 DEBUG }, 21:51:54 DEBUG } 21:51:54 DEBUG ] 21:51:54 DEBUG } 21:51:54 DEBUG }, 21:51:54 DEBUG "value": [ 21:51:54 DEBUG }, 21:51:54 DEBUG }, 21:51:54 DEBUG "0" 21:51:54 DEBUG "100285765" 21:51:54 DEBUG "0" 21:51:54 DEBUG "value": [ 21:51:54 DEBUG 21:51:54 DEBUG "value": [ 21:51:54 DEBUG 1713646314, 21:51:54 DEBUG 1713646314, 21:51:54 DEBUG ] 21:51:54 DEBUG ] 21:51:54 DEBUG } 21:51:54 DEBUG }, 21:51:54 DEBUG } 21:51:54 DEBUG ] 21:51:54 DEBUG "value": [ 21:51:54 DEBUG } 21:51:54 DEBUG } 21:51:54 DEBUG ] 21:51:54 DEBUG "value": [ 21:51:54 DEBUG 1713646314, 21:51:54 DEBUG "value": [ 21:51:54 DEBUG "value": [ 21:51:54 DEBUG ] 21:51:54 DEBUG ] 21:51:54 DEBUG ] 21:51:54 DEBUG 1713646314, 21:51:54 DEBUG "instance": "10.142.0.43:9100" 21:51:54 DEBUG 1713646314, 21:51:54 DEBUG "49.377" 21:51:54 DEBUG "100281773.06666666" 21:51:54 DEBUG } 21:51:54 DEBUG } 21:51:54 DEBUG ] 21:51:54 DEBUG "value": [ 21:51:54 DEBUG } 21:51:54 DEBUG } 21:51:54 DEBUG 1713646314, 21:51:54 DEBUG ] 21:51:54 DEBUG } 21:51:54 DEBUG 1713646314, 21:51:54 DEBUG "44305895.70945545" 21:51:54 DEBUG 1713646314, 21:51:54 DEBUG 1713646314, 21:51:54 DEBUG } 21:51:54 DEBUG } 21:51:54 DEBUG } 21:51:54 DEBUG "11.852927207249644" 21:51:54 DEBUG }, 21:51:54 DEBUG "11.895600684923366" 21:51:54 DEBUG ] 21:51:54 DEBUG ] 21:51:54 DEBUG } 21:51:54 DEBUG ] 21:51:54 DEBUG } 21:51:54 DEBUG 1713646314, 21:51:54 DEBUG } 21:51:54 DEBUG "3327.0333333333333" 21:51:54 DEBUG } 21:51:54 DEBUG } 21:51:54 DEBUG "0.3958666666665711" 21:51:54 DEBUG ] 21:51:54 DEBUG "51150823.213670544" 21:51:54 DEBUG "0" 21:51:54 DEBUG ] 21:51:54 DEBUG ] 21:51:54 DEBUG ] 21:51:54 DEBUG ] 21:51:54 DEBUG "value": [ 21:51:54 DEBUG ] 21:51:54 DEBUG } 21:51:54 DEBUG } 21:51:54 DEBUG } 21:51:54 DEBUG } 21:51:54 DEBUG "0" 21:51:54 DEBUG ] 21:51:54 DEBUG } 21:51:54 DEBUG ] 21:51:54 DEBUG } 21:51:54 DEBUG ] 21:51:54 DEBUG ] 21:51:54 DEBUG } 21:51:54 DEBUG } 21:51:54 DEBUG } 21:51:54 DEBUG } 21:51:54 DEBUG 1713646314, 21:51:54 DEBUG } 21:51:54 DEBUG ] 21:51:54 DEBUG ] 21:51:54 DEBUG } 21:51:54 DEBUG ] 21:51:54 DEBUG } 21:51:54 DEBUG } 21:51:54 DEBUG } 21:51:54 DEBUG ] 21:51:54 DEBUG } 21:51:54 DEBUG } 21:51:54 DEBUG } 21:51:54 DEBUG } 21:51:54 DEBUG ] 21:51:54 DEBUG "3987737461760" 21:51:54 DEBUG ] 21:51:54 DEBUG } 21:51:54 DEBUG } 21:51:54 DEBUG } 21:51:54 DEBUG ] 21:51:54 DEBUG ] 21:51:54 DEBUG } 21:51:54 DEBUG } 21:51:54 DEBUG ] 21:51:54 DEBUG } 21:51:54 DEBUG ] 21:51:54 DEBUG } 21:51:54 DEBUG } 21:51:54 DEBUG } 21:51:54 DEBUG ] 21:51:54 DEBUG } 21:51:54 DEBUG } 21:51:54 DEBUG ] 21:51:54 DEBUG } 21:51:54 DEBUG } 21:51:54 DEBUG } 21:51:54 DEBUG } 21:51:54 DEBUG } 21:51:54 DEBUG } 21:51:54 DEBUG } 21:51:54 DEBUG } 21:51:54 DEBUG ] 21:51:54 DEBUG } 21:51:54 DEBUG } 21:51:54 DEBUG } 21:51:54 DEBUG } 21:51:54 DEBUG } 21:52:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646373" 21:52:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713646373" 21:52:53 ERROR [http_cmd]: ERROR 21:52:53 ERROR --- status code --- 21:52:53 ERROR http status code is 400 (expected [200, 401, 403, 404]) 21:52:53 ERROR --- http response --- 21:52:53 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 21:52:53 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 21:52:53 INFO [http_cmd]: http status code OK 21:52:53 DEBUG --- status code --- 21:52:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:52:53 DEBUG --- http response --- 21:52:53 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 21:52:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646373" 21:52:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713646373" 21:52:53 INFO [http_cmd]: http status code OK 21:52:53 INFO [http_cmd]: http status code OK 21:52:53 DEBUG --- status code --- 21:52:53 DEBUG --- status code --- 21:52:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:52:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:52:53 DEBUG --- http response --- 21:52:53 DEBUG --- http response --- 21:52:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:52:53 DEBUG ----- output ----- 21:52:53 DEBUG { 21:52:53 DEBUG "status": "success", 21:52:53 DEBUG "data": { 21:52:53 DEBUG "resultType": "vector", 21:52:53 DEBUG "result": [ 21:52:53 DEBUG { 21:52:53 DEBUG "metric": { 21:52:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 21:52:53 DEBUG "pod": "ds-idrepo-0" 21:52:53 DEBUG ----- output ----- 21:52:53 DEBUG }, 21:52:53 DEBUG { 21:52:53 DEBUG "value": [ 21:52:53 DEBUG "status": "success", 21:52:53 DEBUG "data": { 21:52:53 DEBUG "resultType": "vector", 21:52:53 DEBUG "result": [ 21:52:53 DEBUG { 21:52:53 DEBUG 21:52:53 DEBUG "metric": { 21:52:53 DEBUG 21:52:53 DEBUG 21:52:53 DEBUG [...] 21:52:53 DEBUG 21:52:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:52:53 DEBUG 21:52:53 DEBUG }, 21:52:53 DEBUG 21:52:53 DEBUG "value": [ 21:52:53 DEBUG 21:52:53 DEBUG 21:52:53 DEBUG 21:52:53 DEBUG [...] 21:52:53 DEBUG 21:52:53 DEBUG 21:52:53 DEBUG "pod": "ds-cts-2" 21:52:53 DEBUG 21:52:53 DEBUG }, 21:52:53 DEBUG "pod": "ds-cts-0" 21:52:53 DEBUG "value": [ 21:52:53 DEBUG }, 21:52:53 DEBUG 1713646373, 21:52:53 DEBUG "value": [ 21:52:53 DEBUG "0" 21:52:53 DEBUG 1713646373, 21:52:53 DEBUG ] 21:52:53 DEBUG "0" 21:52:53 DEBUG } 21:52:53 DEBUG ] 21:52:53 DEBUG ] 21:52:53 DEBUG } 21:52:53 DEBUG } 21:52:53 DEBUG ] 21:52:53 DEBUG } 21:52:53 DEBUG } 21:52:53 DEBUG } 21:52:54 INFO 21:52:54 INFO [loop_until]: kubectl --namespace=xlou top pods 21:52:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:52:54 INFO [loop_until]: OK (rc = 0) 21:52:54 DEBUG --- stdout --- 21:52:54 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 21971m 8768Mi am-767f6b67b5-qcqz2 21644m 8888Mi am-767f6b67b5-vbltz 24208m 8958Mi ds-cts-0 20288m 12114Mi ds-cts-1 20129m 12120Mi ds-cts-2 20970m 12280Mi ds-idrepo-0 9515m 3896Mi ds-idrepo-1 9603m 3975Mi ds-idrepo-2 9606m 3875Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-7dd48676f6-tdxl9 13m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-967d957f7-zk66v 11657m 882Mi 21:52:54 DEBUG --- stderr --- 21:52:54 DEBUG 21:52:54 INFO 21:52:54 INFO [loop_until]: kubectl --namespace=xlou top node 21:52:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646374" 21:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646374" 21:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646374" 21:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646374" 21:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646374" 21:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713646374" 21:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646374" 21:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646374" 21:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646374" 21:52:54 INFO [http_cmd]: http status code OK 21:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646374" 21:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646374" 21:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646374" 21:52:54 INFO [http_cmd]: http status code OK 21:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646374" 21:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646374" 21:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713646374" 21:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713646374" 21:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646374" 21:52:54 INFO [loop_until]: OK (rc = 0) 21:52:54 DEBUG --- status code --- 21:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646374" 21:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646374" 21:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646374" 21:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646374" 21:52:54 INFO [http_cmd]: http status code OK 21:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646374" 21:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646374" 21:52:54 INFO [http_cmd]: http status code OK 21:52:54 INFO [http_cmd]: http status code OK 21:52:54 INFO [http_cmd]: http status code OK 21:52:54 INFO [http_cmd]: http status code OK 21:52:54 DEBUG --- status code --- 21:52:54 INFO [http_cmd]: http status code OK 21:52:54 INFO [http_cmd]: http status code OK 21:52:54 INFO [http_cmd]: http status code OK 21:52:54 DEBUG --- stdout --- 21:52:54 INFO [http_cmd]: http status code OK 21:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:52:54 INFO [http_cmd]: http status code OK 21:52:54 INFO [http_cmd]: http status code OK 21:52:54 INFO [http_cmd]: http status code OK 21:52:54 DEBUG --- status code --- 21:52:54 INFO [http_cmd]: http status code OK 21:52:54 INFO [http_cmd]: http status code OK 21:52:54 INFO [http_cmd]: http status code OK 21:52:54 DEBUG --- status code --- 21:52:54 DEBUG --- status code --- 21:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:52:54 DEBUG --- status code --- 21:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:52:54 INFO [http_cmd]: http status code OK 21:52:54 INFO [http_cmd]: http status code OK 21:52:54 INFO [http_cmd]: http status code OK 21:52:54 INFO [http_cmd]: http status code OK 21:52:54 DEBUG --- status code --- 21:52:54 DEBUG --- status code --- 21:52:54 DEBUG --- status code --- 21:52:54 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6220Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 10012m 33% 7661Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 21763m 72% 11501Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 84m 0% 3371Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 20308m 68% 15349Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 160m 0% 5238Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 21463m 71% 11787Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19615m 65% 3059Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 24714m 82% 11207Mi 10% gke-xlou-cdm-ds-0f601751-j488 10015m 33% 7287Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 20066m 67% 17183Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 10171m 34% 7953Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20434m 68% 14991Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 12036m 75% 8813Mi 15% 21:52:54 DEBUG --- status code --- 21:52:54 INFO [http_cmd]: http status code OK 21:52:54 DEBUG --- http response --- 21:52:54 INFO [http_cmd]: http status code OK 21:52:54 DEBUG --- status code --- 21:52:54 DEBUG --- status code --- 21:52:54 DEBUG --- status code --- 21:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:52:54 DEBUG --- status code --- 21:52:54 DEBUG --- status code --- 21:52:54 DEBUG --- status code --- 21:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:52:54 DEBUG --- status code --- 21:52:54 DEBUG --- http response --- 21:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:52:54 DEBUG --- http response --- 21:52:54 DEBUG --- status code --- 21:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:52:54 DEBUG --- status code --- 21:52:54 DEBUG --- status code --- 21:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:52:54 DEBUG --- stderr --- 21:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:52:54 DEBUG --- status code --- 21:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:52:54 DEBUG --- status code --- 21:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:52:54 DEBUG --- http response --- 21:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:52:54 DEBUG --- http response --- 21:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:52:54 DEBUG --- http response --- 21:52:54 DEBUG --- status code --- 21:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:52:54 DEBUG --- http response --- 21:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:52:54 DEBUG --- http response --- 21:52:54 DEBUG --- http response --- 21:52:54 DEBUG --- http response --- 21:52:54 DEBUG --- http response --- 21:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:52:54 DEBUG 21:52:54 DEBUG ----- output ----- 21:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:52:54 DEBUG --- http response --- 21:52:54 DEBUG --- http response --- 21:52:54 DEBUG --- http response --- 21:52:54 DEBUG --- http response --- 21:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:52:54 DEBUG --- http response --- 21:52:54 DEBUG --- http response --- 21:52:54 DEBUG --- http response --- 21:52:54 DEBUG ----- output ----- 21:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:52:54 DEBUG ----- output ----- 21:52:54 DEBUG ----- output ----- 21:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:52:54 DEBUG ----- output ----- 21:52:54 DEBUG --- http response --- 21:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:52:54 DEBUG ----- output ----- 21:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:52:54 DEBUG { 21:52:54 DEBUG "status": "success", 21:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 21:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 21:52:54 DEBUG ----- output ----- 21:52:54 DEBUG { 21:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:52:54 DEBUG { 21:52:54 DEBUG ----- output ----- 21:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:52:54 DEBUG { 21:52:54 DEBUG { 21:52:54 DEBUG { 21:52:54 DEBUG --- http response --- 21:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:52:54 DEBUG ----- output ----- 21:52:54 DEBUG ----- output ----- 21:52:54 DEBUG { 21:52:54 DEBUG --- http response --- 21:52:54 DEBUG ----- output ----- 21:52:54 DEBUG "data": { 21:52:54 DEBUG --- http response --- 21:52:54 DEBUG ----- output ----- 21:52:54 DEBUG ----- output ----- 21:52:54 DEBUG ----- output ----- 21:52:54 DEBUG ----- output ----- 21:52:54 DEBUG "status": "success", 21:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:52:54 DEBUG ----- output ----- 21:52:54 DEBUG { 21:52:54 DEBUG "status": "success", 21:52:54 DEBUG { 21:52:54 DEBUG --- http response --- 21:52:54 DEBUG "status": "success", 21:52:54 DEBUG "status": "success", 21:52:54 DEBUG "status": "success", 21:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:52:54 DEBUG ----- output ----- 21:52:54 DEBUG { 21:52:54 DEBUG { 21:52:54 DEBUG "status": "success", 21:52:54 DEBUG "data": { 21:52:54 DEBUG { 21:52:54 DEBUG "resultType": "vector", 21:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 21:52:54 DEBUG { 21:52:54 DEBUG { 21:52:54 DEBUG { 21:52:54 DEBUG { 21:52:54 DEBUG "data": { 21:52:54 DEBUG "data": { 21:52:54 DEBUG "status": "success", 21:52:54 DEBUG ----- output ----- 21:52:54 DEBUG ----- output ----- 21:52:54 DEBUG "status": "success", 21:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:52:54 DEBUG "data": { 21:52:54 DEBUG "data": { 21:52:54 DEBUG "data": { 21:52:54 DEBUG ----- output ----- 21:52:54 DEBUG { 21:52:54 DEBUG "status": "success", 21:52:54 DEBUG "status": "success", 21:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:52:54 DEBUG "resultType": "vector", 21:52:54 DEBUG "status": "success", 21:52:54 DEBUG "result": [ 21:52:54 DEBUG ----- output ----- 21:52:54 DEBUG "status": "success", 21:52:54 DEBUG "status": "success", 21:52:54 DEBUG "status": "success", 21:52:54 DEBUG "data": { 21:52:54 DEBUG "resultType": "vector", 21:52:54 DEBUG "resultType": "vector", 21:52:54 DEBUG "data": { 21:52:54 DEBUG { 21:52:54 DEBUG { 21:52:54 DEBUG "data": { 21:52:54 DEBUG ----- output ----- 21:52:54 DEBUG "resultType": "vector", 21:52:54 DEBUG "resultType": "vector", 21:52:54 DEBUG "resultType": "vector", 21:52:54 DEBUG { 21:52:54 DEBUG "status": "success", 21:52:54 DEBUG "data": { 21:52:54 DEBUG "data": { 21:52:54 DEBUG ----- output ----- 21:52:54 DEBUG "result": [ 21:52:54 DEBUG "data": { 21:52:54 DEBUG { 21:52:54 DEBUG { 21:52:54 DEBUG "data": { 21:52:54 DEBUG "resultType": "vector", 21:52:54 DEBUG "resultType": "vector", 21:52:54 DEBUG "result": [ 21:52:54 DEBUG "result": [ 21:52:54 DEBUG "result": [ 21:52:54 DEBUG "resultType": "vector", 21:52:54 DEBUG "status": "success", 21:52:54 DEBUG "status": "success", 21:52:54 DEBUG "resultType": "vector", 21:52:54 DEBUG { 21:52:54 DEBUG "result": [ 21:52:54 DEBUG "result": [ 21:52:54 DEBUG "result": [ 21:52:54 DEBUG "status": "success", 21:52:54 DEBUG "data": { 21:52:54 DEBUG "resultType": "vector", 21:52:54 DEBUG "resultType": "vector", 21:52:54 DEBUG { 21:52:54 DEBUG { 21:52:54 DEBUG "resultType": "vector", 21:52:54 DEBUG "metric": { 21:52:54 DEBUG "status": "success", 21:52:54 DEBUG "result": [ 21:52:54 DEBUG "data": { 21:52:54 DEBUG { 21:52:54 DEBUG { 21:52:54 DEBUG { 21:52:54 DEBUG "result": [ 21:52:54 DEBUG { 21:52:54 DEBUG "status": "success", 21:52:54 DEBUG "data": { 21:52:54 DEBUG "result": [ 21:52:54 DEBUG "status": "success", 21:52:54 DEBUG { 21:52:54 DEBUG { 21:52:54 DEBUG { 21:52:54 DEBUG "data": { 21:52:54 DEBUG "resultType": "vector", 21:52:54 DEBUG "result": [ 21:52:54 DEBUG { 21:52:54 DEBUG "result": [ 21:52:54 DEBUG "metric": { 21:52:54 DEBUG "result": [ 21:52:54 DEBUG "pod": "am-767f6b67b5-9zw22" 21:52:54 DEBUG "data": { 21:52:54 DEBUG { 21:52:54 DEBUG "resultType": "vector", 21:52:54 DEBUG "metric": { 21:52:54 DEBUG "metric": { 21:52:54 DEBUG "metric": { 21:52:54 DEBUG "data": { 21:52:54 DEBUG "metric": { 21:52:54 DEBUG "data": { 21:52:54 DEBUG "resultType": "vector", 21:52:54 DEBUG { 21:52:54 DEBUG "data": { 21:52:54 DEBUG "metric": { 21:52:54 DEBUG "metric": { 21:52:54 DEBUG "metric": { 21:52:54 DEBUG "resultType": "vector", 21:52:54 DEBUG "result": [ 21:52:54 DEBUG "metric": { 21:52:54 DEBUG "status": "success", 21:52:54 DEBUG { 21:52:54 DEBUG "pod": "ds-cts-0" 21:52:54 DEBUG { 21:52:54 DEBUG }, 21:52:54 DEBUG "resultType": "vector", 21:52:54 DEBUG "metric": { 21:52:54 DEBUG "result": [ 21:52:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:52:54 DEBUG "pod": "am-767f6b67b5-9zw22" 21:52:54 DEBUG "pod": "ds-idrepo-0" 21:52:54 DEBUG "resultType": "vector", 21:52:54 DEBUG "pod": "am-767f6b67b5-9zw22" 21:52:54 DEBUG "resultType": "vector", 21:52:54 DEBUG "result": [ 21:52:54 DEBUG "metric": { 21:52:54 DEBUG "resultType": "vector", 21:52:54 DEBUG "pod": "ds-cts-0" 21:52:54 DEBUG "instance": "10.142.0.107:9100" 21:52:54 DEBUG "pod": "am-767f6b67b5-9zw22" 21:52:54 DEBUG "result": [ 21:52:54 DEBUG { 21:52:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 21:52:54 DEBUG "data": { 21:52:54 DEBUG "metric": { 21:52:54 DEBUG }, 21:52:54 DEBUG "metric": { 21:52:54 DEBUG "value": [ 21:52:54 DEBUG "result": [ 21:52:54 DEBUG "pod": "am-767f6b67b5-9zw22" 21:52:55 DEBUG }, 21:52:54 DEBUG }, 21:52:54 DEBUG }, 21:52:54 DEBUG }, 21:52:55 DEBUG "result": [ 21:52:55 DEBUG }, 21:52:55 DEBUG "result": [ 21:52:55 DEBUG { 21:52:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 21:52:55 DEBUG "result": [ 21:52:55 DEBUG }, 21:52:55 DEBUG }, 21:52:55 DEBUG }, 21:52:55 DEBUG { 21:52:55 DEBUG "metric": { 21:52:55 DEBUG }, 21:52:55 DEBUG "resultType": "vector", 21:52:55 DEBUG "instance": "10.142.0.107:9100" 21:52:55 DEBUG "value": [ 21:52:55 DEBUG "pod": "ds-idrepo-0" 21:52:55 DEBUG 21:52:55 DEBUG { 21:52:55 DEBUG "value": [ 21:52:54 DEBUG { 21:52:55 DEBUG "value": [ 21:52:55 DEBUG "value": [ 21:52:55 DEBUG "value": [ 21:52:55 DEBUG { 21:52:55 DEBUG "value": [ 21:52:55 DEBUG { 21:52:55 DEBUG "metric": { 21:52:55 DEBUG }, 21:52:55 DEBUG { 21:52:55 DEBUG "value": [ 21:52:55 DEBUG "value": [ 21:52:55 DEBUG "value": [ 21:52:55 DEBUG "instance": "10.142.0.107:9100" 21:52:55 DEBUG "value": [ 21:52:55 DEBUG "metric": { 21:52:55 DEBUG "result": [ 21:52:55 DEBUG }, 21:52:55 DEBUG 21:52:55 DEBUG }, 21:52:55 DEBUG 21:52:55 DEBUG "metric": { 21:52:55 DEBUG 21:52:55 DEBUG "metric": { 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG "metric": { 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG "pod": "am-767f6b67b5-9zw22" 21:52:55 DEBUG "value": [ 21:52:55 DEBUG "metric": { 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG }, 21:52:55 DEBUG 21:52:55 DEBUG "pod": "ds-cts-0" 21:52:55 DEBUG { 21:52:55 DEBUG "value": [ 21:52:55 DEBUG 21:52:55 DEBUG "value": [ 21:52:55 DEBUG 21:52:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:52:55 DEBUG 21:52:55 DEBUG "instance": "10.142.0.107:9100" 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG "pod": "am-767f6b67b5-9zw22" 21:52:55 DEBUG }, 21:52:55 DEBUG 21:52:55 DEBUG }, 21:52:55 DEBUG 21:52:55 DEBUG "pod": "ds-idrepo-0" 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG }, 21:52:55 DEBUG "metric": { 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG [...] 21:52:55 DEBUG }, 21:52:55 DEBUG 21:52:55 DEBUG }, 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG "value": [ 21:52:55 DEBUG "metric": { 21:52:55 DEBUG [...] 21:52:55 DEBUG "value": [ 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG [...] 21:52:55 DEBUG "value": [ 21:52:55 DEBUG 21:52:55 DEBUG "value": [ 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG [...] 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG "value": [ 21:52:55 DEBUG [...] 21:52:55 DEBUG "value": [ 21:52:55 DEBUG [...] 21:52:55 DEBUG [...] 21:52:55 DEBUG [...] 21:52:55 DEBUG 21:52:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG [...] 21:52:55 DEBUG }, 21:52:55 DEBUG [...] 21:52:55 DEBUG [...] 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG [...] 21:52:55 DEBUG 21:52:55 DEBUG "instance": "10.142.0.107:9100" 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG }, 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG "value": [ 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG }, 21:52:55 DEBUG [...] 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG "pod": "am-767f6b67b5-vbltz" 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG "value": [ 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG [...] 21:52:55 DEBUG "value": [ 21:52:55 DEBUG 21:52:55 DEBUG "pod": "ds-cts-2" 21:52:55 DEBUG }, 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG [...] 21:52:55 DEBUG 21:52:55 DEBUG "pod": "am-767f6b67b5-vbltz" 21:52:55 DEBUG [...] 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG "pod": "am-767f6b67b5-vbltz" 21:52:55 DEBUG [...] 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG }, 21:52:55 DEBUG "value": [ 21:52:55 DEBUG [...] 21:52:55 DEBUG [...] 21:52:55 DEBUG "pod": "am-767f6b67b5-vbltz" 21:52:55 DEBUG [...] 21:52:55 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 21:52:55 DEBUG "pod": "am-767f6b67b5-vbltz" 21:52:55 DEBUG "pod": "ds-idrepo-2" 21:52:55 DEBUG }, 21:52:55 DEBUG 21:52:55 DEBUG }, 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG "pod": "ds-cts-2" 21:52:55 DEBUG "instance": "10.142.0.43:9100" 21:52:55 DEBUG }, 21:52:55 DEBUG 21:52:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG "value": [ 21:52:55 DEBUG 1713646374, 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG }, 21:52:55 DEBUG 21:52:55 DEBUG }, 21:52:55 DEBUG "value": [ 21:52:55 DEBUG 21:52:55 DEBUG "value": [ 21:52:55 DEBUG 21:52:55 DEBUG "value": [ 21:52:55 DEBUG 21:52:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 21:52:55 DEBUG [...] 21:52:55 DEBUG 21:52:55 DEBUG "value": [ 21:52:55 DEBUG 21:52:55 DEBUG }, 21:52:55 DEBUG "value": [ 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG "instance": "10.142.0.43:9100" 21:52:55 DEBUG 1713646374, 21:52:55 DEBUG "0" 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG "value": [ 21:52:55 DEBUG 1713646374, 21:52:55 DEBUG }, 21:52:55 DEBUG 1713646374, 21:52:55 DEBUG 21:52:55 DEBUG 1713646374, 21:52:55 DEBUG [...] 21:52:55 DEBUG 1713646374, 21:52:55 DEBUG "pod": "am-767f6b67b5-vbltz" 21:52:55 DEBUG }, 21:52:55 DEBUG 21:52:55 DEBUG }, 21:52:55 DEBUG 1713646374, 21:52:55 DEBUG 21:52:55 DEBUG 1713646374, 21:52:55 DEBUG }, 21:52:55 DEBUG "pod": "ds-cts-2" 21:52:55 DEBUG [...] 21:52:55 DEBUG }, 21:52:55 DEBUG "0" 21:52:55 DEBUG ] 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG "3326.4" 21:52:55 DEBUG "value": [ 21:52:55 DEBUG 21:52:55 DEBUG "0" 21:52:55 DEBUG 21:52:55 DEBUG "0" 21:52:55 DEBUG 21:52:55 DEBUG "599527.5" 21:52:55 DEBUG }, 21:52:55 DEBUG "value": [ 21:52:55 DEBUG 21:52:55 DEBUG "value": [ 21:52:55 DEBUG "100475217.83333334" 21:52:55 DEBUG "instance": "10.142.0.43:9100" 21:52:55 DEBUG "3322.6" 21:52:55 DEBUG "value": [ 21:52:55 DEBUG }, 21:52:55 DEBUG 21:52:55 DEBUG 21:52:55 DEBUG ] 21:52:55 DEBUG } 21:52:55 DEBUG "pod": "ds-idrepo-2" 21:52:55 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 21:52:55 DEBUG ] 21:52:55 DEBUG 1713646374, 21:52:55 DEBUG 21:52:55 DEBUG ] 21:52:55 DEBUG "pod": "am-767f6b67b5-vbltz" 21:52:55 DEBUG ] 21:52:55 DEBUG 21:52:55 DEBUG ] 21:52:55 DEBUG "value": [ 21:52:55 DEBUG 1713646374, 21:52:55 DEBUG "pod": "ds-idrepo-2" 21:52:55 DEBUG 1713646374, 21:52:55 DEBUG "0" 21:52:55 DEBUG }, 21:52:55 DEBUG ] 21:52:55 DEBUG 1713646374, 21:52:55 DEBUG "value": [ 21:52:55 DEBUG 21:52:55 DEBUG "value": [ 21:52:55 DEBUG } 21:52:55 DEBUG ] 21:52:55 DEBUG }, 21:52:55 DEBUG }, 21:52:55 DEBUG } 21:52:55 DEBUG "3329.0666666666666" 21:52:55 DEBUG "instance": "10.142.0.43:9100" 21:52:55 DEBUG }, 21:52:55 DEBUG "value": [ 21:52:55 DEBUG } 21:52:55 DEBUG ] 21:52:55 DEBUG } 21:52:55 DEBUG 1713646374, 21:52:55 DEBUG "1020248064" 21:52:55 DEBUG }, 21:52:55 DEBUG ] 21:52:55 DEBUG ] 21:52:55 DEBUG "value": [ 21:52:55 DEBUG } 21:52:55 DEBUG "11.84199182308683" 21:52:55 DEBUG 1713646374, 21:52:55 DEBUG "instance": "10.142.0.43:9100" 21:52:55 DEBUG 1713646374, 21:52:55 DEBUG } 21:52:55 DEBUG ] 21:52:55 DEBUG "value": [ 21:52:55 DEBUG "value": [ 21:52:55 DEBUG ] 21:52:55 DEBUG ] 21:52:55 DEBUG } 21:52:55 DEBUG 1713646374, 21:52:55 DEBUG }, 21:52:55 DEBUG } 21:52:55 DEBUG } 21:52:55 DEBUG 21:52:55 DEBUG "49.377" 21:52:55 DEBUG ] 21:52:55 DEBUG } 21:52:55 DEBUG } 21:52:55 DEBUG } 21:52:55 DEBUG 1713646374, 21:52:55 DEBUG ] 21:52:55 DEBUG ] 21:52:55 DEBUG "0.4596666666666669" 21:52:55 DEBUG }, 21:52:55 DEBUG "0.07466666666666509" 21:52:55 DEBUG } 21:52:55 DEBUG } 21:52:55 DEBUG 1713646374, 21:52:55 DEBUG 1713646374, 21:52:55 DEBUG } 21:52:55 DEBUG } 21:52:55 DEBUG ] 21:52:55 DEBUG "100478289.69999999" 21:52:55 DEBUG "value": [ 21:52:55 DEBUG ] 21:52:55 DEBUG "pod": "ds-cts-0" 21:52:55 DEBUG ] 21:52:55 DEBUG ] 21:52:55 DEBUG "value": [ 21:52:55 DEBUG ] 21:52:55 DEBUG ] 21:52:55 DEBUG "0.40629999999998934" 21:52:55 DEBUG } 21:52:55 DEBUG } 21:52:55 DEBUG ] 21:52:55 DEBUG "value": [ 21:52:55 DEBUG ] 21:52:55 DEBUG } 21:52:55 DEBUG "0" 21:52:55 DEBUG "0" 21:52:55 DEBUG } 21:52:55 DEBUG ] 21:52:55 DEBUG } 21:52:55 DEBUG ] 21:52:55 DEBUG 1713646374, 21:52:55 DEBUG } 21:52:55 DEBUG }, 21:52:55 DEBUG } 21:52:55 DEBUG } 21:52:55 DEBUG 1713646374, 21:52:55 DEBUG } 21:52:55 DEBUG } 21:52:55 DEBUG ] 21:52:55 DEBUG } 21:52:55 DEBUG } 21:52:55 DEBUG ] 21:52:55 DEBUG 1713646374, 21:52:55 DEBUG } 21:52:55 DEBUG ] 21:52:55 DEBUG ] 21:52:55 DEBUG } 21:52:55 DEBUG } 21:52:55 DEBUG } 21:52:55 DEBUG "3334.3333333333335" 21:52:55 DEBUG } 21:52:55 DEBUG "value": [ 21:52:55 DEBUG ] 21:52:55 DEBUG } 21:52:55 DEBUG "0" 21:52:55 DEBUG } 21:52:55 DEBUG } 21:52:55 DEBUG } 21:52:55 DEBUG } 21:52:55 DEBUG } 21:52:55 DEBUG "3992688663552" 21:52:55 DEBUG ] 21:52:55 DEBUG } 21:52:55 DEBUG } 21:52:55 DEBUG } 21:52:55 DEBUG ] 21:52:55 DEBUG ] 21:52:55 DEBUG 1713646374, 21:52:55 DEBUG } 21:52:55 DEBUG ] 21:52:55 DEBUG ] 21:52:55 DEBUG ] 21:52:55 DEBUG ] 21:52:55 DEBUG } 21:52:55 DEBUG ] 21:52:55 DEBUG ] 21:52:55 DEBUG } 21:52:55 DEBUG } 21:52:55 DEBUG "0" 21:52:55 DEBUG } 21:52:55 DEBUG } 21:52:55 DEBUG ] 21:52:55 DEBUG } 21:52:55 DEBUG } 21:52:55 DEBUG } 21:52:55 DEBUG } 21:52:55 DEBUG } 21:52:55 DEBUG } 21:52:55 DEBUG ] 21:52:55 DEBUG ] 21:52:55 DEBUG } 21:52:55 DEBUG } 21:52:55 DEBUG ] 21:52:55 DEBUG } 21:52:55 DEBUG } 21:52:55 DEBUG } 21:52:55 DEBUG } 21:52:55 DEBUG } 21:52:55 DEBUG } 21:52:55 DEBUG } 21:52:55 DEBUG } 21:52:55 DEBUG } 21:52:55 DEBUG ] 21:52:55 DEBUG } 21:52:55 DEBUG } 21:52:55 DEBUG } 21:53:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713646433" 21:53:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646433" 21:53:53 ERROR [http_cmd]: ERROR 21:53:53 ERROR --- status code --- 21:53:53 ERROR http status code is 400 (expected [200, 401, 403, 404]) 21:53:53 ERROR --- http response --- 21:53:53 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 21:53:53 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 21:53:53 INFO [http_cmd]: http status code OK 21:53:53 DEBUG --- status code --- 21:53:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:53:53 DEBUG --- http response --- 21:53:53 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 21:53:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646433" 21:53:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713646433" 21:53:53 INFO [http_cmd]: http status code OK 21:53:53 DEBUG --- status code --- 21:53:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:53:53 DEBUG --- http response --- 21:53:53 INFO [http_cmd]: http status code OK 21:53:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:53:53 DEBUG --- status code --- 21:53:53 DEBUG ----- output ----- 21:53:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:53:53 DEBUG { 21:53:53 DEBUG --- http response --- 21:53:53 DEBUG "status": "success", 21:53:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 21:53:53 DEBUG "data": { 21:53:53 DEBUG ----- output ----- 21:53:53 DEBUG "resultType": "vector", 21:53:53 DEBUG { 21:53:53 DEBUG "result": [ 21:53:53 DEBUG "status": "success", 21:53:53 DEBUG { 21:53:53 DEBUG "data": { 21:53:53 DEBUG "resultType": "vector", 21:53:53 DEBUG "result": [ 21:53:53 DEBUG { 21:53:53 DEBUG "metric": { 21:53:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:53:53 DEBUG }, 21:53:53 DEBUG "value": [ 21:53:53 DEBUG 21:53:53 DEBUG 21:53:53 DEBUG 21:53:53 DEBUG [...] 21:53:53 DEBUG 21:53:53 DEBUG 21:53:53 DEBUG 21:53:53 DEBUG "pod": "am-767f6b67b5-vbltz" 21:53:53 DEBUG }, 21:53:53 DEBUG "value": [ 21:53:53 DEBUG 1713646433, 21:53:53 DEBUG "0" 21:53:53 DEBUG ] 21:53:53 DEBUG } 21:53:53 DEBUG "metric": { 21:53:53 DEBUG ] 21:53:53 DEBUG "pod": "ds-idrepo-0" 21:53:53 DEBUG } 21:53:53 DEBUG }, 21:53:53 DEBUG } 21:53:53 DEBUG "value": [ 21:53:53 DEBUG 21:53:53 DEBUG 21:53:53 DEBUG 21:53:53 DEBUG [...] 21:53:53 DEBUG 21:53:53 DEBUG 21:53:53 DEBUG 21:53:53 DEBUG "pod": "ds-cts-2" 21:53:53 DEBUG }, 21:53:53 DEBUG "value": [ 21:53:53 DEBUG 1713646433, 21:53:53 DEBUG "0" 21:53:53 DEBUG ] 21:53:53 DEBUG } 21:53:53 DEBUG ] 21:53:53 DEBUG } 21:53:53 DEBUG } 21:53:54 INFO 21:53:54 INFO [loop_until]: kubectl --namespace=xlou top pods 21:53:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:53:54 INFO [loop_until]: OK (rc = 0) 21:53:54 DEBUG --- stdout --- 21:53:54 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 22276m 8765Mi am-767f6b67b5-qcqz2 22085m 8887Mi am-767f6b67b5-vbltz 24432m 8937Mi ds-cts-0 19768m 12151Mi ds-cts-1 19603m 12122Mi ds-cts-2 20382m 12381Mi ds-idrepo-0 9206m 3896Mi ds-idrepo-1 9382m 3975Mi ds-idrepo-2 9513m 3874Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1178Mi idm-b85f9d978-wnhxb 4m 1238Mi lodemon-7dd48676f6-tdxl9 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-967d957f7-zk66v 11793m 885Mi 21:53:54 DEBUG --- stderr --- 21:53:54 DEBUG 21:53:55 INFO 21:53:55 INFO [loop_until]: kubectl --namespace=xlou top node 21:53:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:53:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646435" 21:53:55 INFO [http_cmd]: http status code OK 21:53:55 DEBUG --- status code --- 21:53:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:53:55 DEBUG --- http response --- 21:53:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:53:55 DEBUG ----- output ----- 21:53:55 DEBUG { 21:53:55 DEBUG "status": "success", 21:53:55 DEBUG "data": { 21:53:55 DEBUG "resultType": "vector", 21:53:55 DEBUG "result": [ 21:53:55 DEBUG { 21:53:55 DEBUG "metric": { 21:53:55 DEBUG "pod": "am-767f6b67b5-9zw22" 21:53:55 DEBUG }, 21:53:55 DEBUG "value": [ 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646435" 21:53:55 DEBUG 21:53:55 DEBUG [...] 21:53:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646435" 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646435" 21:53:55 DEBUG "pod": "am-767f6b67b5-vbltz" 21:53:55 DEBUG }, 21:53:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646435" 21:53:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646435" 21:53:55 DEBUG "value": [ 21:53:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646435" 21:53:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646435" 21:53:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713646435" 21:53:55 DEBUG 1713646435, 21:53:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646435" 21:53:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713646435" 21:53:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646435" 21:53:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713646435" 21:53:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646435" 21:53:55 INFO [http_cmd]: http status code OK 21:53:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646435" 21:53:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646435" 21:53:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646435" 21:53:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646435" 21:53:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646435" 21:53:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646435" 21:53:55 INFO [http_cmd]: http status code OK 21:53:55 INFO [http_cmd]: http status code OK 21:53:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646435" 21:53:55 INFO [http_cmd]: http status code OK 21:53:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646435" 21:53:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646435" 21:53:55 INFO [loop_until]: OK (rc = 0) 21:53:55 DEBUG "3332.7" 21:53:55 INFO [http_cmd]: http status code OK 21:53:55 DEBUG --- status code --- 21:53:55 INFO [http_cmd]: http status code OK 21:53:55 DEBUG --- status code --- 21:53:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:53:55 DEBUG --- http response --- 21:53:55 INFO [http_cmd]: http status code OK 21:53:55 INFO [http_cmd]: http status code OK 21:53:55 INFO [http_cmd]: http status code OK 21:53:55 INFO [http_cmd]: http status code OK 21:53:55 DEBUG --- status code --- 21:53:55 DEBUG --- status code --- 21:53:55 DEBUG --- status code --- 21:53:55 INFO [http_cmd]: http status code OK 21:53:55 INFO [http_cmd]: http status code OK 21:53:55 INFO [http_cmd]: http status code OK 21:53:55 INFO [http_cmd]: http status code OK 21:53:55 DEBUG --- stdout --- 21:53:55 INFO [http_cmd]: http status code OK 21:53:55 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 6224Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 9996m 33% 7659Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 21718m 72% 11487Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 84m 0% 3375Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 19308m 64% 15401Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 164m 0% 5214Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 22300m 74% 11785Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19727m 66% 3055Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 24712m 82% 11196Mi 10% gke-xlou-cdm-ds-0f601751-j488 10308m 34% 7282Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19749m 66% 17171Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 10191m 34% 7953Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20482m 68% 15077Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 12129m 76% 8814Mi 15% 21:53:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:53:55 INFO [http_cmd]: http status code OK 21:53:55 INFO [http_cmd]: http status code OK 21:53:55 INFO [http_cmd]: http status code OK 21:53:55 DEBUG --- status code --- 21:53:55 DEBUG --- status code --- 21:53:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:53:55 DEBUG --- status code --- 21:53:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:53:55 INFO [http_cmd]: http status code OK 21:53:55 DEBUG --- status code --- 21:53:55 INFO [http_cmd]: http status code OK 21:53:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:53:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:53:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:53:55 DEBUG --- status code --- 21:53:55 DEBUG --- status code --- 21:53:55 DEBUG --- status code --- 21:53:55 DEBUG --- status code --- 21:53:55 INFO [http_cmd]: http status code OK 21:53:55 DEBUG --- status code --- 21:53:55 DEBUG ] 21:53:55 DEBUG --- status code --- 21:53:55 DEBUG --- stderr --- 21:53:55 DEBUG --- http response --- 21:53:55 DEBUG --- status code --- 21:53:55 DEBUG --- status code --- 21:53:55 DEBUG --- status code --- 21:53:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:53:55 DEBUG --- http response --- 21:53:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:53:55 DEBUG ----- output ----- 21:53:55 DEBUG { 21:53:55 DEBUG "status": "success", 21:53:55 DEBUG --- status code --- 21:53:55 DEBUG --- status code --- 21:53:55 DEBUG --- http response --- 21:53:55 DEBUG --- http response --- 21:53:55 DEBUG --- http response --- 21:53:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:53:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:53:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:53:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:53:55 INFO [http_cmd]: http status code OK 21:53:55 DEBUG --- status code --- 21:53:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:53:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:53:55 DEBUG 21:53:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:53:55 DEBUG ----- output ----- 21:53:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:53:55 DEBUG "data": { 21:53:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:53:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:53:55 DEBUG --- http response --- 21:53:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:53:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:53:55 DEBUG --- http response --- 21:53:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:53:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:53:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 21:53:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:53:55 DEBUG --- http response --- 21:53:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:53:55 DEBUG --- http response --- 21:53:55 DEBUG --- http response --- 21:53:55 DEBUG --- http response --- 21:53:55 DEBUG --- http response --- 21:53:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 21:53:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:53:55 DEBUG --- http response --- 21:53:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:53:55 DEBUG { 21:53:55 DEBUG "status": "success", 21:53:55 DEBUG "data": { 21:53:55 DEBUG --- http response --- 21:53:55 DEBUG --- http response --- 21:53:55 DEBUG ----- output ----- 21:53:55 DEBUG --- status code --- 21:53:55 DEBUG ----- output ----- 21:53:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:53:55 DEBUG --- http response --- 21:53:55 DEBUG --- http response --- 21:53:55 DEBUG ----- output ----- 21:53:55 DEBUG ----- output ----- 21:53:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:53:55 DEBUG ----- output ----- 21:53:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:53:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 21:53:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:53:55 DEBUG ----- output ----- 21:53:55 DEBUG } 21:53:55 DEBUG --- http response --- 21:53:55 DEBUG --- http response --- 21:53:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:53:55 DEBUG "resultType": "vector", 21:53:55 DEBUG "resultType": "vector", 21:53:55 DEBUG --- http response --- 21:53:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:53:55 DEBUG { 21:53:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:53:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:53:55 DEBUG { 21:53:55 DEBUG ----- output ----- 21:53:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:53:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:53:55 DEBUG { 21:53:55 DEBUG { 21:53:55 DEBUG "status": "success", 21:53:55 DEBUG "data": { 21:53:55 DEBUG ----- output ----- 21:53:55 DEBUG ----- output ----- 21:53:55 DEBUG ----- output ----- 21:53:55 DEBUG { 21:53:55 DEBUG ] 21:53:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:53:55 DEBUG ----- output ----- 21:53:55 DEBUG { 21:53:55 DEBUG "result": [ 21:53:55 DEBUG { 21:53:55 DEBUG "metric": { 21:53:55 DEBUG "pod": "ds-cts-0" 21:53:55 DEBUG "status": "success", 21:53:55 DEBUG ----- output ----- 21:53:55 DEBUG --- http response --- 21:53:55 DEBUG "status": "success", 21:53:55 DEBUG { 21:53:55 DEBUG ----- output ----- 21:53:55 DEBUG ----- output ----- 21:53:55 DEBUG { 21:53:55 DEBUG ----- output ----- 21:53:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:53:55 DEBUG ----- output ----- 21:53:55 DEBUG { 21:53:55 DEBUG { 21:53:55 DEBUG { 21:53:55 DEBUG "status": "success", 21:53:55 DEBUG } 21:53:55 DEBUG "status": "success", 21:53:55 DEBUG "data": { 21:53:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:53:55 DEBUG ----- output ----- 21:53:55 DEBUG "data": { 21:53:55 DEBUG { 21:53:55 DEBUG ----- output ----- 21:53:55 DEBUG "result": [ 21:53:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:53:55 DEBUG "data": { 21:53:55 DEBUG "status": "success", 21:53:55 DEBUG { 21:53:55 DEBUG "status": "success", 21:53:55 DEBUG "data": { 21:53:55 DEBUG { 21:53:55 DEBUG { 21:53:55 DEBUG { 21:53:55 DEBUG "status": "success", 21:53:55 DEBUG "status": "success", 21:53:55 DEBUG "status": "success", 21:53:55 DEBUG "status": "success", 21:53:55 DEBUG "data": { 21:53:55 DEBUG } 21:53:55 DEBUG "resultType": "vector", 21:53:55 DEBUG ----- output ----- 21:53:55 DEBUG }, 21:53:55 DEBUG { 21:53:55 DEBUG "resultType": "vector", 21:53:55 DEBUG "status": "success", 21:53:55 DEBUG { 21:53:55 DEBUG { 21:53:55 DEBUG "metric": { 21:53:55 DEBUG "resultType": "vector", 21:53:55 DEBUG "data": { 21:53:55 DEBUG "status": "success", 21:53:55 DEBUG "resultType": "vector", 21:53:55 DEBUG "status": "success", 21:53:55 DEBUG "status": "success", 21:53:55 DEBUG "status": "success", 21:53:55 DEBUG "data": { 21:53:55 DEBUG "resultType": "vector", 21:53:55 DEBUG "result": [ 21:53:55 DEBUG "data": { 21:53:55 DEBUG "data": { 21:53:55 DEBUG "resultType": "vector", 21:53:55 DEBUG "result": [ 21:53:55 DEBUG { 21:53:55 DEBUG "value": [ 21:53:55 DEBUG "status": "success", 21:53:55 DEBUG "result": [ 21:53:55 DEBUG { 21:53:55 DEBUG "metric": { 21:53:55 DEBUG "pod": "am-767f6b67b5-9zw22" 21:53:55 DEBUG }, 21:53:55 DEBUG "result": [ 21:53:55 DEBUG { 21:53:55 DEBUG "metric": { 21:53:55 DEBUG "result": [ 21:53:55 DEBUG "data": { 21:53:55 DEBUG "data": { 21:53:55 DEBUG "data": { 21:53:55 DEBUG { 21:53:55 DEBUG "metric": { 21:53:55 DEBUG "pod": "am-767f6b67b5-9zw22" 21:53:55 DEBUG }, 21:53:55 DEBUG "resultType": "vector", 21:53:55 DEBUG "result": [ 21:53:55 DEBUG "metric": { 21:53:55 DEBUG { 21:53:55 DEBUG 21:53:55 DEBUG "value": [ 21:53:55 DEBUG "status": "success", 21:53:55 DEBUG "data": { 21:53:55 DEBUG "data": { 21:53:55 DEBUG "data": { 21:53:55 DEBUG "pod": "am-767f6b67b5-9zw22" 21:53:55 DEBUG "resultType": "vector", 21:53:55 DEBUG ----- output ----- 21:53:55 DEBUG { 21:53:55 DEBUG "instance": "10.142.0.107:9100" 21:53:55 DEBUG "resultType": "vector", 21:53:55 DEBUG "resultType": "vector", 21:53:55 DEBUG "resultType": "vector", 21:53:55 DEBUG "resultType": "vector", 21:53:55 DEBUG "value": [ 21:53:55 DEBUG "data": { 21:53:55 DEBUG "resultType": "vector", 21:53:55 DEBUG "result": [ 21:53:55 DEBUG { 21:53:55 DEBUG "pod": "ds-idrepo-0" 21:53:55 DEBUG "status": "success", 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG "data": { 21:53:55 DEBUG "resultType": "vector", 21:53:55 DEBUG "resultType": "vector", 21:53:55 DEBUG "result": [ 21:53:55 DEBUG "resultType": "vector", 21:53:55 DEBUG "result": [ 21:53:55 DEBUG { 21:53:55 DEBUG "metric": { 21:53:55 DEBUG }, 21:53:55 DEBUG "result": [ 21:53:55 DEBUG "result": [ 21:53:55 DEBUG "result": [ 21:53:55 DEBUG "result": [ 21:53:55 DEBUG 21:53:55 DEBUG "resultType": "vector", 21:53:55 DEBUG "result": [ 21:53:55 DEBUG { 21:53:55 DEBUG "metric": { 21:53:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:53:55 DEBUG "data": { 21:53:55 DEBUG "resultType": "vector", 21:53:55 DEBUG 21:53:55 DEBUG "result": [ 21:53:55 DEBUG { 21:53:55 DEBUG "result": [ 21:53:55 DEBUG }, 21:53:55 DEBUG "value": [ 21:53:55 DEBUG "status": "success", 21:53:55 DEBUG { 21:53:55 DEBUG "value": [ 21:53:55 DEBUG { 21:53:55 DEBUG { 21:53:55 DEBUG { 21:53:55 DEBUG "metric": { 21:53:55 DEBUG 21:53:55 DEBUG "result": [ 21:53:55 DEBUG { 21:53:55 DEBUG "metric": { 21:53:55 DEBUG }, 21:53:55 DEBUG "result": [ 21:53:55 DEBUG 21:53:55 DEBUG }, 21:53:55 DEBUG 21:53:55 DEBUG { 21:53:55 DEBUG "metric": { 21:53:55 DEBUG { 21:53:55 DEBUG 21:53:55 DEBUG "resultType": "vector", 21:53:55 DEBUG "instance": "10.142.0.107:9100" 21:53:55 DEBUG "data": { 21:53:55 DEBUG "metric": { 21:53:55 DEBUG 21:53:55 DEBUG "metric": { 21:53:55 DEBUG "pod": "am-767f6b67b5-9zw22" 21:53:55 DEBUG }, 21:53:55 DEBUG "value": [ 21:53:55 DEBUG 21:53:55 DEBUG { 21:53:55 DEBUG "metric": { 21:53:55 DEBUG "pod": "ds-cts-0" 21:53:55 DEBUG "value": [ 21:53:55 DEBUG { 21:53:55 DEBUG "metric": { 21:53:55 DEBUG "value": [ 21:53:55 DEBUG [...] 21:53:55 DEBUG "metric": { 21:53:55 DEBUG "pod": "am-767f6b67b5-9zw22" 21:53:55 DEBUG "metric": { 21:53:55 DEBUG 21:53:55 DEBUG "result": [ 21:53:55 DEBUG }, 21:53:55 DEBUG "resultType": "vector", 21:53:55 DEBUG "instance": "10.142.0.107:9100" 21:53:55 DEBUG 21:53:55 DEBUG { 21:53:55 DEBUG 21:53:55 DEBUG "metric": { 21:53:55 DEBUG [...] 21:53:55 DEBUG "metric": { 21:53:55 DEBUG }, 21:53:55 DEBUG "pod": "ds-cts-0" 21:53:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 21:53:55 DEBUG 21:53:55 DEBUG [...] 21:53:55 DEBUG "pod": "am-767f6b67b5-9zw22" 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG }, 21:53:55 DEBUG "instance": "10.142.0.107:9100" 21:53:55 DEBUG 21:53:55 DEBUG { 21:53:55 DEBUG "value": [ 21:53:55 DEBUG "result": [ 21:53:55 DEBUG }, 21:53:55 DEBUG 21:53:55 DEBUG "metric": { 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG "instance": "10.142.0.107:9100" 21:53:55 DEBUG "value": [ 21:53:55 DEBUG }, 21:53:55 DEBUG }, 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG }, 21:53:55 DEBUG 21:53:55 DEBUG [...] 21:53:55 DEBUG "pod": "ds-idrepo-0" 21:53:55 DEBUG }, 21:53:55 DEBUG }, 21:53:55 DEBUG [...] 21:53:55 DEBUG "metric": { 21:53:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 21:53:55 DEBUG { 21:53:55 DEBUG "value": [ 21:53:55 DEBUG [...] 21:53:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:53:55 DEBUG [...] 21:53:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:53:55 DEBUG 21:53:55 DEBUG }, 21:53:55 DEBUG 21:53:55 DEBUG "value": [ 21:53:55 DEBUG "value": [ 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG "value": [ 21:53:55 DEBUG "value": [ 21:53:55 DEBUG "value": [ 21:53:55 DEBUG 21:53:55 DEBUG }, 21:53:55 DEBUG "metric": { 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG }, 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG "value": [ 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG [...] 21:53:55 DEBUG "pod": "ds-cts-2" 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG "value": [ 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG "value": [ 21:53:55 DEBUG "pod": "ds-idrepo-0" 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG "value": [ 21:53:55 DEBUG 21:53:55 DEBUG }, 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG }, 21:53:55 DEBUG "pod": "am-767f6b67b5-vbltz" 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG [...] 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG [...] 21:53:55 DEBUG 21:53:55 DEBUG "pod": "am-767f6b67b5-vbltz" 21:53:55 DEBUG "value": [ 21:53:55 DEBUG "pod": "am-767f6b67b5-vbltz" 21:53:55 DEBUG [...] 21:53:55 DEBUG 21:53:55 DEBUG [...] 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG "value": [ 21:53:55 DEBUG }, 21:53:55 DEBUG "pod": "ds-idrepo-2" 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG "pod": "am-767f6b67b5-vbltz" 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG }, 21:53:55 DEBUG }, 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG }, 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG [...] 21:53:55 DEBUG 21:53:55 DEBUG [...] 21:53:55 DEBUG 21:53:55 DEBUG 1713646435, 21:53:55 DEBUG "value": [ 21:53:55 DEBUG }, 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG }, 21:53:55 DEBUG [...] 21:53:55 DEBUG 21:53:55 DEBUG "instance": "10.142.0.43:9100" 21:53:55 DEBUG 21:53:55 DEBUG "value": [ 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG "value": [ 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG "0" 21:53:55 DEBUG "pod": "overseer-0-967d957f7-zk66v" 21:53:55 DEBUG 1713646435, 21:53:55 DEBUG "value": [ 21:53:55 DEBUG [...] 21:53:55 DEBUG [...] 21:53:55 DEBUG 21:53:55 DEBUG "value": [ 21:53:55 DEBUG 21:53:55 DEBUG [...] 21:53:55 DEBUG 21:53:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 21:53:55 DEBUG }, 21:53:55 DEBUG [...] 21:53:55 DEBUG 21:53:55 DEBUG "value": [ 21:53:55 DEBUG 1713646435, 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG [...] 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG }, 21:53:55 DEBUG "3332.9666666666667" 21:53:55 DEBUG 1713646435, 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG "pod": "ds-idrepo-2" 21:53:55 DEBUG 1713646435, 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG "value": [ 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG 1713646435, 21:53:55 DEBUG "0" 21:53:55 DEBUG ] 21:53:55 DEBUG "instance": "10.142.0.43:9100" 21:53:55 DEBUG }, 21:53:55 DEBUG 21:53:55 DEBUG "pod": "ds-cts-2" 21:53:55 DEBUG ] 21:53:55 DEBUG 21:53:55 DEBUG "value": [ 21:53:55 DEBUG ] 21:53:55 DEBUG "0" 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG }, 21:53:55 DEBUG "600169.5" 21:53:55 DEBUG "instance": "10.142.0.43:9100" 21:53:55 DEBUG 21:53:55 DEBUG 1713646435, 21:53:55 DEBUG "11.858795456578186" 21:53:55 DEBUG ] 21:53:55 DEBUG } 21:53:55 DEBUG ] 21:53:55 DEBUG } 21:53:55 DEBUG }, 21:53:55 DEBUG "value": [ 21:53:55 DEBUG 21:53:55 DEBUG "pod": "ds-cts-2" 21:53:55 DEBUG }, 21:53:55 DEBUG "value": [ 21:53:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 21:53:55 DEBUG 1713646435, 21:53:55 DEBUG } 21:53:55 DEBUG ] 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG "value": [ 21:53:55 DEBUG ] 21:53:55 DEBUG }, 21:53:55 DEBUG "value": [ 21:53:55 DEBUG [...] 21:53:55 DEBUG } 21:53:55 DEBUG 21:53:55 DEBUG ] 21:53:55 DEBUG 1713646435, 21:53:55 DEBUG "100185817.43333334" 21:53:55 DEBUG "value": [ 21:53:55 DEBUG 21:53:55 DEBUG "instance": "10.142.0.43:9100" 21:53:55 DEBUG }, 21:53:55 DEBUG "value": [ 21:53:55 DEBUG }, 21:53:55 DEBUG "19.778091833688627" 21:53:55 DEBUG ] 21:53:55 DEBUG } 21:53:55 DEBUG } 21:53:55 DEBUG "pod": "am-767f6b67b5-vbltz" 21:53:55 DEBUG "pod": "am-767f6b67b5-vbltz" 21:53:55 DEBUG 1713646435, 21:53:55 DEBUG } 21:53:55 DEBUG ] 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG } 21:53:55 DEBUG 21:53:55 DEBUG "pod": "ds-idrepo-2" 21:53:55 DEBUG ] 21:53:55 DEBUG "49.377" 21:53:55 DEBUG ] 21:53:55 DEBUG [...] 21:53:55 DEBUG }, 21:53:55 DEBUG 1713646435, 21:53:55 DEBUG 1713646435, 21:53:55 DEBUG "value": [ 21:53:55 DEBUG ] 21:53:55 DEBUG } 21:53:55 DEBUG } 21:53:55 DEBUG }, 21:53:55 DEBUG }, 21:53:55 DEBUG "0" 21:53:55 DEBUG } 21:53:55 DEBUG 1713646435, 21:53:55 DEBUG "instance": "10.142.0.43:9100" 21:53:55 DEBUG 21:53:55 DEBUG }, 21:53:55 DEBUG } 21:53:55 DEBUG } 21:53:55 DEBUG } 21:53:55 DEBUG 1713646435, 21:53:55 DEBUG 21:53:55 DEBUG 21:53:55 DEBUG "0.5220999999999993" 21:53:55 DEBUG "0" 21:53:55 DEBUG 1713646435, 21:53:55 DEBUG "1025429504" 21:53:55 DEBUG ] 21:53:55 DEBUG } 21:53:55 DEBUG "value": [ 21:53:55 DEBUG "value": [ 21:53:55 DEBUG ] 21:53:55 DEBUG } 21:53:55 DEBUG "3998013029376" 21:53:55 DEBUG }, 21:53:55 DEBUG 21:53:55 DEBUG "value": [ 21:53:55 DEBUG ] 21:53:55 DEBUG "0.08366666666666064" 21:53:55 DEBUG } 21:53:55 DEBUG ] 21:53:55 DEBUG } 21:53:55 DEBUG ] 21:53:55 DEBUG ] 21:53:55 DEBUG } 21:53:55 DEBUG ] 21:53:55 DEBUG ] 21:53:55 DEBUG ] 21:53:55 DEBUG } 21:53:55 DEBUG 1713646435, 21:53:55 DEBUG } 21:53:55 DEBUG ] 21:53:55 DEBUG "value": [ 21:53:55 DEBUG "pod": "overseer-0-967d957f7-zk66v" 21:53:55 DEBUG }, 21:53:55 DEBUG } 21:53:55 DEBUG ] 21:53:55 DEBUG 21:53:55 DEBUG "value": [ 21:53:55 DEBUG } 21:53:55 DEBUG } 21:53:55 DEBUG } 21:53:55 DEBUG ] 21:53:55 DEBUG } 21:53:55 DEBUG } 21:53:55 DEBUG 1713646435, 21:53:55 DEBUG } 21:53:55 DEBUG "3329.7666666666664" 21:53:55 DEBUG ] 21:53:55 DEBUG } 21:53:55 DEBUG 1713646435, 21:53:55 DEBUG "value": [ 21:53:55 DEBUG 1713646435, 21:53:55 DEBUG } 21:53:55 DEBUG } 21:53:55 DEBUG "pod": "overseer-0-967d957f7-zk66v" 21:53:55 DEBUG 1713646435, 21:53:55 DEBUG ] 21:53:55 DEBUG } 21:53:55 DEBUG } 21:53:55 DEBUG } 21:53:55 DEBUG } 21:53:55 DEBUG "3334.0333333333333" 21:53:55 DEBUG ] 21:53:55 DEBUG } 21:53:55 DEBUG ] 21:53:55 DEBUG "100187736.73333333" 21:53:55 DEBUG 1713646435, 21:53:55 DEBUG "0" 21:53:55 DEBUG ] 21:53:55 DEBUG }, 21:53:55 DEBUG "0.43560000000002314" 21:53:55 DEBUG ] 21:53:55 DEBUG } 21:53:55 DEBUG ] 21:53:55 DEBUG } 21:53:55 DEBUG } 21:53:55 DEBUG } 21:53:55 DEBUG ] 21:53:55 DEBUG "0" 21:53:55 DEBUG ] 21:53:55 DEBUG } 21:53:55 DEBUG "value": [ 21:53:55 DEBUG ] 21:53:55 DEBUG } 21:53:55 DEBUG } 21:53:55 DEBUG } 21:53:55 DEBUG ] 21:53:55 DEBUG } 21:53:55 DEBUG } 21:53:55 DEBUG ] 21:53:55 DEBUG } 21:53:55 DEBUG } 21:53:55 DEBUG 1713646435, 21:53:55 DEBUG } 21:53:55 DEBUG } 21:53:55 DEBUG ] 21:53:55 DEBUG } 21:53:55 DEBUG } 21:53:55 DEBUG ] 21:53:55 DEBUG ] 21:53:55 DEBUG "0" 21:53:55 DEBUG ] 21:53:55 DEBUG } 21:53:55 DEBUG } 21:53:55 DEBUG } 21:53:55 DEBUG } 21:53:55 DEBUG ] 21:53:55 DEBUG } 21:53:55 DEBUG } 21:53:55 DEBUG } 21:53:55 DEBUG } 21:53:55 DEBUG ] 21:53:55 DEBUG } 21:53:55 DEBUG } 21:53:55 DEBUG } 21:53:55 DEBUG } 21:53:55 DEBUG ] 21:53:55 DEBUG } 21:53:55 DEBUG } 21:54:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713646493" 21:54:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646493" 21:54:53 ERROR [http_cmd]: ERROR 21:54:53 ERROR --- status code --- 21:54:53 ERROR http status code is 400 (expected [200, 401, 403, 404]) 21:54:53 ERROR --- http response --- 21:54:53 INFO [http_cmd]: http status code OK 21:54:53 DEBUG --- status code --- 21:54:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:54:53 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 21:54:53 DEBUG --- http response --- 21:54:53 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 21:54:53 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 21:54:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646493" 21:54:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713646493" 21:54:53 INFO [http_cmd]: http status code OK 21:54:53 DEBUG --- status code --- 21:54:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:54:53 DEBUG --- http response --- 21:54:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 21:54:53 DEBUG ----- output ----- 21:54:53 DEBUG { 21:54:53 DEBUG "status": "success", 21:54:53 DEBUG "data": { 21:54:53 DEBUG "resultType": "vector", 21:54:53 DEBUG "result": [ 21:54:53 DEBUG { 21:54:53 DEBUG "metric": { 21:54:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:54:53 DEBUG }, 21:54:53 DEBUG "value": [ 21:54:53 DEBUG 21:54:53 DEBUG 21:54:53 DEBUG 21:54:53 DEBUG [...] 21:54:53 DEBUG 21:54:53 DEBUG 21:54:53 DEBUG 21:54:53 DEBUG "pod": "am-767f6b67b5-vbltz" 21:54:53 DEBUG }, 21:54:53 DEBUG "value": [ 21:54:53 DEBUG 1713646493, 21:54:53 DEBUG "117809.12021341095" 21:54:53 DEBUG ] 21:54:53 DEBUG } 21:54:53 DEBUG ] 21:54:53 DEBUG } 21:54:53 DEBUG } 21:54:53 INFO [http_cmd]: http status code OK 21:54:53 DEBUG --- status code --- 21:54:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:54:53 DEBUG --- http response --- 21:54:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:54:53 DEBUG ----- output ----- 21:54:53 DEBUG { 21:54:53 DEBUG "status": "success", 21:54:53 DEBUG "data": { 21:54:53 DEBUG "resultType": "vector", 21:54:53 DEBUG "result": [ 21:54:53 DEBUG { 21:54:53 DEBUG "metric": { 21:54:53 DEBUG "pod": "ds-idrepo-0" 21:54:53 DEBUG }, 21:54:53 DEBUG "value": [ 21:54:53 DEBUG 21:54:53 DEBUG 21:54:53 DEBUG 21:54:53 DEBUG [...] 21:54:53 DEBUG 21:54:53 DEBUG 21:54:53 DEBUG 21:54:53 DEBUG "pod": "ds-cts-2" 21:54:53 DEBUG }, 21:54:53 DEBUG "value": [ 21:54:53 DEBUG 1713646493, 21:54:53 DEBUG "0" 21:54:53 DEBUG ] 21:54:53 DEBUG } 21:54:53 DEBUG ] 21:54:53 DEBUG } 21:54:53 DEBUG } 21:54:54 INFO 21:54:54 INFO [loop_until]: kubectl --namespace=xlou top pods 21:54:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:54:54 INFO [loop_until]: OK (rc = 0) 21:54:54 DEBUG --- stdout --- 21:54:54 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 22565m 8768Mi am-767f6b67b5-qcqz2 21569m 8884Mi am-767f6b67b5-vbltz 24104m 8939Mi ds-cts-0 20030m 12214Mi ds-cts-1 20084m 12162Mi ds-cts-2 20545m 12404Mi ds-idrepo-0 9237m 3899Mi ds-idrepo-1 9725m 3975Mi ds-idrepo-2 9880m 3878Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-7dd48676f6-tdxl9 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-967d957f7-zk66v 11725m 886Mi 21:54:54 DEBUG --- stderr --- 21:54:54 DEBUG 21:54:55 INFO 21:54:55 INFO [loop_until]: kubectl --namespace=xlou top node 21:54:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:54:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646495" 21:54:55 INFO [http_cmd]: http status code OK 21:54:55 DEBUG --- status code --- 21:54:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:54:55 DEBUG --- http response --- 21:54:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:54:55 DEBUG ----- output ----- 21:54:55 DEBUG { 21:54:55 DEBUG "status": "success", 21:54:55 DEBUG "data": { 21:54:55 DEBUG "resultType": "vector", 21:54:55 DEBUG "result": [ 21:54:55 DEBUG { 21:54:55 DEBUG "metric": { 21:54:55 DEBUG "pod": "am-767f6b67b5-9zw22" 21:54:55 DEBUG }, 21:54:55 DEBUG "value": [ 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG [...] 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG "pod": "am-767f6b67b5-vbltz" 21:54:55 DEBUG }, 21:54:55 DEBUG "value": [ 21:54:55 DEBUG 1713646495, 21:54:55 DEBUG "3317.4333333333334" 21:54:55 DEBUG ] 21:54:55 DEBUG } 21:54:55 DEBUG ] 21:54:55 DEBUG } 21:54:55 DEBUG } 21:54:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646495" 21:54:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713646495" 21:54:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646495" 21:54:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646495" 21:54:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646495" 21:54:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646495" 21:54:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713646495" 21:54:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713646495" 21:54:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646495" 21:54:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646495" 21:54:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646495" 21:54:55 INFO [http_cmd]: http status code OK 21:54:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646495" 21:54:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646495" 21:54:55 INFO [http_cmd]: http status code OK 21:54:55 DEBUG --- status code --- 21:54:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646495" 21:54:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646495" 21:54:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646495" 21:54:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646495" 21:54:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646495" 21:54:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646495" 21:54:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646495" 21:54:55 INFO [http_cmd]: http status code OK 21:54:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646495" 21:54:55 INFO [http_cmd]: http status code OK 21:54:55 DEBUG --- status code --- 21:54:55 INFO [http_cmd]: http status code OK 21:54:55 INFO [loop_until]: OK (rc = 0) 21:54:55 DEBUG --- status code --- 21:54:55 INFO [http_cmd]: http status code OK 21:54:55 DEBUG --- status code --- 21:54:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:54:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646495" 21:54:55 INFO [http_cmd]: http status code OK 21:54:55 INFO [http_cmd]: http status code OK 21:54:55 DEBUG --- status code --- 21:54:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:54:55 DEBUG --- http response --- 21:54:55 DEBUG --- status code --- 21:54:55 INFO [http_cmd]: http status code OK 21:54:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:54:55 INFO [http_cmd]: http status code OK 21:54:55 INFO [http_cmd]: http status code OK 21:54:55 DEBUG --- status code --- 21:54:55 INFO [http_cmd]: http status code OK 21:54:55 INFO [http_cmd]: http status code OK 21:54:55 DEBUG --- stdout --- 21:54:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:54:55 INFO [http_cmd]: http status code OK 21:54:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:54:55 DEBUG --- http response --- 21:54:55 INFO [http_cmd]: http status code OK 21:54:55 DEBUG --- status code --- 21:54:55 DEBUG --- http response --- 21:54:55 DEBUG --- status code --- 21:54:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:54:55 INFO [http_cmd]: http status code OK 21:54:55 DEBUG --- status code --- 21:54:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:54:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:54:55 DEBUG --- status code --- 21:54:55 DEBUG --- http response --- 21:54:55 DEBUG --- status code --- 21:54:55 DEBUG --- status code --- 21:54:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:54:55 DEBUG --- status code --- 21:54:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:54:55 DEBUG --- http response --- 21:54:55 INFO [http_cmd]: http status code OK 21:54:55 DEBUG --- status code --- 21:54:55 DEBUG --- status code --- 21:54:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:54:55 INFO [http_cmd]: http status code OK 21:54:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:54:55 INFO [http_cmd]: http status code OK 21:54:55 DEBUG --- status code --- 21:54:55 DEBUG --- http response --- 21:54:55 INFO [http_cmd]: http status code OK 21:54:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:54:55 INFO [http_cmd]: http status code OK 21:54:55 DEBUG ----- output ----- 21:54:55 DEBUG { 21:54:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:54:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:54:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:54:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:54:55 DEBUG --- http response --- 21:54:55 DEBUG --- http response --- 21:54:55 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 66m 0% 6228Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 9978m 33% 7666Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 21753m 72% 11486Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 84m 0% 3374Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 20145m 67% 15449Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 161m 0% 5217Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 22293m 74% 11775Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19681m 65% 3054Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 24576m 82% 11198Mi 10% gke-xlou-cdm-ds-0f601751-j488 10420m 34% 7287Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19887m 66% 17242Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 10134m 33% 7946Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20569m 68% 15109Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 11752m 73% 8815Mi 15% 21:54:55 DEBUG --- http response --- 21:54:55 INFO [http_cmd]: http status code OK 21:54:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 21:54:55 DEBUG ----- output ----- 21:54:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:54:55 DEBUG ----- output ----- 21:54:55 DEBUG --- status code --- 21:54:55 DEBUG --- http response --- 21:54:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:54:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:54:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:54:55 DEBUG --- status code --- 21:54:55 DEBUG --- http response --- 21:54:55 DEBUG --- status code --- 21:54:55 DEBUG "status": "success", 21:54:55 DEBUG --- http response --- 21:54:55 DEBUG ----- output ----- 21:54:55 DEBUG --- http response --- 21:54:55 DEBUG --- status code --- 21:54:55 DEBUG --- http response --- 21:54:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:54:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:54:55 DEBUG --- stderr --- 21:54:55 DEBUG --- status code --- 21:54:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:54:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:54:55 DEBUG { 21:54:55 DEBUG --- http response --- 21:54:55 DEBUG { 21:54:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:54:55 DEBUG --- http response --- 21:54:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:54:55 DEBUG ----- output ----- 21:54:55 DEBUG ----- output ----- 21:54:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:54:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:54:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:54:55 DEBUG "data": { 21:54:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:54:55 DEBUG { 21:54:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:54:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:54:55 DEBUG ----- output ----- 21:54:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:54:55 DEBUG ----- output ----- 21:54:55 DEBUG 21:54:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:54:55 DEBUG ----- output ----- 21:54:55 DEBUG --- http response --- 21:54:55 DEBUG "status": "success", 21:54:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:54:55 DEBUG "status": "success", 21:54:55 DEBUG --- http response --- 21:54:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:54:55 DEBUG ----- output ----- 21:54:55 DEBUG { 21:54:55 DEBUG { 21:54:55 DEBUG --- http response --- 21:54:55 DEBUG --- http response --- 21:54:55 DEBUG ----- output ----- 21:54:55 DEBUG "resultType": "vector", 21:54:55 DEBUG ----- output ----- 21:54:55 DEBUG "status": "success", 21:54:55 DEBUG ----- output ----- 21:54:55 DEBUG --- http response --- 21:54:55 DEBUG { 21:54:55 DEBUG ----- output ----- 21:54:55 DEBUG { 21:54:55 DEBUG --- http response --- 21:54:55 DEBUG { 21:54:55 DEBUG "data": { 21:54:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:54:55 DEBUG ----- output ----- 21:54:55 DEBUG "data": { 21:54:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:54:55 DEBUG ----- output ----- 21:54:55 DEBUG { 21:54:55 DEBUG "status": "success", 21:54:55 DEBUG "status": "success", 21:54:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:54:55 DEBUG { 21:54:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:54:55 DEBUG ----- output ----- 21:54:55 DEBUG { 21:54:55 DEBUG "data": { 21:54:55 DEBUG { 21:54:55 DEBUG "status": "success", 21:54:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:54:55 DEBUG { 21:54:55 DEBUG "status": "success", 21:54:55 DEBUG "status": "success", 21:54:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:54:55 DEBUG "resultType": "vector", 21:54:55 DEBUG ----- output ----- 21:54:55 DEBUG { 21:54:55 DEBUG "resultType": "vector", 21:54:55 DEBUG ----- output ----- 21:54:55 DEBUG { 21:54:55 DEBUG "status": "success", 21:54:55 DEBUG "data": { 21:54:55 DEBUG "resultType": "vector", 21:54:55 DEBUG ----- output ----- 21:54:55 DEBUG { 21:54:55 DEBUG "status": "success", 21:54:55 DEBUG "result": [ 21:54:55 DEBUG "status": "success", 21:54:55 DEBUG "resultType": "vector", 21:54:55 DEBUG "status": "success", 21:54:55 DEBUG "data": { 21:54:55 DEBUG ----- output ----- 21:54:55 DEBUG "status": "success", 21:54:55 DEBUG "data": { 21:54:55 DEBUG "data": { 21:54:55 DEBUG ----- output ----- 21:54:55 DEBUG "result": [ 21:54:55 DEBUG { 21:54:55 DEBUG "status": "success", 21:54:55 DEBUG "result": [ 21:54:55 DEBUG { 21:54:55 DEBUG "data": { 21:54:55 DEBUG "status": "success", 21:54:55 DEBUG "result": [ 21:54:55 DEBUG { 21:54:55 DEBUG "status": "success", 21:54:55 DEBUG { 21:54:55 DEBUG "data": { 21:54:55 DEBUG { 21:54:55 DEBUG "data": { 21:54:55 DEBUG "result": [ 21:54:55 DEBUG "resultType": "vector", 21:54:55 DEBUG "result": [ 21:54:55 DEBUG { 21:54:55 DEBUG "metric": { 21:54:55 DEBUG "pod": "ds-idrepo-1" 21:54:55 DEBUG { 21:54:55 DEBUG "resultType": "vector", 21:54:55 DEBUG "status": "success", 21:54:55 DEBUG "data": { 21:54:55 DEBUG { 21:54:55 DEBUG "status": "success", 21:54:55 DEBUG "data": { 21:54:55 DEBUG "resultType": "vector", 21:54:55 DEBUG "result": [ 21:54:55 DEBUG "metric": { 21:54:55 DEBUG "data": { 21:54:55 DEBUG "data": { 21:54:55 DEBUG "resultType": "vector", 21:54:55 DEBUG "status": "success", 21:54:55 DEBUG "metric": { 21:54:55 DEBUG "resultType": "vector", 21:54:55 DEBUG { 21:54:55 DEBUG "data": { 21:54:55 DEBUG }, 21:54:55 DEBUG { 21:54:55 DEBUG "resultType": "vector", 21:54:55 DEBUG { 21:54:55 DEBUG "status": "success", 21:54:55 DEBUG "result": [ 21:54:55 DEBUG "data": { 21:54:55 DEBUG "resultType": "vector", 21:54:55 DEBUG "metric": { 21:54:55 DEBUG "data": { 21:54:55 DEBUG "resultType": "vector", 21:54:55 DEBUG { 21:54:55 DEBUG "data": { 21:54:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 21:54:55 DEBUG "resultType": "vector", 21:54:55 DEBUG "resultType": "vector", 21:54:55 DEBUG "result": [ 21:54:55 DEBUG "data": { 21:54:55 DEBUG "resultType": "vector", 21:54:55 DEBUG "result": [ 21:54:55 DEBUG "metric": { 21:54:55 DEBUG "resultType": "vector", 21:54:55 DEBUG "value": [ 21:54:55 DEBUG "status": "success", 21:54:55 DEBUG "result": [ 21:54:55 DEBUG "metric": { 21:54:55 DEBUG "data": { 21:54:55 DEBUG { 21:54:55 DEBUG "resultType": "vector", 21:54:55 DEBUG "result": [ 21:54:55 DEBUG "pod": "am-767f6b67b5-9zw22" 21:54:55 DEBUG "resultType": "vector", 21:54:55 DEBUG "result": [ 21:54:55 DEBUG { 21:54:55 DEBUG "resultType": "vector", 21:54:55 DEBUG }, 21:54:55 DEBUG "value": [ 21:54:55 DEBUG "result": [ 21:54:55 DEBUG { 21:54:55 DEBUG "result": [ 21:54:55 DEBUG "pod": "ds-cts-0" 21:54:55 DEBUG { 21:54:55 DEBUG "instance": "10.142.0.107:9100" 21:54:55 DEBUG "result": [ 21:54:55 DEBUG 21:54:55 DEBUG "data": { 21:54:55 DEBUG { 21:54:55 DEBUG "pod": "ds-idrepo-1" 21:54:55 DEBUG "resultType": "vector", 21:54:55 DEBUG "metric": { 21:54:55 DEBUG "result": [ 21:54:55 DEBUG { 21:54:55 DEBUG }, 21:54:55 DEBUG "result": [ 21:54:55 DEBUG "metric": { 21:54:55 DEBUG "metric": { 21:54:55 DEBUG "result": [ 21:54:55 DEBUG "result": [ 21:54:55 DEBUG 21:54:55 DEBUG { 21:54:55 DEBUG "metric": { 21:54:55 DEBUG { 21:54:55 DEBUG }, 21:54:55 DEBUG "metric": { 21:54:55 DEBUG }, 21:54:55 DEBUG { 21:54:55 DEBUG 21:54:55 DEBUG "resultType": "vector", 21:54:55 DEBUG "metric": { 21:54:55 DEBUG }, 21:54:55 DEBUG "result": [ 21:54:55 DEBUG "pod": "am-767f6b67b5-9zw22" 21:54:55 DEBUG { 21:54:55 DEBUG "metric": { 21:54:55 DEBUG "value": [ 21:54:55 DEBUG { 21:54:55 DEBUG "instance": "10.142.0.107:9100" 21:54:55 DEBUG "pod": "am-767f6b67b5-9zw22" 21:54:55 DEBUG { 21:54:55 DEBUG { 21:54:55 DEBUG 21:54:55 DEBUG "metric": { 21:54:55 DEBUG "pod": "ds-idrepo-0" 21:54:55 DEBUG }, 21:54:55 DEBUG "value": [ 21:54:55 DEBUG "pod": "am-767f6b67b5-9zw22" 21:54:55 DEBUG "metric": { 21:54:55 DEBUG "value": [ 21:54:55 DEBUG 21:54:55 DEBUG "result": [ 21:54:55 DEBUG "pod": "am-767f6b67b5-9zw22" 21:54:55 DEBUG "value": [ 21:54:55 DEBUG { 21:54:55 DEBUG }, 21:54:55 DEBUG "metric": { 21:54:55 DEBUG "pod": "ds-idrepo-0" 21:54:55 DEBUG }, 21:54:55 DEBUG "value": [ 21:54:55 DEBUG 21:54:55 DEBUG "metric": { 21:54:55 DEBUG "metric": { 21:54:55 DEBUG "metric": { 21:54:55 DEBUG 21:54:55 DEBUG "instance": "10.142.0.107:9100" 21:54:55 DEBUG "value": [ 21:54:55 DEBUG "metric": { 21:54:55 DEBUG 21:54:55 DEBUG }, 21:54:55 DEBUG "pod": "ds-cts-0" 21:54:55 DEBUG 21:54:55 DEBUG [...] 21:54:55 DEBUG { 21:54:55 DEBUG }, 21:54:55 DEBUG 21:54:55 DEBUG "metric": { 21:54:55 DEBUG "value": [ 21:54:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 21:54:55 DEBUG }, 21:54:55 DEBUG }, 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG "pod": "ds-idrepo-1" 21:54:55 DEBUG }, 21:54:55 DEBUG [...] 21:54:55 DEBUG }, 21:54:55 DEBUG "value": [ 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG "value": [ 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG "value": [ 21:54:55 DEBUG 21:54:55 DEBUG "pod": "am-767f6b67b5-9zw22" 21:54:55 DEBUG 21:54:55 DEBUG "value": [ 21:54:55 DEBUG "value": [ 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG "instance": "10.142.0.107:9100" 21:54:55 DEBUG "value": [ 21:54:55 DEBUG "pod": "ds-idrepo-0" 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG "pod": "ds-cts-0" 21:54:55 DEBUG 21:54:55 DEBUG [...] 21:54:55 DEBUG }, 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG "metric": { 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG }, 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG [...] 21:54:55 DEBUG [...] 21:54:55 DEBUG }, 21:54:55 DEBUG "value": [ 21:54:55 DEBUG }, 21:54:55 DEBUG 21:54:55 DEBUG }, 21:54:55 DEBUG 21:54:55 DEBUG }, 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG "value": [ 21:54:55 DEBUG [...] 21:54:55 DEBUG 21:54:55 DEBUG "pod": "ds-cts-0" 21:54:55 DEBUG "instance": "10.142.0.107:9100" 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG "value": [ 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG [...] 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG "value": [ 21:54:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 21:54:55 DEBUG "value": [ 21:54:55 DEBUG [...] 21:54:55 DEBUG "value": [ 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG [...] 21:54:55 DEBUG }, 21:54:55 DEBUG }, 21:54:55 DEBUG [...] 21:54:55 DEBUG [...] 21:54:55 DEBUG 21:54:55 DEBUG [...] 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG }, 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG "value": [ 21:54:55 DEBUG "value": [ 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG [...] 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG "pod": "am-767f6b67b5-vbltz" 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG "value": [ 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG [...] 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG [...] 21:54:55 DEBUG }, 21:54:55 DEBUG 21:54:55 DEBUG [...] 21:54:55 DEBUG 21:54:55 DEBUG 1713646495, 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG [...] 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG "pod": "ds-cts-2" 21:54:55 DEBUG "pod": "am-767f6b67b5-vbltz" 21:54:55 DEBUG "instance": "10.142.0.43:9100" 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG 1713646495, 21:54:55 DEBUG "39657548.51254412" 21:54:55 DEBUG ] 21:54:55 DEBUG 21:54:55 DEBUG [...] 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG "value": [ 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG [...] 21:54:55 DEBUG "11.867745706991617" 21:54:55 DEBUG [...] 21:54:55 DEBUG 21:54:55 DEBUG "instance": "10.142.0.43:9100" 21:54:55 DEBUG [...] 21:54:55 DEBUG 21:54:55 DEBUG }, 21:54:55 DEBUG }, 21:54:55 DEBUG }, 21:54:55 DEBUG 21:54:55 DEBUG "pod": "am-767f6b67b5-vbltz" 21:54:55 DEBUG } 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG "pod": "am-767f6b67b5-vbltz" 21:54:55 DEBUG 21:54:55 DEBUG "pod": "ds-idrepo-2" 21:54:55 DEBUG 21:54:55 DEBUG 1713646495, 21:54:55 DEBUG "pod": "am-767f6b67b5-vbltz" 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG ] 21:54:55 DEBUG 21:54:55 DEBUG }, 21:54:55 DEBUG "value": [ 21:54:55 DEBUG 1713646495, 21:54:55 DEBUG "4003078978560" 21:54:55 DEBUG "value": [ 21:54:55 DEBUG "value": [ 21:54:55 DEBUG "value": [ 21:54:55 DEBUG 1713646495, 21:54:55 DEBUG }, 21:54:55 DEBUG ] 21:54:55 DEBUG "pod": "ds-cts-0" 21:54:55 DEBUG 21:54:55 DEBUG }, 21:54:55 DEBUG "value": [ 21:54:55 DEBUG }, 21:54:55 DEBUG 21:54:55 DEBUG "3327.1666666666665" 21:54:55 DEBUG }, 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG ] 21:54:55 DEBUG "pod": "ds-cts-2" 21:54:55 DEBUG 1713646495, 21:54:55 DEBUG 1713646495, 21:54:55 DEBUG "100367328.73333333" 21:54:55 DEBUG "value": [ 21:54:55 DEBUG } 21:54:55 DEBUG }, 21:54:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 21:54:55 DEBUG [...] 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG "value": [ 21:54:55 DEBUG "pod": "ds-cts-0" 21:54:55 DEBUG ] 21:54:55 DEBUG "value": [ 21:54:55 DEBUG "instance": "10.142.0.43:9100" 21:54:55 DEBUG "pod": "ds-idrepo-2" 21:54:55 DEBUG } 21:54:55 DEBUG "instance": "10.142.0.43:9100" 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG } 21:54:55 DEBUG }, 21:54:55 DEBUG "0.5043999999999993" 21:54:55 DEBUG ] 21:54:55 DEBUG ] 21:54:55 DEBUG 1713646495, 21:54:55 DEBUG "0" 21:54:55 DEBUG "value": [ 21:54:55 DEBUG }, 21:54:55 DEBUG 21:54:55 DEBUG 1713646495, 21:54:55 DEBUG 21:54:55 DEBUG 1713646495, 21:54:55 DEBUG }, 21:54:55 DEBUG } 21:54:55 DEBUG 1713646495, 21:54:55 DEBUG }, 21:54:55 DEBUG }, 21:54:55 DEBUG "value": [ 21:54:55 DEBUG }, 21:54:55 DEBUG 21:54:55 DEBUG 21:54:55 DEBUG ] 21:54:55 DEBUG } 21:54:55 DEBUG } 21:54:55 DEBUG ] 21:54:55 DEBUG } 21:54:55 DEBUG ] 21:54:55 DEBUG 1713646495, 21:54:55 DEBUG "value": [ 21:54:55 DEBUG } 21:54:55 DEBUG 21:54:55 DEBUG "instance": "10.142.0.43:9100" 21:54:55 DEBUG "pod": "am-767f6b67b5-vbltz" 21:54:55 DEBUG "0" 21:54:55 DEBUG ] 21:54:55 DEBUG } 21:54:55 DEBUG "600138.5" 21:54:55 DEBUG "value": [ 21:54:55 DEBUG 1713646495, 21:54:55 DEBUG "value": [ 21:54:55 DEBUG 1713646495, 21:54:55 DEBUG "pod": "ds-idrepo-2" 21:54:55 DEBUG }, 21:54:55 DEBUG "value": [ 21:54:55 DEBUG "49.377" 21:54:55 DEBUG ] 21:54:55 DEBUG } 21:54:55 DEBUG ] 21:54:55 DEBUG } 21:54:55 DEBUG "9.930694534374274" 21:54:55 DEBUG 1713646495, 21:54:55 DEBUG "1025064960" 21:54:55 DEBUG }, 21:54:55 DEBUG }, 21:54:55 DEBUG "value": [ 21:54:55 DEBUG 1713646495, 21:54:55 DEBUG ] 21:54:55 DEBUG ] 21:54:55 DEBUG "0.4177666666667003" 21:54:55 DEBUG ] 21:54:55 DEBUG 1713646495, 21:54:55 DEBUG "0" 21:54:55 DEBUG "value": [ 21:54:55 DEBUG 1713646495, 21:54:55 DEBUG ] 21:54:55 DEBUG } 21:54:55 DEBUG ] 21:54:55 DEBUG } 21:54:55 DEBUG } 21:54:55 DEBUG ] 21:54:55 DEBUG ] 21:54:55 DEBUG } 21:54:55 DEBUG ] 21:54:55 DEBUG "value": [ 21:54:55 DEBUG "value": [ 21:54:55 DEBUG "35390513.04818737" 21:54:55 DEBUG } 21:54:55 DEBUG } 21:54:55 DEBUG ] 21:54:55 DEBUG } 21:54:55 DEBUG ] 21:54:55 DEBUG } 21:54:55 DEBUG "pod": "ds-cts-2" 21:54:55 DEBUG 1713646495, 21:54:55 DEBUG "0" 21:54:55 DEBUG ] 21:54:55 DEBUG } 21:54:55 DEBUG } 21:54:55 DEBUG } 21:54:55 DEBUG } 21:54:55 DEBUG ] 21:54:55 DEBUG "3317.5666666666666" 21:54:55 DEBUG } 21:54:55 DEBUG 1713646495, 21:54:55 DEBUG 1713646495, 21:54:55 DEBUG ] 21:54:55 DEBUG } 21:54:55 DEBUG ] 21:54:55 DEBUG } 21:54:55 DEBUG } 21:54:55 DEBUG ] 21:54:55 DEBUG "0.08266666666667105" 21:54:55 DEBUG ] 21:54:55 DEBUG } 21:54:55 DEBUG } 21:54:55 DEBUG "0" 21:54:55 DEBUG ] 21:54:55 DEBUG } 21:54:55 DEBUG } 21:54:55 DEBUG ] 21:54:55 DEBUG } 21:54:55 DEBUG "100371365.36666667" 21:54:55 DEBUG "3302.1" 21:54:55 DEBUG ] 21:54:55 DEBUG } 21:54:55 DEBUG } 21:54:55 DEBUG } 21:54:55 DEBUG } 21:54:55 DEBUG }, 21:54:55 DEBUG "value": [ 21:54:55 DEBUG ] 21:54:55 DEBUG ] 21:54:55 DEBUG } 21:54:55 DEBUG } 21:54:55 DEBUG } 21:54:55 DEBUG ] 21:54:55 DEBUG ] 21:54:55 DEBUG ] 21:54:55 DEBUG } 21:54:55 DEBUG } 21:54:55 DEBUG } 21:54:55 DEBUG ] 21:54:55 DEBUG } 21:54:55 DEBUG 1713646495, 21:54:55 DEBUG } 21:54:55 DEBUG } 21:54:55 DEBUG } 21:54:55 DEBUG } 21:54:55 DEBUG } 21:54:55 DEBUG ] 21:54:55 DEBUG } 21:54:55 DEBUG ] 21:54:55 DEBUG } 21:54:55 DEBUG "21.1" 21:54:55 DEBUG } 21:54:55 DEBUG } 21:54:55 DEBUG } 21:54:55 DEBUG } 21:54:55 DEBUG } 21:54:55 DEBUG } 21:54:55 DEBUG ] 21:54:55 DEBUG } 21:54:55 DEBUG ] 21:54:55 DEBUG } 21:54:55 DEBUG ] 21:54:55 DEBUG } 21:54:55 DEBUG } 21:54:55 DEBUG } 21:54:55 DEBUG } 21:54:55 DEBUG } 21:54:55 DEBUG ] 21:54:55 DEBUG } 21:54:55 DEBUG } 21:55:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713646553" 21:55:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646553" 21:55:53 ERROR [http_cmd]: ERROR 21:55:53 INFO [http_cmd]: http status code OK 21:55:53 ERROR --- status code --- 21:55:53 DEBUG --- status code --- 21:55:53 ERROR http status code is 400 (expected [200, 401, 403, 404]) 21:55:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:55:53 ERROR --- http response --- 21:55:53 DEBUG --- http response --- 21:55:53 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 21:55:53 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 21:55:53 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 21:55:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646553" 21:55:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713646553" 21:55:53 INFO [http_cmd]: http status code OK 21:55:53 DEBUG --- status code --- 21:55:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:55:53 DEBUG --- http response --- 21:55:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:55:53 DEBUG ----- output ----- 21:55:53 DEBUG { 21:55:53 DEBUG "status": "success", 21:55:53 DEBUG "data": { 21:55:53 DEBUG "resultType": "vector", 21:55:53 DEBUG "result": [ 21:55:53 DEBUG { 21:55:53 DEBUG "metric": { 21:55:53 DEBUG "pod": "ds-idrepo-0" 21:55:53 DEBUG }, 21:55:53 DEBUG "value": [ 21:55:53 DEBUG 21:55:53 DEBUG 21:55:53 DEBUG 21:55:53 DEBUG [...] 21:55:53 DEBUG 21:55:53 DEBUG 21:55:53 DEBUG 21:55:53 DEBUG "pod": "ds-cts-2" 21:55:53 DEBUG }, 21:55:53 DEBUG "value": [ 21:55:53 DEBUG 1713646553, 21:55:53 DEBUG "0" 21:55:53 DEBUG ] 21:55:53 DEBUG } 21:55:53 DEBUG ] 21:55:53 DEBUG } 21:55:53 DEBUG } 21:55:53 INFO [http_cmd]: http status code OK 21:55:53 DEBUG --- status code --- 21:55:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:55:53 DEBUG --- http response --- 21:55:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 21:55:53 DEBUG ----- output ----- 21:55:53 DEBUG { 21:55:53 DEBUG "status": "success", 21:55:53 DEBUG "data": { 21:55:53 DEBUG "resultType": "vector", 21:55:53 DEBUG "result": [ 21:55:53 DEBUG { 21:55:53 DEBUG "metric": { 21:55:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:55:53 DEBUG }, 21:55:53 DEBUG "value": [ 21:55:53 DEBUG 21:55:53 DEBUG 21:55:53 DEBUG 21:55:53 DEBUG [...] 21:55:53 DEBUG 21:55:53 DEBUG 21:55:53 DEBUG 21:55:53 DEBUG "pod": "am-767f6b67b5-9zw22" 21:55:53 DEBUG }, 21:55:53 DEBUG "value": [ 21:55:53 DEBUG 1713646553, 21:55:53 DEBUG "14596.340530500036" 21:55:53 DEBUG ] 21:55:53 DEBUG } 21:55:53 DEBUG ] 21:55:53 DEBUG } 21:55:53 DEBUG } 21:55:54 INFO 21:55:54 INFO [loop_until]: kubectl --namespace=xlou top pods 21:55:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:55:54 INFO [loop_until]: OK (rc = 0) 21:55:54 DEBUG --- stdout --- 21:55:54 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 21660m 8767Mi am-767f6b67b5-qcqz2 22464m 8886Mi am-767f6b67b5-vbltz 24215m 8937Mi ds-cts-0 19476m 12255Mi ds-cts-1 20405m 12165Mi ds-cts-2 20205m 12429Mi ds-idrepo-0 9551m 3901Mi ds-idrepo-1 9518m 3975Mi ds-idrepo-2 9462m 3878Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 4m 1238Mi lodemon-7dd48676f6-tdxl9 20m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-967d957f7-zk66v 11697m 889Mi 21:55:54 DEBUG --- stderr --- 21:55:54 DEBUG 21:55:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646555" 21:55:55 INFO [http_cmd]: http status code OK 21:55:55 DEBUG --- status code --- 21:55:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:55:55 DEBUG --- http response --- 21:55:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:55:55 DEBUG ----- output ----- 21:55:55 DEBUG { 21:55:55 DEBUG "status": "success", 21:55:55 DEBUG "data": { 21:55:55 DEBUG "resultType": "vector", 21:55:55 DEBUG "result": [ 21:55:55 DEBUG { 21:55:55 DEBUG "metric": { 21:55:55 DEBUG "pod": "am-767f6b67b5-9zw22" 21:55:55 DEBUG }, 21:55:55 DEBUG "value": [ 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG [...] 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG "pod": "am-767f6b67b5-vbltz" 21:55:55 DEBUG }, 21:55:55 DEBUG "value": [ 21:55:55 DEBUG 1713646555, 21:55:55 DEBUG "3328.7999999999997" 21:55:55 DEBUG ] 21:55:55 DEBUG } 21:55:55 DEBUG ] 21:55:55 DEBUG } 21:55:55 DEBUG } 21:55:55 INFO 21:55:55 INFO [loop_until]: kubectl --namespace=xlou top node 21:55:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:55:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646555" 21:55:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713646555" 21:55:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646555" 21:55:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646555" 21:55:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646555" 21:55:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713646555" 21:55:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646555" 21:55:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646555" 21:55:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646555" 21:55:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646555" 21:55:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646555" 21:55:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646555" 21:55:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646555" 21:55:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646555" 21:55:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646555" 21:55:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713646555" 21:55:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646555" 21:55:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646555" 21:55:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646555" 21:55:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646555" 21:55:55 INFO [http_cmd]: http status code OK 21:55:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646555" 21:55:55 INFO [http_cmd]: http status code OK 21:55:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646555" 21:55:55 INFO [http_cmd]: http status code OK 21:55:55 INFO [http_cmd]: http status code OK 21:55:55 INFO [http_cmd]: http status code OK 21:55:55 INFO [http_cmd]: http status code OK 21:55:55 INFO [http_cmd]: http status code OK 21:55:55 INFO [http_cmd]: http status code OK 21:55:55 INFO [loop_until]: OK (rc = 0) 21:55:55 INFO [http_cmd]: http status code OK 21:55:55 INFO [http_cmd]: http status code OK 21:55:55 INFO [http_cmd]: http status code OK 21:55:55 INFO [http_cmd]: http status code OK 21:55:55 INFO [http_cmd]: http status code OK 21:55:55 INFO [http_cmd]: http status code OK 21:55:55 INFO [http_cmd]: http status code OK 21:55:55 DEBUG --- status code --- 21:55:55 DEBUG --- status code --- 21:55:55 INFO [http_cmd]: http status code OK 21:55:55 INFO [http_cmd]: http status code OK 21:55:55 DEBUG --- status code --- 21:55:55 INFO [http_cmd]: http status code OK 21:55:55 DEBUG --- status code --- 21:55:55 DEBUG --- status code --- 21:55:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:55:55 DEBUG --- status code --- 21:55:55 DEBUG --- status code --- 21:55:55 DEBUG --- stdout --- 21:55:55 DEBUG --- status code --- 21:55:55 DEBUG --- status code --- 21:55:55 DEBUG --- status code --- 21:55:55 DEBUG --- status code --- 21:55:55 DEBUG --- status code --- 21:55:55 DEBUG --- status code --- 21:55:55 DEBUG --- status code --- 21:55:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:55:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:55:55 DEBUG --- status code --- 21:55:55 DEBUG --- status code --- 21:55:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:55:55 DEBUG --- status code --- 21:55:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:55:55 DEBUG --- http response --- 21:55:55 DEBUG --- status code --- 21:55:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:55:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:55:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:55:55 DEBUG --- http response --- 21:55:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:55:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:55:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:55:55 DEBUG --- http response --- 21:55:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:55:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:55:55 DEBUG --- http response --- 21:55:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:55:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:55:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:55:55 DEBUG --- http response --- 21:55:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:55:55 DEBUG --- http response --- 21:55:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:55:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:55:55 DEBUG --- http response --- 21:55:55 DEBUG --- http response --- 21:55:55 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 61m 0% 6223Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 10034m 33% 7665Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 21134m 70% 11494Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 85m 0% 3372Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 19870m 66% 15496Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 159m 0% 5246Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 22332m 74% 11782Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19750m 66% 3055Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 23826m 79% 11188Mi 10% gke-xlou-cdm-ds-0f601751-j488 10296m 34% 7293Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19875m 66% 17239Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 10225m 34% 7946Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20260m 67% 15129Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 12090m 76% 8814Mi 15% 21:55:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:55:55 DEBUG --- http response --- 21:55:55 DEBUG --- http response --- 21:55:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:55:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:55:55 DEBUG --- http response --- 21:55:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:55:55 DEBUG ----- output ----- 21:55:55 DEBUG --- http response --- 21:55:55 DEBUG --- http response --- 21:55:55 DEBUG --- http response --- 21:55:55 INFO [http_cmd]: http status code OK 21:55:55 INFO [http_cmd]: http status code OK 21:55:55 INFO [http_cmd]: http status code OK 21:55:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:55:55 DEBUG --- http response --- 21:55:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:55:55 DEBUG ----- output ----- 21:55:55 INFO [http_cmd]: http status code OK 21:55:55 DEBUG --- http response --- 21:55:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:55:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:55:55 DEBUG --- stderr --- 21:55:55 DEBUG ----- output ----- 21:55:55 DEBUG { 21:55:55 DEBUG "status": "success", 21:55:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:55:55 DEBUG ----- output ----- 21:55:55 DEBUG ----- output ----- 21:55:55 DEBUG --- http response --- 21:55:55 DEBUG { 21:55:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:55:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:55:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:55:55 DEBUG --- status code --- 21:55:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:55:55 DEBUG --- status code --- 21:55:55 DEBUG ----- output ----- 21:55:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:55:55 DEBUG ----- output ----- 21:55:55 DEBUG { 21:55:55 DEBUG --- status code --- 21:55:55 DEBUG ----- output ----- 21:55:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:55:55 DEBUG ----- output ----- 21:55:55 DEBUG 21:55:55 DEBUG --- http response --- 21:55:55 DEBUG "data": { 21:55:55 DEBUG ----- output ----- 21:55:55 DEBUG { 21:55:55 DEBUG { 21:55:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:55:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:55:55 DEBUG "status": "success", 21:55:55 DEBUG ----- output ----- 21:55:55 DEBUG ----- output ----- 21:55:55 DEBUG ----- output ----- 21:55:55 DEBUG --- http response --- 21:55:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:55:55 DEBUG --- http response --- 21:55:55 DEBUG ----- output ----- 21:55:55 DEBUG { 21:55:55 DEBUG { 21:55:55 DEBUG "status": "success", 21:55:55 DEBUG --- status code --- 21:55:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:55:55 DEBUG ----- output ----- 21:55:55 DEBUG { 21:55:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:55:55 DEBUG ----- output ----- 21:55:55 DEBUG { 21:55:55 DEBUG "status": "success", 21:55:55 DEBUG "status": "success", 21:55:55 DEBUG ----- output ----- 21:55:55 DEBUG ----- output ----- 21:55:55 DEBUG "data": { 21:55:55 DEBUG { 21:55:55 DEBUG { 21:55:55 DEBUG "status": "success", 21:55:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:55:55 DEBUG { 21:55:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:55:55 DEBUG "status": "success", 21:55:55 DEBUG "status": "success", 21:55:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:55:55 DEBUG "data": { 21:55:55 DEBUG --- http response --- 21:55:55 DEBUG { 21:55:55 DEBUG "status": "success", 21:55:55 DEBUG "status": "success", 21:55:55 DEBUG "data": { 21:55:55 DEBUG { 21:55:55 DEBUG "resultType": "vector", 21:55:55 DEBUG "data": { 21:55:55 DEBUG "data": { 21:55:55 DEBUG { 21:55:55 DEBUG { 21:55:55 DEBUG "resultType": "vector", 21:55:55 DEBUG "status": "success", 21:55:55 DEBUG { 21:55:55 DEBUG "data": { 21:55:55 DEBUG ----- output ----- 21:55:55 DEBUG "status": "success", 21:55:55 DEBUG ----- output ----- 21:55:55 DEBUG "data": { 21:55:55 DEBUG "data": { 21:55:55 DEBUG --- http response --- 21:55:55 DEBUG "resultType": "vector", 21:55:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:55:55 DEBUG "data": { 21:55:55 DEBUG { 21:55:55 DEBUG "status": "success", 21:55:55 DEBUG "resultType": "vector", 21:55:55 DEBUG "status": "success", 21:55:55 DEBUG "result": [ 21:55:55 DEBUG "resultType": "vector", 21:55:55 DEBUG "result": [ 21:55:55 DEBUG { 21:55:55 DEBUG "status": "success", 21:55:55 DEBUG "result": [ 21:55:55 DEBUG "data": { 21:55:55 DEBUG "status": "success", 21:55:55 DEBUG "resultType": "vector", 21:55:55 DEBUG { 21:55:55 DEBUG "data": { 21:55:55 DEBUG "resultType": "vector", 21:55:55 DEBUG "result": [ 21:55:55 DEBUG "resultType": "vector", 21:55:55 DEBUG "result": [ 21:55:55 DEBUG { 21:55:55 DEBUG ----- output ----- 21:55:55 DEBUG "resultType": "vector", 21:55:55 DEBUG "data": { 21:55:55 DEBUG "result": [ 21:55:55 DEBUG "status": "success", 21:55:55 DEBUG "data": { 21:55:55 DEBUG { 21:55:55 DEBUG "status": "success", 21:55:55 DEBUG "metric": { 21:55:55 DEBUG "resultType": "vector", 21:55:55 DEBUG "data": { 21:55:55 DEBUG { 21:55:55 DEBUG "resultType": "vector", 21:55:55 DEBUG "data": { 21:55:55 DEBUG "result": [ 21:55:55 DEBUG "status": "success", 21:55:55 DEBUG { 21:55:55 DEBUG { 21:55:55 DEBUG "resultType": "vector", 21:55:55 DEBUG "result": [ 21:55:55 DEBUG "metric": { 21:55:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:55:55 DEBUG { 21:55:55 DEBUG "result": [ 21:55:55 DEBUG "resultType": "vector", 21:55:55 DEBUG { 21:55:55 DEBUG "data": { 21:55:55 DEBUG "resultType": "vector", 21:55:55 DEBUG "metric": { 21:55:55 DEBUG "data": { 21:55:55 DEBUG "pod": "ds-idrepo-1" 21:55:55 DEBUG "result": [ 21:55:55 DEBUG "resultType": "vector", 21:55:55 DEBUG "metric": { 21:55:55 DEBUG "result": [ 21:55:55 DEBUG "resultType": "vector", 21:55:55 DEBUG { 21:55:55 DEBUG "data": { 21:55:55 DEBUG "metric": { 21:55:55 DEBUG "status": "success", 21:55:55 DEBUG "result": [ 21:55:55 DEBUG { 21:55:55 DEBUG "instance": "10.142.0.107:9100" 21:55:55 DEBUG ----- output ----- 21:55:55 DEBUG "status": "success", 21:55:55 DEBUG { 21:55:55 DEBUG "result": [ 21:55:55 DEBUG "metric": { 21:55:55 DEBUG "pod": "ds-idrepo-0" 21:55:55 DEBUG "result": [ 21:55:55 DEBUG "pod": "am-767f6b67b5-9zw22" 21:55:55 DEBUG "resultType": "vector", 21:55:55 DEBUG }, 21:55:55 DEBUG { 21:55:55 DEBUG "result": [ 21:55:55 DEBUG "pod": "am-767f6b67b5-9zw22" 21:55:55 DEBUG { 21:55:55 DEBUG "result": [ 21:55:55 DEBUG "metric": { 21:55:55 DEBUG "resultType": "vector", 21:55:55 DEBUG "instance": "10.142.0.107:9100" 21:55:55 DEBUG "data": { 21:55:55 DEBUG { 21:55:55 DEBUG "metric": { 21:55:55 DEBUG }, 21:55:55 DEBUG { 21:55:55 DEBUG "status": "success", 21:55:55 DEBUG "metric": { 21:55:55 DEBUG { 21:55:55 DEBUG }, 21:55:55 DEBUG { 21:55:55 DEBUG "resultType": "vector", 21:55:55 DEBUG "result": [ 21:55:55 DEBUG "result": [ 21:55:55 DEBUG "value": [ 21:55:55 DEBUG "metric": { 21:55:55 DEBUG { 21:55:55 DEBUG }, 21:55:55 DEBUG "value": [ 21:55:55 DEBUG 21:55:55 DEBUG "pod": "ds-cts-0" 21:55:55 DEBUG "result": [ 21:55:55 DEBUG }, 21:55:55 DEBUG "resultType": "vector", 21:55:55 DEBUG "instance": "10.142.0.107:9100" 21:55:55 DEBUG "metric": { 21:55:55 DEBUG "value": [ 21:55:55 DEBUG 21:55:55 DEBUG "data": { 21:55:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 21:55:55 DEBUG "metric": { 21:55:55 DEBUG "value": [ 21:55:55 DEBUG "metric": { 21:55:55 DEBUG { 21:55:55 DEBUG { 21:55:55 DEBUG }, 21:55:55 DEBUG 21:55:55 DEBUG "pod": "am-767f6b67b5-9zw22" 21:55:55 DEBUG "metric": { 21:55:55 DEBUG { 21:55:55 DEBUG 21:55:55 DEBUG "metric": { 21:55:55 DEBUG }, 21:55:55 DEBUG { 21:55:55 DEBUG "value": [ 21:55:55 DEBUG 21:55:55 DEBUG "result": [ 21:55:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 21:55:55 DEBUG 21:55:55 DEBUG "resultType": "vector", 21:55:55 DEBUG "data": { 21:55:55 DEBUG }, 21:55:55 DEBUG "pod": "am-767f6b67b5-9zw22" 21:55:55 DEBUG 21:55:55 DEBUG "pod": "ds-cts-0" 21:55:55 DEBUG "metric": { 21:55:55 DEBUG "metric": { 21:55:55 DEBUG "value": [ 21:55:55 DEBUG 21:55:55 DEBUG }, 21:55:55 DEBUG "pod": "ds-idrepo-0" 21:55:55 DEBUG "metric": { 21:55:55 DEBUG 21:55:55 DEBUG [...] 21:55:55 DEBUG 21:55:55 DEBUG "value": [ 21:55:55 DEBUG 21:55:55 DEBUG { 21:55:55 DEBUG }, 21:55:55 DEBUG }, 21:55:55 DEBUG 21:55:55 DEBUG "result": [ 21:55:55 DEBUG "resultType": "vector", 21:55:55 DEBUG "value": [ 21:55:55 DEBUG }, 21:55:55 DEBUG 21:55:55 DEBUG }, 21:55:55 DEBUG "pod": "ds-idrepo-0" 21:55:55 DEBUG "pod": "ds-idrepo-1" 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG [...] 21:55:55 DEBUG }, 21:55:55 DEBUG "pod": "ds-cts-0" 21:55:55 DEBUG "metric": { 21:55:55 DEBUG 21:55:55 DEBUG "pod": "ds-idrepo-1" 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG "metric": { 21:55:55 DEBUG "value": [ 21:55:55 DEBUG "value": [ 21:55:55 DEBUG [...] 21:55:55 DEBUG { 21:55:55 DEBUG "result": [ 21:55:55 DEBUG 21:55:55 DEBUG "value": [ 21:55:55 DEBUG 21:55:55 DEBUG "value": [ 21:55:55 DEBUG }, 21:55:55 DEBUG }, 21:55:55 DEBUG 21:55:55 DEBUG "value": [ 21:55:55 DEBUG 21:55:55 DEBUG "value": [ 21:55:55 DEBUG }, 21:55:55 DEBUG "pod": "am-767f6b67b5-9zw22" 21:55:55 DEBUG 21:55:55 DEBUG }, 21:55:55 DEBUG 21:55:55 DEBUG [...] 21:55:55 DEBUG "instance": "10.142.0.107:9100" 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG "metric": { 21:55:55 DEBUG { 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG [...] 21:55:55 DEBUG 21:55:55 DEBUG "value": [ 21:55:55 DEBUG "value": [ 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG "value": [ 21:55:55 DEBUG }, 21:55:55 DEBUG "pod": "am-767f6b67b5-vbltz" 21:55:55 DEBUG "value": [ 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG }, 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG "pod": "am-767f6b67b5-9zw22" 21:55:55 DEBUG }, 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG [...] 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG "value": [ 21:55:55 DEBUG 21:55:55 DEBUG }, 21:55:55 DEBUG [...] 21:55:55 DEBUG 21:55:55 DEBUG "value": [ 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG "metric": { 21:55:55 DEBUG "value": [ 21:55:55 DEBUG [...] 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG "value": [ 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG [...] 21:55:55 DEBUG [...] 21:55:55 DEBUG "instance": "10.142.0.43:9100" 21:55:55 DEBUG "instance": "10.142.0.107:9100" 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG [...] 21:55:55 DEBUG "pod": "ds-idrepo-2" 21:55:55 DEBUG 21:55:55 DEBUG [...] 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG [...] 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG 1713646555, 21:55:55 DEBUG 21:55:55 DEBUG "instance": "10.142.0.43:9100" 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG }, 21:55:55 DEBUG }, 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG }, 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG [...] 21:55:55 DEBUG 21:55:55 DEBUG [...] 21:55:55 DEBUG 21:55:55 DEBUG [...] 21:55:55 DEBUG 21:55:55 DEBUG [...] 21:55:55 DEBUG "49.377" 21:55:55 DEBUG 21:55:55 DEBUG }, 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG "value": [ 21:55:55 DEBUG "value": [ 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG "value": [ 21:55:55 DEBUG 21:55:55 DEBUG [...] 21:55:55 DEBUG "pod": "am-767f6b67b5-vbltz" 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG }, 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG ] 21:55:55 DEBUG "pod": "ds-cts-2" 21:55:55 DEBUG "value": [ 21:55:55 DEBUG [...] 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG 1713646555, 21:55:55 DEBUG 21:55:55 DEBUG [...] 21:55:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 21:55:55 DEBUG 21:55:55 DEBUG 1713646555, 21:55:55 DEBUG "pod": "ds-cts-2" 21:55:55 DEBUG 21:55:55 DEBUG }, 21:55:55 DEBUG 21:55:55 DEBUG "pod": "am-767f6b67b5-vbltz" 21:55:55 DEBUG 21:55:55 DEBUG "value": [ 21:55:55 DEBUG [...] 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG } 21:55:55 DEBUG }, 21:55:55 DEBUG 1713646555, 21:55:55 DEBUG 21:55:55 DEBUG "instance": "10.142.0.43:9100" 21:55:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 21:55:55 DEBUG "4008353721344" 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG }, 21:55:55 DEBUG "pod": "am-767f6b67b5-vbltz" 21:55:55 DEBUG "0" 21:55:55 DEBUG }, 21:55:55 DEBUG 21:55:55 DEBUG "value": [ 21:55:55 DEBUG 21:55:55 DEBUG }, 21:55:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:55:55 DEBUG 1713646555, 21:55:55 DEBUG 21:55:55 DEBUG "pod": "ds-cts-2" 21:55:55 DEBUG 21:55:55 DEBUG ] 21:55:55 DEBUG "value": [ 21:55:55 DEBUG "100247693.9" 21:55:55 DEBUG 21:55:55 DEBUG }, 21:55:55 DEBUG }, 21:55:55 DEBUG ] 21:55:55 DEBUG 21:55:55 DEBUG 21:55:55 DEBUG "value": [ 21:55:55 DEBUG }, 21:55:55 DEBUG ] 21:55:55 DEBUG "value": [ 21:55:55 DEBUG 21:55:55 DEBUG 1713646555, 21:55:55 DEBUG "pod": "ds-idrepo-2" 21:55:55 DEBUG "value": [ 21:55:55 DEBUG }, 21:55:55 DEBUG "0" 21:55:55 DEBUG 21:55:55 DEBUG }, 21:55:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:55:55 DEBUG } 21:55:55 DEBUG 1713646555, 21:55:55 DEBUG ] 21:55:55 DEBUG 21:55:55 DEBUG "instance": "10.142.0.43:9100" 21:55:55 DEBUG "value": [ 21:55:55 DEBUG } 21:55:55 DEBUG [...] 21:55:55 DEBUG 21:55:55 DEBUG 1713646555, 21:55:55 DEBUG "value": [ 21:55:55 DEBUG } 21:55:55 DEBUG 1713646555, 21:55:55 DEBUG "pod": "ds-idrepo-2" 21:55:55 DEBUG "0" 21:55:55 DEBUG }, 21:55:55 DEBUG 1713646555, 21:55:55 DEBUG "value": [ 21:55:55 DEBUG ] 21:55:55 DEBUG 21:55:55 DEBUG "value": [ 21:55:55 DEBUG }, 21:55:55 DEBUG } 21:55:55 DEBUG "0" 21:55:55 DEBUG } 21:55:55 DEBUG }, 21:55:55 DEBUG "value": [ 21:55:55 DEBUG 1713646555, 21:55:55 DEBUG ] 21:55:55 DEBUG 21:55:55 DEBUG "pod": "am-767f6b67b5-vbltz" 21:55:55 DEBUG "11.878970840336052" 21:55:55 DEBUG 1713646555, 21:55:55 DEBUG ] 21:55:55 DEBUG "0.573" 21:55:55 DEBUG }, 21:55:55 DEBUG ] 21:55:55 DEBUG "value": [ 21:55:55 DEBUG 1713646555, 21:55:55 DEBUG 1713646555, 21:55:55 DEBUG } 21:55:55 DEBUG "pod": "am-767f6b67b5-vbltz" 21:55:55 DEBUG 1713646555, 21:55:55 DEBUG "value": [ 21:55:55 DEBUG ] 21:55:55 DEBUG ] 21:55:55 DEBUG "value": [ 21:55:55 DEBUG 1713646555, 21:55:55 DEBUG "1027760128" 21:55:55 DEBUG } 21:55:55 DEBUG 21:55:55 DEBUG }, 21:55:55 DEBUG ] 21:55:55 DEBUG "3325.5" 21:55:55 DEBUG } 21:55:55 DEBUG ] 21:55:55 DEBUG "value": [ 21:55:55 DEBUG } 21:55:55 DEBUG "0" 21:55:55 DEBUG "600072" 21:55:55 DEBUG "0.00003193854200715578" 21:55:55 DEBUG ] 21:55:55 DEBUG }, 21:55:55 DEBUG "28.53333333333333" 21:55:55 DEBUG 1713646555, 21:55:55 DEBUG } 21:55:55 DEBUG } 21:55:55 DEBUG 1713646555, 21:55:55 DEBUG "0.38396666666659296" 21:55:55 DEBUG ] 21:55:55 DEBUG } 21:55:55 DEBUG 21:55:55 DEBUG "value": [ 21:55:55 DEBUG } 21:55:55 DEBUG ] 21:55:55 DEBUG } 21:55:55 DEBUG } 21:55:55 DEBUG 1713646555, 21:55:55 DEBUG ] 21:55:55 DEBUG ] 21:55:55 DEBUG ] 21:55:55 DEBUG } 21:55:55 DEBUG } 21:55:55 DEBUG "value": [ 21:55:55 DEBUG ] 21:55:55 DEBUG "0" 21:55:55 DEBUG ] 21:55:55 DEBUG } 21:55:55 DEBUG "100251234.6" 21:55:55 DEBUG ] 21:55:55 DEBUG } 21:55:55 DEBUG "instance": "10.142.0.43:9100" 21:55:55 DEBUG 1713646555, 21:55:55 DEBUG ] 21:55:55 DEBUG } 21:55:55 DEBUG ] 21:55:55 DEBUG "0" 21:55:55 DEBUG } 21:55:55 DEBUG } 21:55:55 DEBUG ] 21:55:55 DEBUG ] 21:55:55 DEBUG } 21:55:55 DEBUG 1713646555, 21:55:55 DEBUG } 21:55:55 DEBUG ] 21:55:55 DEBUG } 21:55:55 DEBUG } 21:55:55 DEBUG ] 21:55:55 DEBUG ] 21:55:55 DEBUG }, 21:55:55 DEBUG "3334.5666666666666" 21:55:55 DEBUG } 21:55:55 DEBUG ] 21:55:55 DEBUG } 21:55:55 DEBUG ] 21:55:55 DEBUG } 21:55:55 DEBUG ] 21:55:55 DEBUG } 21:55:55 DEBUG ] 21:55:55 DEBUG "3331.866666666667" 21:55:55 DEBUG ] 21:55:55 DEBUG } 21:55:55 DEBUG } 21:55:55 DEBUG ] 21:55:55 DEBUG } 21:55:55 DEBUG } 21:55:55 DEBUG "value": [ 21:55:55 DEBUG ] 21:55:55 DEBUG } 21:55:55 DEBUG } 21:55:55 DEBUG } 21:55:55 DEBUG } 21:55:55 DEBUG } 21:55:55 DEBUG } 21:55:55 DEBUG } 21:55:55 DEBUG ] 21:55:55 DEBUG } 21:55:55 DEBUG ] 21:55:55 DEBUG } 21:55:55 DEBUG } 21:55:55 DEBUG ] 21:55:55 DEBUG } 21:55:55 DEBUG } 21:55:55 DEBUG } 21:55:55 DEBUG ] 21:55:55 DEBUG } 21:55:55 DEBUG } 21:55:55 DEBUG } 21:55:55 DEBUG } 21:55:55 DEBUG } 21:55:55 DEBUG } 21:55:55 DEBUG } 21:55:55 DEBUG } 21:55:55 DEBUG 1713646555, 21:55:55 DEBUG ] 21:55:55 DEBUG } 21:55:55 DEBUG } 21:55:55 DEBUG ] 21:55:55 DEBUG } 21:55:55 DEBUG "0.07333333333334811" 21:55:55 DEBUG } 21:55:55 DEBUG } 21:55:55 DEBUG ] 21:55:55 DEBUG } 21:55:55 DEBUG } 21:55:55 DEBUG } 21:55:55 DEBUG ] 21:55:55 DEBUG } 21:55:55 DEBUG } 21:56:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713646613" 21:56:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646613" 21:56:53 INFO [http_cmd]: http status code OK 21:56:53 ERROR [http_cmd]: ERROR 21:56:53 DEBUG --- status code --- 21:56:53 ERROR --- status code --- 21:56:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:56:53 ERROR http status code is 400 (expected [200, 401, 403, 404]) 21:56:53 DEBUG --- http response --- 21:56:53 ERROR --- http response --- 21:56:53 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 21:56:53 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 21:56:53 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 21:56:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713646613" 21:56:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646613" 21:56:53 INFO [http_cmd]: http status code OK 21:56:53 DEBUG --- status code --- 21:56:53 INFO [http_cmd]: http status code OK 21:56:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:56:53 DEBUG --- status code --- 21:56:53 DEBUG --- http response --- 21:56:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:56:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:56:53 DEBUG --- http response --- 21:56:53 DEBUG ----- output ----- 21:56:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 21:56:53 DEBUG { 21:56:53 DEBUG ----- output ----- 21:56:53 DEBUG "status": "success", 21:56:53 DEBUG { 21:56:53 DEBUG "status": "success", 21:56:53 DEBUG "data": { 21:56:53 DEBUG "data": { 21:56:53 DEBUG "resultType": "vector", 21:56:53 DEBUG "resultType": "vector", 21:56:53 DEBUG "result": [ 21:56:53 DEBUG "result": [ 21:56:53 DEBUG { 21:56:53 DEBUG { 21:56:53 DEBUG "metric": { 21:56:53 DEBUG "metric": { 21:56:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:56:53 DEBUG }, 21:56:53 DEBUG "pod": "ds-idrepo-0" 21:56:53 DEBUG "value": [ 21:56:53 DEBUG }, 21:56:53 DEBUG 21:56:53 DEBUG "value": [ 21:56:53 DEBUG 21:56:53 DEBUG 21:56:53 DEBUG 21:56:53 DEBUG 21:56:53 DEBUG [...] 21:56:53 DEBUG 21:56:53 DEBUG 21:56:53 DEBUG [...] 21:56:53 DEBUG 21:56:53 DEBUG 21:56:53 DEBUG 21:56:53 DEBUG 21:56:53 DEBUG "pod": "overseer-0-967d957f7-zk66v" 21:56:53 DEBUG 21:56:53 DEBUG }, 21:56:53 DEBUG "pod": "ds-cts-2" 21:56:53 DEBUG "value": [ 21:56:53 DEBUG }, 21:56:53 DEBUG 1713646613, 21:56:53 DEBUG "value": [ 21:56:53 DEBUG "0" 21:56:53 DEBUG 1713646613, 21:56:53 DEBUG ] 21:56:53 DEBUG "0" 21:56:53 DEBUG } 21:56:53 DEBUG ] 21:56:53 DEBUG ] 21:56:53 DEBUG } 21:56:53 DEBUG } 21:56:53 DEBUG ] 21:56:53 DEBUG } 21:56:53 DEBUG } 21:56:53 DEBUG } 21:56:54 INFO 21:56:54 INFO [loop_until]: kubectl --namespace=xlou top pods 21:56:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:56:54 INFO [loop_until]: OK (rc = 0) 21:56:54 DEBUG --- stdout --- 21:56:54 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 21506m 8767Mi am-767f6b67b5-qcqz2 21728m 8888Mi am-767f6b67b5-vbltz 24362m 8948Mi ds-cts-0 20362m 12251Mi ds-cts-1 19863m 12261Mi ds-cts-2 20680m 12429Mi ds-idrepo-0 9216m 3906Mi ds-idrepo-1 9421m 3974Mi ds-idrepo-2 9486m 3878Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-7dd48676f6-tdxl9 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-967d957f7-zk66v 11684m 895Mi 21:56:54 DEBUG --- stderr --- 21:56:54 DEBUG 21:56:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646615" 21:56:55 INFO [http_cmd]: http status code OK 21:56:55 DEBUG --- status code --- 21:56:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:56:55 DEBUG --- http response --- 21:56:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:56:55 DEBUG ----- output ----- 21:56:55 DEBUG { 21:56:55 DEBUG "status": "success", 21:56:55 DEBUG "data": { 21:56:55 DEBUG "resultType": "vector", 21:56:55 DEBUG "result": [ 21:56:55 DEBUG { 21:56:55 DEBUG "metric": { 21:56:55 DEBUG "pod": "am-767f6b67b5-9zw22" 21:56:55 DEBUG }, 21:56:55 DEBUG "value": [ 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG [...] 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG "pod": "am-767f6b67b5-vbltz" 21:56:55 DEBUG }, 21:56:55 DEBUG "value": [ 21:56:55 DEBUG 1713646615, 21:56:55 DEBUG "3336.4333333333334" 21:56:55 DEBUG ] 21:56:55 DEBUG } 21:56:55 DEBUG ] 21:56:55 DEBUG } 21:56:55 DEBUG } 21:56:55 INFO 21:56:55 INFO [loop_until]: kubectl --namespace=xlou top node 21:56:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:56:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713646615" 21:56:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646615" 21:56:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646615" 21:56:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646615" 21:56:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646615" 21:56:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646615" 21:56:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646615" 21:56:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713646615" 21:56:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646615" 21:56:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646615" 21:56:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646615" 21:56:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646615" 21:56:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646615" 21:56:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713646615" 21:56:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646615" 21:56:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646615" 21:56:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646615" 21:56:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646615" 21:56:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646615" 21:56:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646615" 21:56:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646615" 21:56:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646615" 21:56:55 INFO [http_cmd]: http status code OK 21:56:55 INFO [http_cmd]: http status code OK 21:56:55 INFO [http_cmd]: http status code OK 21:56:55 INFO [http_cmd]: http status code OK 21:56:55 INFO [loop_until]: OK (rc = 0) 21:56:55 INFO [http_cmd]: http status code OK 21:56:55 INFO [http_cmd]: http status code OK 21:56:55 INFO [http_cmd]: http status code OK 21:56:55 INFO [http_cmd]: http status code OK 21:56:55 INFO [http_cmd]: http status code OK 21:56:55 INFO [http_cmd]: http status code OK 21:56:55 INFO [http_cmd]: http status code OK 21:56:55 INFO [http_cmd]: http status code OK 21:56:55 INFO [http_cmd]: http status code OK 21:56:55 INFO [http_cmd]: http status code OK 21:56:55 INFO [http_cmd]: http status code OK 21:56:55 INFO [http_cmd]: http status code OK 21:56:55 DEBUG --- status code --- 21:56:55 INFO [http_cmd]: http status code OK 21:56:55 DEBUG --- status code --- 21:56:55 DEBUG --- status code --- 21:56:55 INFO [http_cmd]: http status code OK 21:56:55 DEBUG --- status code --- 21:56:55 INFO [http_cmd]: http status code OK 21:56:55 DEBUG --- stdout --- 21:56:55 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 61m 0% 6235Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 9999m 33% 7666Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 21670m 72% 11489Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 79m 0% 3365Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 20299m 67% 15489Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 157m 0% 5236Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 22194m 74% 11776Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19578m 65% 3049Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 24700m 82% 11189Mi 10% gke-xlou-cdm-ds-0f601751-j488 10327m 34% 7287Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19870m 66% 17347Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 10201m 34% 7943Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20501m 68% 15124Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 12008m 75% 8816Mi 15% 21:56:55 DEBUG --- status code --- 21:56:55 DEBUG --- status code --- 21:56:55 DEBUG --- status code --- 21:56:55 DEBUG --- status code --- 21:56:55 DEBUG --- status code --- 21:56:55 DEBUG --- status code --- 21:56:55 INFO [http_cmd]: http status code OK 21:56:55 DEBUG --- status code --- 21:56:55 DEBUG --- status code --- 21:56:55 DEBUG --- status code --- 21:56:55 DEBUG --- status code --- 21:56:55 DEBUG --- status code --- 21:56:55 DEBUG --- status code --- 21:56:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:56:55 DEBUG --- status code --- 21:56:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:56:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:56:55 DEBUG --- status code --- 21:56:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:56:55 DEBUG --- status code --- 21:56:55 DEBUG --- stderr --- 21:56:55 INFO [http_cmd]: http status code OK 21:56:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:56:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:56:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:56:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:56:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:56:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:56:55 DEBUG --- status code --- 21:56:55 INFO [http_cmd]: http status code OK 21:56:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:56:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:56:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:56:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:56:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:56:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:56:55 DEBUG --- http response --- 21:56:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:56:55 DEBUG --- http response --- 21:56:55 DEBUG --- http response --- 21:56:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:56:55 DEBUG --- http response --- 21:56:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:56:55 DEBUG 21:56:55 DEBUG --- status code --- 21:56:55 DEBUG --- http response --- 21:56:55 DEBUG --- http response --- 21:56:55 DEBUG --- http response --- 21:56:55 DEBUG --- http response --- 21:56:55 DEBUG --- http response --- 21:56:55 DEBUG --- http response --- 21:56:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:56:55 DEBUG --- status code --- 21:56:55 DEBUG --- http response --- 21:56:55 DEBUG --- http response --- 21:56:55 DEBUG --- http response --- 21:56:55 DEBUG --- http response --- 21:56:55 DEBUG --- http response --- 21:56:55 DEBUG --- http response --- 21:56:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:56:55 DEBUG --- http response --- 21:56:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:56:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:56:55 DEBUG --- http response --- 21:56:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:56:55 DEBUG --- http response --- 21:56:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:56:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:56:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:56:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:56:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:56:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:56:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:56:55 DEBUG --- http response --- 21:56:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:56:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 21:56:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:56:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:56:55 DEBUG ----- output ----- 21:56:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:56:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:56:55 DEBUG ----- output ----- 21:56:55 DEBUG ----- output ----- 21:56:55 DEBUG ----- output ----- 21:56:55 DEBUG { 21:56:55 DEBUG ----- output ----- 21:56:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:56:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:56:55 DEBUG --- http response --- 21:56:55 DEBUG ----- output ----- 21:56:55 DEBUG ----- output ----- 21:56:55 DEBUG ----- output ----- 21:56:55 DEBUG ----- output ----- 21:56:55 DEBUG ----- output ----- 21:56:55 DEBUG ----- output ----- 21:56:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:56:55 DEBUG --- http response --- 21:56:55 DEBUG ----- output ----- 21:56:55 DEBUG ----- output ----- 21:56:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:56:55 DEBUG { 21:56:55 DEBUG ----- output ----- 21:56:55 DEBUG ----- output ----- 21:56:55 DEBUG { 21:56:55 DEBUG { 21:56:55 DEBUG "status": "success", 21:56:55 DEBUG { 21:56:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:56:55 DEBUG ----- output ----- 21:56:55 DEBUG ----- output ----- 21:56:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:56:55 DEBUG { 21:56:55 DEBUG { 21:56:55 DEBUG { 21:56:55 DEBUG { 21:56:55 DEBUG { 21:56:55 DEBUG { 21:56:55 DEBUG ----- output ----- 21:56:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:56:55 DEBUG { 21:56:55 DEBUG { 21:56:55 DEBUG ----- output ----- 21:56:55 DEBUG "status": "success", 21:56:55 DEBUG { 21:56:55 DEBUG "status": "success", 21:56:55 DEBUG "status": "success", 21:56:55 DEBUG "status": "success", 21:56:55 DEBUG "data": { 21:56:55 DEBUG "status": "success", 21:56:55 DEBUG ----- output ----- 21:56:55 DEBUG { 21:56:55 DEBUG { 21:56:55 DEBUG ----- output ----- 21:56:55 DEBUG { 21:56:55 DEBUG "status": "success", 21:56:55 DEBUG "status": "success", 21:56:55 DEBUG "status": "success", 21:56:55 DEBUG "status": "success", 21:56:55 DEBUG "status": "success", 21:56:55 DEBUG { 21:56:55 DEBUG ----- output ----- 21:56:55 DEBUG "status": "success", 21:56:55 DEBUG "status": "success", 21:56:55 DEBUG { 21:56:55 DEBUG "data": { 21:56:55 DEBUG "data": { 21:56:55 DEBUG { 21:56:55 DEBUG "data": { 21:56:55 DEBUG "data": { 21:56:55 DEBUG "resultType": "vector", 21:56:55 DEBUG "data": { 21:56:55 DEBUG { 21:56:55 DEBUG "status": "success", 21:56:55 DEBUG "status": "success", 21:56:55 DEBUG "status": "success", 21:56:55 DEBUG "data": { 21:56:55 DEBUG "resultType": "vector", 21:56:55 DEBUG "result": [ 21:56:55 DEBUG "data": { 21:56:55 DEBUG "data": { 21:56:55 DEBUG "data": { 21:56:55 DEBUG "status": "success", 21:56:55 DEBUG { 21:56:55 DEBUG "data": { 21:56:55 DEBUG "resultType": "vector", 21:56:55 DEBUG "status": "success", 21:56:55 DEBUG "resultType": "vector", 21:56:55 DEBUG "result": [ 21:56:55 DEBUG "status": "success", 21:56:55 DEBUG "resultType": "vector", 21:56:55 DEBUG "resultType": "vector", 21:56:55 DEBUG "result": [ 21:56:55 DEBUG "resultType": "vector", 21:56:55 DEBUG "status": "success", 21:56:55 DEBUG "data": { 21:56:55 DEBUG "data": { 21:56:55 DEBUG "data": { 21:56:55 DEBUG "status": "success", 21:56:55 DEBUG { 21:56:55 DEBUG "resultType": "vector", 21:56:55 DEBUG "resultType": "vector", 21:56:55 DEBUG "resultType": "vector", 21:56:55 DEBUG "data": { 21:56:55 DEBUG "data": { 21:56:55 DEBUG "status": "success", 21:56:55 DEBUG "data": { 21:56:55 DEBUG "result": [ 21:56:55 DEBUG "data": { 21:56:55 DEBUG { 21:56:55 DEBUG "resultType": "vector", 21:56:55 DEBUG "data": { 21:56:55 DEBUG "result": [ 21:56:55 DEBUG "result": [ 21:56:55 DEBUG { 21:56:55 DEBUG "result": [ 21:56:55 DEBUG "data": { 21:56:55 DEBUG "resultType": "vector", 21:56:55 DEBUG "resultType": "vector", 21:56:55 DEBUG "resultType": "vector", 21:56:55 DEBUG "data": { 21:56:55 DEBUG "metric": { 21:56:55 DEBUG "result": [ 21:56:55 DEBUG "result": [ 21:56:55 DEBUG "result": [ 21:56:55 DEBUG { 21:56:55 DEBUG "metric": { 21:56:55 DEBUG "data": { 21:56:55 DEBUG "resultType": "vector", 21:56:55 DEBUG { 21:56:55 DEBUG "resultType": "vector", 21:56:55 DEBUG "metric": { 21:56:55 DEBUG "result": [ 21:56:55 DEBUG "resultType": "vector", 21:56:55 DEBUG { 21:56:55 DEBUG { 21:56:55 DEBUG "metric": { 21:56:55 DEBUG "instance": "10.142.0.107:9100" 21:56:55 DEBUG "resultType": "vector", 21:56:55 DEBUG "result": [ 21:56:55 DEBUG { 21:56:55 DEBUG "result": [ 21:56:55 DEBUG "resultType": "vector", 21:56:55 DEBUG "instance": "10.142.0.107:9100" 21:56:55 DEBUG { 21:56:55 DEBUG { 21:56:55 DEBUG "resultType": "vector", 21:56:55 DEBUG "result": [ 21:56:55 DEBUG "resultType": "vector", 21:56:55 DEBUG "result": [ 21:56:55 DEBUG { 21:56:55 DEBUG "metric": { 21:56:55 DEBUG "result": [ 21:56:55 DEBUG "pod": "ds-idrepo-0" 21:56:55 DEBUG { 21:56:55 DEBUG "result": [ 21:56:55 DEBUG "metric": { 21:56:55 DEBUG }, 21:56:55 DEBUG "metric": { 21:56:55 DEBUG "pod": "ds-idrepo-0" 21:56:55 DEBUG }, 21:56:55 DEBUG "metric": { 21:56:55 DEBUG "result": [ 21:56:55 DEBUG { 21:56:55 DEBUG "result": [ 21:56:55 DEBUG }, 21:56:55 DEBUG "metric": { 21:56:55 DEBUG "metric": { 21:56:55 DEBUG { 21:56:55 DEBUG "metric": { 21:56:55 DEBUG "result": [ 21:56:55 DEBUG "resultType": "vector", 21:56:55 DEBUG "metric": { 21:56:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:56:55 DEBUG { 21:56:55 DEBUG }, 21:56:55 DEBUG "metric": { 21:56:55 DEBUG { 21:56:55 DEBUG "metric": { 21:56:55 DEBUG "value": [ 21:56:55 DEBUG "value": [ 21:56:55 DEBUG "result": [ 21:56:55 DEBUG { 21:56:55 DEBUG "pod": "am-767f6b67b5-9zw22" 21:56:55 DEBUG { 21:56:55 DEBUG "metric": { 21:56:55 DEBUG { 21:56:55 DEBUG "value": [ 21:56:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 21:56:55 DEBUG "pod": "ds-cts-0" 21:56:55 DEBUG "pod": "am-767f6b67b5-9zw22" 21:56:55 DEBUG }, 21:56:55 DEBUG { 21:56:55 DEBUG "result": [ 21:56:55 DEBUG "instance": "10.142.0.107:9100" 21:56:55 DEBUG }, 21:56:55 DEBUG "metric": { 21:56:55 DEBUG "value": [ 21:56:55 DEBUG "pod": "am-767f6b67b5-9zw22" 21:56:55 DEBUG "pod": "am-767f6b67b5-9zw22" 21:56:55 DEBUG "pod": "ds-cts-0" 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG { 21:56:55 DEBUG "metric": { 21:56:55 DEBUG }, 21:56:55 DEBUG "metric": { 21:56:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:56:55 DEBUG "metric": { 21:56:55 DEBUG 21:56:55 DEBUG }, 21:56:55 DEBUG }, 21:56:55 DEBUG "pod": "ds-idrepo-0" 21:56:55 DEBUG "value": [ 21:56:55 DEBUG "metric": { 21:56:55 DEBUG { 21:56:55 DEBUG }, 21:56:55 DEBUG "value": [ 21:56:55 DEBUG "pod": "am-767f6b67b5-9zw22" 21:56:55 DEBUG 21:56:55 DEBUG }, 21:56:55 DEBUG }, 21:56:55 DEBUG }, 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG "metric": { 21:56:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:56:55 DEBUG "value": [ 21:56:55 DEBUG "instance": "10.142.0.107:9100" 21:56:55 DEBUG }, 21:56:55 DEBUG "pod": "ds-cts-0" 21:56:55 DEBUG 21:56:55 DEBUG "value": [ 21:56:55 DEBUG "value": [ 21:56:55 DEBUG }, 21:56:55 DEBUG 21:56:55 DEBUG "pod": "am-767f6b67b5-9zw22" 21:56:55 DEBUG "metric": { 21:56:55 DEBUG "value": [ 21:56:55 DEBUG 21:56:55 DEBUG }, 21:56:55 DEBUG "value": [ 21:56:55 DEBUG "value": [ 21:56:55 DEBUG "value": [ 21:56:55 DEBUG "value": [ 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG "instance": "10.142.0.107:9100" 21:56:55 DEBUG }, 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG }, 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG "value": [ 21:56:55 DEBUG 21:56:55 DEBUG }, 21:56:55 DEBUG "value": [ 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG [...] 21:56:55 DEBUG }, 21:56:55 DEBUG "value": [ 21:56:55 DEBUG [...] 21:56:55 DEBUG "value": [ 21:56:55 DEBUG 21:56:55 DEBUG "value": [ 21:56:55 DEBUG [...] 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG [...] 21:56:55 DEBUG 21:56:55 DEBUG "value": [ 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG }, 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG [...] 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG [...] 21:56:55 DEBUG [...] 21:56:55 DEBUG }, 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG [...] 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG "value": [ 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG [...] 21:56:55 DEBUG [...] 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG [...] 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG "value": [ 21:56:55 DEBUG [...] 21:56:55 DEBUG [...] 21:56:55 DEBUG [...] 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG "pod": "ds-idrepo-2" 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG [...] 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG "instance": "10.142.0.43:9100" 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG }, 21:56:55 DEBUG 21:56:55 DEBUG [...] 21:56:55 DEBUG "pod": "am-767f6b67b5-vbltz" 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG [...] 21:56:55 DEBUG }, 21:56:55 DEBUG [...] 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG "pod": "am-767f6b67b5-vbltz" 21:56:55 DEBUG [...] 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG "pod": "overseer-0-967d957f7-zk66v" 21:56:55 DEBUG }, 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG "instance": "10.142.0.43:9100" 21:56:55 DEBUG "value": [ 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG }, 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG "value": [ 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG }, 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG "value": [ 21:56:55 DEBUG 1713646615, 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG "pod": "am-767f6b67b5-vbltz" 21:56:55 DEBUG 21:56:55 DEBUG }, 21:56:55 DEBUG 1713646615, 21:56:55 DEBUG [...] 21:56:55 DEBUG 21:56:55 DEBUG "value": [ 21:56:55 DEBUG [...] 21:56:55 DEBUG 21:56:55 DEBUG 1713646615, 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG "pod": "ds-cts-2" 21:56:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 21:56:55 DEBUG }, 21:56:55 DEBUG 21:56:55 DEBUG "instance": "10.142.0.43:9100" 21:56:55 DEBUG 21:56:55 DEBUG "0" 21:56:55 DEBUG 21:56:55 DEBUG "pod": "am-767f6b67b5-vbltz" 21:56:55 DEBUG "pod": "am-767f6b67b5-vbltz" 21:56:55 DEBUG }, 21:56:55 DEBUG "pod": "ds-cts-2" 21:56:55 DEBUG "value": [ 21:56:55 DEBUG "0" 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG 1713646615, 21:56:55 DEBUG 21:56:55 DEBUG "pod": "idm-b85f9d978-wnhxb" 21:56:55 DEBUG "4013453837312" 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG }, 21:56:55 DEBUG "value": [ 21:56:55 DEBUG "value": [ 21:56:55 DEBUG 21:56:55 DEBUG }, 21:56:55 DEBUG [...] 21:56:55 DEBUG ] 21:56:55 DEBUG } 21:56:55 DEBUG }, 21:56:55 DEBUG }, 21:56:55 DEBUG "value": [ 21:56:55 DEBUG 1713646615, 21:56:55 DEBUG 1713646615, 21:56:55 DEBUG ] 21:56:55 DEBUG 21:56:55 DEBUG 21:56:55 DEBUG "3323.866666666667" 21:56:55 DEBUG 21:56:55 DEBUG }, 21:56:55 DEBUG ] 21:56:55 DEBUG "pod": "ds-cts-2" 21:56:55 DEBUG "pod": "ds-idrepo-2" 21:56:55 DEBUG "value": [ 21:56:55 DEBUG 1713646615, 21:56:55 DEBUG 1713646615, 21:56:55 DEBUG "pod": "am-767f6b67b5-vbltz" 21:56:55 DEBUG "value": [ 21:56:55 DEBUG 21:56:55 DEBUG ] 21:56:55 DEBUG "pod": "ds-idrepo-2" 21:56:55 DEBUG "value": [ 21:56:55 DEBUG "value": [ 21:56:55 DEBUG "49.377" 21:56:55 DEBUG }, 21:56:55 DEBUG "100013163.7" 21:56:55 DEBUG } 21:56:55 DEBUG "instance": "10.142.0.43:9100" 21:56:55 DEBUG "pod": "idm-b85f9d978-wnhxb" 21:56:55 DEBUG ] 21:56:55 DEBUG 21:56:55 DEBUG "value": [ 21:56:55 DEBUG } 21:56:55 DEBUG }, 21:56:55 DEBUG }, 21:56:55 DEBUG 1713646615, 21:56:55 DEBUG "1033961472" 21:56:55 DEBUG "3336.3333333333335" 21:56:55 DEBUG }, 21:56:55 DEBUG 1713646615, 21:56:55 DEBUG 21:56:55 DEBUG } 21:56:55 DEBUG } 21:56:55 DEBUG 1713646615, 21:56:55 DEBUG 1713646615, 21:56:55 DEBUG ] 21:56:55 DEBUG "value": [ 21:56:55 DEBUG ] 21:56:55 DEBUG ] 21:56:55 DEBUG }, 21:56:55 DEBUG }, 21:56:55 DEBUG } 21:56:55 DEBUG "instance": "10.142.0.43:9100" 21:56:55 DEBUG 1713646615, 21:56:55 DEBUG ] 21:56:55 DEBUG "value": [ 21:56:55 DEBUG "value": [ 21:56:55 DEBUG "0" 21:56:55 DEBUG ] 21:56:55 DEBUG ] 21:56:55 DEBUG "value": [ 21:56:55 DEBUG "0.4078666666667194" 21:56:55 DEBUG 21:56:55 DEBUG }, 21:56:55 DEBUG "0" 21:56:55 DEBUG "3322.0666666666666" 21:56:55 DEBUG } 21:56:55 DEBUG 1713646615, 21:56:55 DEBUG } 21:56:55 DEBUG } 21:56:55 DEBUG "value": [ 21:56:55 DEBUG "value": [ 21:56:55 DEBUG ] 21:56:55 DEBUG }, 21:56:55 DEBUG "1121.2834402055707" 21:56:55 DEBUG } 21:56:55 DEBUG 1713646615, 21:56:55 DEBUG 1713646615, 21:56:55 DEBUG ] 21:56:55 DEBUG } 21:56:55 DEBUG } 21:56:55 DEBUG 1713646615, 21:56:55 DEBUG ] 21:56:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 21:56:55 DEBUG "value": [ 21:56:55 DEBUG ] 21:56:55 DEBUG ] 21:56:55 DEBUG ] 21:56:55 DEBUG "28.599999999999998" 21:56:55 DEBUG ] 21:56:55 DEBUG } 21:56:55 DEBUG 1713646615, 21:56:55 DEBUG 1713646615, 21:56:55 DEBUG } 21:56:55 DEBUG "value": [ 21:56:55 DEBUG ] 21:56:55 DEBUG } 21:56:55 DEBUG "0.5232666666666659" 21:56:55 DEBUG "0" 21:56:55 DEBUG } 21:56:55 DEBUG ] 21:56:55 DEBUG ] 21:56:55 DEBUG "599356.5" 21:56:55 DEBUG } 21:56:55 DEBUG }, 21:56:55 DEBUG 1713646615, 21:56:55 DEBUG } 21:56:55 DEBUG } 21:56:55 DEBUG ] 21:56:55 DEBUG } 21:56:55 DEBUG } 21:56:55 DEBUG "100017587.73333333" 21:56:55 DEBUG "0" 21:56:55 DEBUG } 21:56:55 DEBUG 1713646615, 21:56:55 DEBUG } 21:56:55 DEBUG ] 21:56:55 DEBUG ] 21:56:55 DEBUG ] 21:56:55 DEBUG } 21:56:55 DEBUG } 21:56:55 DEBUG ] 21:56:55 DEBUG ] 21:56:55 DEBUG "value": [ 21:56:55 DEBUG 1713646615, 21:56:55 DEBUG ] 21:56:55 DEBUG ] 21:56:55 DEBUG } 21:56:55 DEBUG } 21:56:55 DEBUG } 21:56:55 DEBUG ] 21:56:55 DEBUG ] 21:56:55 DEBUG "0.08300000000000789" 21:56:55 DEBUG ] 21:56:55 DEBUG } 21:56:55 DEBUG } 21:56:55 DEBUG } 21:56:55 DEBUG } 21:56:55 DEBUG } 21:56:55 DEBUG } 21:56:55 DEBUG } 21:56:55 DEBUG "0" 21:56:55 DEBUG "11.787474126134992" 21:56:55 DEBUG } 21:56:55 DEBUG } 21:56:55 DEBUG } 21:56:55 DEBUG } 21:56:55 DEBUG } 21:56:55 DEBUG ] 21:56:55 DEBUG } 21:56:55 DEBUG ] 21:56:55 DEBUG ] 21:56:55 DEBUG } 21:56:55 DEBUG ] 21:56:55 DEBUG } 21:56:55 DEBUG ] 21:56:55 DEBUG } 21:56:55 DEBUG ] 21:56:55 DEBUG ] 21:56:55 DEBUG ] 21:56:55 DEBUG ] 21:56:55 DEBUG } 21:56:55 DEBUG } 21:56:55 DEBUG } 21:56:55 DEBUG } 21:56:56 DEBUG } 21:56:55 DEBUG } 21:56:55 DEBUG } 21:56:55 DEBUG } 21:56:55 DEBUG } 21:56:55 DEBUG } 21:56:55 DEBUG ] 21:56:56 DEBUG } 21:56:55 DEBUG } 21:56:56 DEBUG ] 21:56:56 DEBUG ] 21:56:56 DEBUG } 21:56:56 DEBUG } 21:56:56 DEBUG } 21:56:56 DEBUG } 21:56:56 DEBUG } 21:56:56 DEBUG } 21:56:56 DEBUG } 21:56:56 DEBUG } 21:56:56 DEBUG } 21:56:56 DEBUG } 21:57:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713646673" 21:57:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646673" 21:57:53 ERROR [http_cmd]: ERROR 21:57:53 INFO [http_cmd]: http status code OK 21:57:53 ERROR --- status code --- 21:57:53 DEBUG --- status code --- 21:57:53 ERROR http status code is 400 (expected [200, 401, 403, 404]) 21:57:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:57:53 DEBUG --- http response --- 21:57:53 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 21:57:53 ERROR --- http response --- 21:57:53 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 21:57:53 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 21:57:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646673" 21:57:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713646673" 21:57:53 INFO [http_cmd]: http status code OK 21:57:53 INFO [http_cmd]: http status code OK 21:57:53 DEBUG --- status code --- 21:57:53 DEBUG --- status code --- 21:57:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:57:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:57:53 DEBUG --- http response --- 21:57:53 DEBUG --- http response --- 21:57:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:57:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 21:57:53 DEBUG ----- output ----- 21:57:53 DEBUG ----- output ----- 21:57:53 DEBUG { 21:57:53 DEBUG { 21:57:53 DEBUG "status": "success", 21:57:53 DEBUG "status": "success", 21:57:53 DEBUG "data": { 21:57:53 DEBUG "data": { 21:57:53 DEBUG "resultType": "vector", 21:57:53 DEBUG "resultType": "vector", 21:57:53 DEBUG "result": [ 21:57:53 DEBUG "result": [ 21:57:53 DEBUG { 21:57:53 DEBUG "metric": { 21:57:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:57:53 DEBUG }, 21:57:53 DEBUG { 21:57:53 DEBUG "value": [ 21:57:53 DEBUG "metric": { 21:57:53 DEBUG 21:57:53 DEBUG "pod": "ds-idrepo-0" 21:57:53 DEBUG 21:57:53 DEBUG }, 21:57:53 DEBUG 21:57:53 DEBUG "value": [ 21:57:53 DEBUG [...] 21:57:53 DEBUG 21:57:53 DEBUG 21:57:53 DEBUG 21:57:53 DEBUG 21:57:53 DEBUG 21:57:53 DEBUG 21:57:53 DEBUG [...] 21:57:53 DEBUG "pod": "overseer-0-967d957f7-zk66v" 21:57:53 DEBUG 21:57:53 DEBUG }, 21:57:53 DEBUG 21:57:53 DEBUG "value": [ 21:57:53 DEBUG 21:57:53 DEBUG 1713646673, 21:57:53 DEBUG "pod": "ds-cts-2" 21:57:53 DEBUG "127470.60283054653" 21:57:53 DEBUG }, 21:57:53 DEBUG ] 21:57:53 DEBUG } 21:57:53 DEBUG ] 21:57:53 DEBUG } 21:57:53 DEBUG } 21:57:53 DEBUG "value": [ 21:57:53 DEBUG 1713646673, 21:57:53 DEBUG "0" 21:57:53 DEBUG ] 21:57:53 DEBUG } 21:57:53 DEBUG ] 21:57:53 DEBUG } 21:57:53 DEBUG } 21:57:54 INFO 21:57:54 INFO [loop_until]: kubectl --namespace=xlou top pods 21:57:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:57:54 INFO [loop_until]: OK (rc = 0) 21:57:54 DEBUG --- stdout --- 21:57:54 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 21440m 8767Mi am-767f6b67b5-qcqz2 20856m 8889Mi am-767f6b67b5-vbltz 23725m 8948Mi ds-cts-0 19798m 12265Mi ds-cts-1 20328m 12263Mi ds-cts-2 20559m 12428Mi ds-idrepo-0 9404m 3906Mi ds-idrepo-1 9645m 3975Mi ds-idrepo-2 9909m 3878Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-7dd48676f6-tdxl9 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-967d957f7-zk66v 11737m 894Mi 21:57:54 DEBUG --- stderr --- 21:57:54 DEBUG 21:57:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646675" 21:57:55 INFO [http_cmd]: http status code OK 21:57:55 DEBUG --- status code --- 21:57:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:57:55 DEBUG --- http response --- 21:57:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:57:55 DEBUG ----- output ----- 21:57:55 DEBUG { 21:57:55 DEBUG "status": "success", 21:57:55 DEBUG "data": { 21:57:55 DEBUG "resultType": "vector", 21:57:55 DEBUG "result": [ 21:57:55 DEBUG { 21:57:55 DEBUG "metric": { 21:57:55 DEBUG "pod": "am-767f6b67b5-9zw22" 21:57:55 DEBUG }, 21:57:55 DEBUG "value": [ 21:57:55 DEBUG 21:57:55 DEBUG 21:57:55 DEBUG 21:57:55 DEBUG [...] 21:57:55 DEBUG 21:57:55 DEBUG 21:57:55 DEBUG 21:57:55 DEBUG "pod": "am-767f6b67b5-vbltz" 21:57:55 DEBUG }, 21:57:55 DEBUG "value": [ 21:57:55 DEBUG 1713646675, 21:57:55 DEBUG "3339.7" 21:57:55 DEBUG ] 21:57:55 DEBUG } 21:57:55 DEBUG ] 21:57:55 DEBUG } 21:57:55 DEBUG } 21:57:55 INFO 21:57:55 INFO [loop_until]: kubectl --namespace=xlou top node 21:57:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:57:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646676" 21:57:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646676" 21:57:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646676" 21:57:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646676" 21:57:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646676" 21:57:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646676" 21:57:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646676" 21:57:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646676" 21:57:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713646676" 21:57:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646676" 21:57:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713646676" 21:57:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646676" 21:57:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646676" 21:57:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646676" 21:57:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646676" 21:57:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646676" 21:57:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646676" 21:57:56 INFO [http_cmd]: http status code OK 21:57:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646676" 21:57:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646676" 21:57:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646676" 21:57:56 INFO [http_cmd]: http status code OK 21:57:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713646676" 21:57:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646676" 21:57:56 INFO [loop_until]: OK (rc = 0) 21:57:56 INFO [http_cmd]: http status code OK 21:57:56 INFO [http_cmd]: http status code OK 21:57:56 INFO [http_cmd]: http status code OK 21:57:56 INFO [http_cmd]: http status code OK 21:57:56 INFO [http_cmd]: http status code OK 21:57:56 INFO [http_cmd]: http status code OK 21:57:56 INFO [http_cmd]: http status code OK 21:57:56 DEBUG --- status code --- 21:57:56 INFO [http_cmd]: http status code OK 21:57:56 INFO [http_cmd]: http status code OK 21:57:56 INFO [http_cmd]: http status code OK 21:57:56 DEBUG --- status code --- 21:57:56 INFO [http_cmd]: http status code OK 21:57:56 INFO [http_cmd]: http status code OK 21:57:56 INFO [http_cmd]: http status code OK 21:57:56 INFO [http_cmd]: http status code OK 21:57:56 INFO [http_cmd]: http status code OK 21:57:56 INFO [http_cmd]: http status code OK 21:57:56 DEBUG --- stdout --- 21:57:56 DEBUG --- status code --- 21:57:56 DEBUG --- status code --- 21:57:56 DEBUG --- status code --- 21:57:56 DEBUG --- status code --- 21:57:56 DEBUG --- status code --- 21:57:56 DEBUG --- status code --- 21:57:56 DEBUG --- status code --- 21:57:56 INFO [http_cmd]: http status code OK 21:57:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:57:56 DEBUG --- status code --- 21:57:56 DEBUG --- status code --- 21:57:56 DEBUG --- status code --- 21:57:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:57:56 DEBUG --- status code --- 21:57:56 DEBUG --- status code --- 21:57:56 DEBUG --- status code --- 21:57:56 DEBUG --- status code --- 21:57:56 DEBUG --- status code --- 21:57:56 DEBUG --- status code --- 21:57:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:57:56 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 6224Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 9738m 32% 7667Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 21785m 72% 11488Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 86m 0% 3362Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 19777m 66% 15502Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 155m 0% 5231Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 22313m 74% 11773Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19726m 66% 3059Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 24609m 82% 11190Mi 10% gke-xlou-cdm-ds-0f601751-j488 10381m 34% 7298Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19773m 66% 17347Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 10168m 34% 7944Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20483m 68% 15139Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 12055m 75% 8815Mi 15% 21:57:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:57:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:57:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:57:56 INFO [http_cmd]: http status code OK 21:57:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:57:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:57:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:57:56 DEBUG --- status code --- 21:57:56 DEBUG --- http response --- 21:57:56 INFO [http_cmd]: http status code OK 21:57:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:57:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:57:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:57:56 DEBUG --- http response --- 21:57:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:57:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:57:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:57:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:57:56 DEBUG --- http response --- 21:57:56 DEBUG --- http response --- 21:57:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:57:56 DEBUG --- stderr --- 21:57:56 DEBUG 21:57:56 DEBUG --- http response --- 21:57:56 DEBUG --- http response --- 21:57:56 DEBUG --- http response --- 21:57:56 DEBUG --- status code --- 21:57:56 DEBUG --- http response --- 21:57:56 DEBUG --- http response --- 21:57:56 DEBUG --- http response --- 21:57:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:57:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:57:56 DEBUG --- status code --- 21:57:56 DEBUG --- http response --- 21:57:56 DEBUG --- http response --- 21:57:56 DEBUG --- http response --- 21:57:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:57:56 DEBUG --- http response --- 21:57:56 DEBUG --- http response --- 21:57:56 DEBUG --- http response --- 21:57:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:57:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:57:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:57:56 DEBUG --- http response --- 21:57:56 INFO [http_cmd]: http status code OK 21:57:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:57:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:57:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:57:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:57:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:57:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:57:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:57:56 DEBUG --- http response --- 21:57:56 DEBUG ----- output ----- 21:57:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:57:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:57:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:57:56 DEBUG ----- output ----- 21:57:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:57:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:57:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:57:56 DEBUG --- http response --- 21:57:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:57:56 DEBUG ----- output ----- 21:57:56 DEBUG ----- output ----- 21:57:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:57:56 DEBUG --- status code --- 21:57:56 DEBUG ----- output ----- 21:57:56 DEBUG ----- output ----- 21:57:56 DEBUG ----- output ----- 21:57:56 DEBUG --- http response --- 21:57:56 DEBUG ----- output ----- 21:57:56 DEBUG { 21:57:56 DEBUG ----- output ----- 21:57:56 DEBUG { 21:57:56 DEBUG "status": "success", 21:57:56 DEBUG --- http response --- 21:57:56 DEBUG ----- output ----- 21:57:56 DEBUG ----- output ----- 21:57:56 DEBUG { 21:57:56 DEBUG ----- output ----- 21:57:56 DEBUG ----- output ----- 21:57:56 DEBUG ----- output ----- 21:57:56 DEBUG { 21:57:56 DEBUG "status": "success", 21:57:56 DEBUG { 21:57:56 DEBUG { 21:57:56 DEBUG ----- output ----- 21:57:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:57:56 DEBUG { 21:57:56 DEBUG { 21:57:56 DEBUG { 21:57:56 DEBUG ----- output ----- 21:57:56 DEBUG "status": "success", 21:57:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:57:56 DEBUG { 21:57:56 DEBUG "data": { 21:57:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:57:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:57:56 DEBUG { 21:57:56 DEBUG "status": "success", 21:57:56 DEBUG "status": "success", 21:57:56 DEBUG { 21:57:56 DEBUG { 21:57:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:57:56 DEBUG "data": { 21:57:56 DEBUG ----- output ----- 21:57:56 DEBUG "status": "success", 21:57:56 DEBUG "status": "success", 21:57:56 DEBUG { 21:57:56 DEBUG --- http response --- 21:57:56 DEBUG "status": "success", 21:57:56 DEBUG "status": "success", 21:57:56 DEBUG "status": "success", 21:57:56 DEBUG { 21:57:56 DEBUG "data": { 21:57:56 DEBUG "resultType": "vector", 21:57:56 DEBUG "status": "success", 21:57:56 DEBUG "resultType": "vector", 21:57:56 DEBUG ----- output ----- 21:57:56 DEBUG ----- output ----- 21:57:56 DEBUG "data": { 21:57:56 DEBUG { 21:57:56 DEBUG "data": { 21:57:56 DEBUG "status": "success", 21:57:56 DEBUG "status": "success", 21:57:56 DEBUG ----- output ----- 21:57:56 DEBUG "resultType": "vector", 21:57:56 DEBUG "result": [ 21:57:56 DEBUG "data": { 21:57:56 DEBUG "data": { 21:57:56 DEBUG "status": "success", 21:57:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:57:56 DEBUG "data": { 21:57:56 DEBUG "data": { 21:57:56 DEBUG "data": { 21:57:56 DEBUG "status": "success", 21:57:56 DEBUG "result": [ 21:57:56 DEBUG ----- output ----- 21:57:56 DEBUG "data": { 21:57:56 DEBUG "result": [ 21:57:56 DEBUG { 21:57:56 DEBUG { 21:57:56 DEBUG "resultType": "vector", 21:57:56 DEBUG "status": "success", 21:57:56 DEBUG "resultType": "vector", 21:57:56 DEBUG "data": { 21:57:56 DEBUG "data": { 21:57:56 DEBUG { 21:57:56 DEBUG { 21:57:56 DEBUG { 21:57:56 DEBUG "resultType": "vector", 21:57:56 DEBUG "resultType": "vector", 21:57:56 DEBUG "data": { 21:57:56 DEBUG ----- output ----- 21:57:56 DEBUG "resultType": "vector", 21:57:56 DEBUG "resultType": "vector", 21:57:56 DEBUG "result": [ 21:57:56 DEBUG "data": { 21:57:56 DEBUG "resultType": "vector", 21:57:56 DEBUG { 21:57:56 DEBUG "resultType": "vector", 21:57:56 DEBUG { 21:57:56 DEBUG "status": "success", 21:57:56 DEBUG "data": { 21:57:56 DEBUG "result": [ 21:57:56 DEBUG "data": { 21:57:56 DEBUG "result": [ 21:57:56 DEBUG "resultType": "vector", 21:57:56 DEBUG "resultType": "vector", 21:57:56 DEBUG "status": "success", 21:57:56 DEBUG "status": "success", 21:57:56 DEBUG "metric": { 21:57:56 DEBUG "result": [ 21:57:56 DEBUG "result": [ 21:57:56 DEBUG "resultType": "vector", 21:57:56 DEBUG { 21:57:56 DEBUG "result": [ 21:57:56 DEBUG { 21:57:56 DEBUG { 21:57:56 DEBUG "resultType": "vector", 21:57:56 DEBUG "result": [ 21:57:56 DEBUG "status": "success", 21:57:56 DEBUG "result": [ 21:57:56 DEBUG { 21:57:56 DEBUG "resultType": "vector", 21:57:56 DEBUG "status": "success", 21:57:56 DEBUG { 21:57:56 DEBUG "resultType": "vector", 21:57:56 DEBUG { 21:57:56 DEBUG "result": [ 21:57:56 DEBUG "result": [ 21:57:56 DEBUG "data": { 21:57:56 DEBUG "data": { 21:57:56 DEBUG "pod": "ds-cts-0" 21:57:56 DEBUG { 21:57:56 DEBUG { 21:57:56 DEBUG "result": [ 21:57:56 DEBUG "status": "success", 21:57:56 DEBUG { 21:57:56 DEBUG "metric": { 21:57:56 DEBUG "metric": { 21:57:56 DEBUG "result": [ 21:57:56 DEBUG { 21:57:56 DEBUG "data": { 21:57:56 DEBUG "metric": { 21:57:56 DEBUG "result": [ 21:57:56 DEBUG "data": { 21:57:56 DEBUG "metric": { 21:57:56 DEBUG "result": [ 21:57:56 DEBUG "metric": { 21:57:56 DEBUG { 21:57:56 DEBUG { 21:57:56 DEBUG "resultType": "vector", 21:57:56 DEBUG "resultType": "vector", 21:57:56 DEBUG "metric": { 21:57:56 DEBUG }, 21:57:56 DEBUG "metric": { 21:57:56 DEBUG "metric": { 21:57:56 DEBUG { 21:57:56 DEBUG "data": { 21:57:56 DEBUG "pod": "am-767f6b67b5-9zw22" 21:57:56 DEBUG "metric": { 21:57:56 DEBUG "pod": "am-767f6b67b5-9zw22" 21:57:56 DEBUG { 21:57:56 DEBUG "metric": { 21:57:56 DEBUG "resultType": "vector", 21:57:56 DEBUG "instance": "10.142.0.107:9100" 21:57:56 DEBUG }, 21:57:56 DEBUG "resultType": "vector", 21:57:56 DEBUG "instance": "10.142.0.107:9100" 21:57:56 DEBUG { 21:57:56 DEBUG "instance": "10.142.0.107:9100" 21:57:56 DEBUG "metric": { 21:57:56 DEBUG "metric": { 21:57:56 DEBUG "result": [ 21:57:56 DEBUG "result": [ 21:57:56 DEBUG "pod": "ds-idrepo-1" 21:57:56 DEBUG "value": [ 21:57:56 DEBUG "pod": "ds-idrepo-1" 21:57:56 DEBUG "instance": "10.142.0.107:9100" 21:57:56 DEBUG "metric": { 21:57:56 DEBUG "resultType": "vector", 21:57:56 DEBUG }, 21:57:56 DEBUG "pod": "ds-cts-0" 21:57:56 DEBUG }, 21:57:56 DEBUG "metric": { 21:57:56 DEBUG "pod": "am-767f6b67b5-9zw22" 21:57:56 DEBUG "result": [ 21:57:56 DEBUG "value": [ 21:57:56 DEBUG { 21:57:56 DEBUG "result": [ 21:57:56 DEBUG }, 21:57:56 DEBUG "value": [ 21:57:56 DEBUG }, 21:57:56 DEBUG "pod": "ds-idrepo-1" 21:57:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 21:57:56 DEBUG { 21:57:56 DEBUG { 21:57:56 DEBUG "metric": { 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG }, 21:57:56 DEBUG "pod": "ds-idrepo-0" 21:57:56 DEBUG "result": [ 21:57:56 DEBUG "value": [ 21:57:56 DEBUG }, 21:57:56 DEBUG "value": [ 21:57:56 DEBUG "pod": "am-767f6b67b5-9zw22" 21:57:56 DEBUG }, 21:57:56 DEBUG { 21:57:56 DEBUG 21:57:56 DEBUG "metric": { 21:57:56 DEBUG { 21:57:56 DEBUG "metric": { 21:57:56 DEBUG 21:57:56 DEBUG "value": [ 21:57:56 DEBUG }, 21:57:56 DEBUG }, 21:57:56 DEBUG "metric": { 21:57:56 DEBUG "pod": "ds-idrepo-0" 21:57:56 DEBUG }, 21:57:56 DEBUG 21:57:56 DEBUG }, 21:57:56 DEBUG "value": [ 21:57:56 DEBUG }, 21:57:56 DEBUG { 21:57:56 DEBUG 21:57:56 DEBUG "value": [ 21:57:56 DEBUG 21:57:56 DEBUG }, 21:57:56 DEBUG "value": [ 21:57:56 DEBUG "metric": { 21:57:56 DEBUG 21:57:56 DEBUG "pod": "ds-idrepo-0" 21:57:56 DEBUG "metric": { 21:57:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG "value": [ 21:57:56 DEBUG "value": [ 21:57:56 DEBUG "pod": "ds-cts-0" 21:57:56 DEBUG }, 21:57:56 DEBUG "value": [ 21:57:56 DEBUG [...] 21:57:56 DEBUG "value": [ 21:57:56 DEBUG 21:57:56 DEBUG "value": [ 21:57:56 DEBUG "metric": { 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG "value": [ 21:57:56 DEBUG 21:57:56 DEBUG "instance": "10.142.0.107:9100" 21:57:56 DEBUG 21:57:56 DEBUG }, 21:57:56 DEBUG "value": [ 21:57:56 DEBUG }, 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG }, 21:57:56 DEBUG "value": [ 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG }, 21:57:56 DEBUG [...] 21:57:56 DEBUG 21:57:56 DEBUG "pod": "am-767f6b67b5-9zw22" 21:57:56 DEBUG "value": [ 21:57:56 DEBUG [...] 21:57:56 DEBUG [...] 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG "value": [ 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG [...] 21:57:56 DEBUG "pod": "am-767f6b67b5-9zw22" 21:57:56 DEBUG [...] 21:57:56 DEBUG 21:57:56 DEBUG [...] 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG "value": [ 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG }, 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG }, 21:57:56 DEBUG 21:57:56 DEBUG [...] 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG [...] 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG "value": [ 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG [...] 21:57:56 DEBUG 21:57:56 DEBUG [...] 21:57:56 DEBUG 21:57:56 DEBUG [...] 21:57:56 DEBUG 21:57:56 DEBUG [...] 21:57:56 DEBUG [...] 21:57:56 DEBUG 21:57:56 DEBUG "value": [ 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG [...] 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG [...] 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG [...] 21:57:56 DEBUG 21:57:56 DEBUG "pod": "ds-cts-2" 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG "instance": "10.142.0.43:9100" 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG [...] 21:57:56 DEBUG "instance": "10.142.0.43:9100" 21:57:56 DEBUG "instance": "10.142.0.43:9100" 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG }, 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG "pod": "am-767f6b67b5-vbltz" 21:57:56 DEBUG 21:57:56 DEBUG "pod": "am-767f6b67b5-vbltz" 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG [...] 21:57:56 DEBUG }, 21:57:56 DEBUG "value": [ 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG }, 21:57:56 DEBUG "value": [ 21:57:56 DEBUG 21:57:56 DEBUG [...] 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG "value": [ 21:57:56 DEBUG 1713646676, 21:57:56 DEBUG "pod": "ds-idrepo-2" 21:57:56 DEBUG "instance": "10.142.0.43:9100" 21:57:56 DEBUG 21:57:56 DEBUG }, 21:57:56 DEBUG "pod": "ds-cts-2" 21:57:56 DEBUG }, 21:57:56 DEBUG 21:57:56 DEBUG "pod": "am-767f6b67b5-vbltz" 21:57:56 DEBUG 21:57:56 DEBUG 1713646676, 21:57:56 DEBUG 21:57:56 DEBUG [...] 21:57:56 DEBUG 21:57:56 DEBUG 1713646676, 21:57:56 DEBUG }, 21:57:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 21:57:56 DEBUG 21:57:56 DEBUG "pod": "overseer-0-967d957f7-zk66v" 21:57:56 DEBUG "pod": "ds-cts-2" 21:57:56 DEBUG "pod": "overseer-0-967d957f7-zk66v" 21:57:56 DEBUG "0.49383333333333324" 21:57:56 DEBUG 21:57:56 DEBUG }, 21:57:56 DEBUG }, 21:57:56 DEBUG [...] 21:57:56 DEBUG "value": [ 21:57:56 DEBUG }, 21:57:56 DEBUG "value": [ 21:57:56 DEBUG "pod": "am-767f6b67b5-vbltz" 21:57:56 DEBUG }, 21:57:56 DEBUG 21:57:56 DEBUG "100426736.26666667" 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG "0.3999333333333927" 21:57:56 DEBUG "value": [ 21:57:56 DEBUG }, 21:57:56 DEBUG 21:57:56 DEBUG }, 21:57:56 DEBUG }, 21:57:56 DEBUG }, 21:57:56 DEBUG ] 21:57:56 DEBUG "pod": "overseer-0-967d957f7-zk66v" 21:57:56 DEBUG "value": [ 21:57:56 DEBUG "value": [ 21:57:56 DEBUG 21:57:56 DEBUG 21:57:56 DEBUG "value": [ 21:57:56 DEBUG 1713646676, 21:57:56 DEBUG }, 21:57:56 DEBUG "value": [ 21:57:56 DEBUG 21:57:56 DEBUG ] 21:57:56 DEBUG "pod": "ds-idrepo-2" 21:57:56 DEBUG 21:57:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 21:57:56 DEBUG ] 21:57:56 DEBUG 1713646676, 21:57:56 DEBUG "value": [ 21:57:56 DEBUG 21:57:56 DEBUG "value": [ 21:57:56 DEBUG "value": [ 21:57:56 DEBUG "value": [ 21:57:56 DEBUG } 21:57:56 DEBUG }, 21:57:56 DEBUG 1713646676, 21:57:56 DEBUG 1713646676, 21:57:56 DEBUG 21:57:56 DEBUG 1713646676, 21:57:56 DEBUG "3328.0333333333333" 21:57:56 DEBUG "value": [ 21:57:56 DEBUG 1713646676, 21:57:56 DEBUG 1713646676, 21:57:56 DEBUG "instance": "10.142.0.43:9100" 21:57:56 DEBUG } 21:57:56 DEBUG ] 21:57:56 DEBUG 21:57:56 DEBUG }, 21:57:56 DEBUG } 21:57:56 DEBUG "4018745709568" 21:57:56 DEBUG 1713646676, 21:57:56 DEBUG "pod": "ds-idrepo-2" 21:57:56 DEBUG 1713646676, 21:57:56 DEBUG 1713646676, 21:57:56 DEBUG 1713646676, 21:57:56 DEBUG ] 21:57:56 DEBUG "value": [ 21:57:56 DEBUG "0" 21:57:56 DEBUG "100431674.13333333" 21:57:56 DEBUG "pod": "am-767f6b67b5-vbltz" 21:57:56 DEBUG "0" 21:57:56 DEBUG ] 21:57:56 DEBUG 1713646676, 21:57:56 DEBUG "3341.0666666666666" 21:57:56 DEBUG "49.377" 21:57:56 DEBUG }, 21:57:56 DEBUG } 21:57:56 DEBUG }, 21:57:56 DEBUG "pod": "am-767f6b67b5-vbltz" 21:57:56 DEBUG "value": [ 21:57:56 DEBUG ] 21:57:56 DEBUG ] 21:57:56 DEBUG "1033998336" 21:57:56 DEBUG }, 21:57:56 DEBUG "value": [ 21:57:56 DEBUG "29.466666666666665" 21:57:56 DEBUG ] 21:57:56 DEBUG } 21:57:56 DEBUG 1713646676, 21:57:56 DEBUG ] 21:57:56 DEBUG ] 21:57:56 DEBUG }, 21:57:56 DEBUG ] 21:57:56 DEBUG } 21:57:56 DEBUG "3343.6666666666665" 21:57:56 DEBUG ] 21:57:56 DEBUG ] 21:57:56 DEBUG "value": [ 21:57:56 DEBUG } 21:57:56 DEBUG "value": [ 21:57:56 DEBUG }, 21:57:56 DEBUG 1713646676, 21:57:56 DEBUG } 21:57:56 DEBUG } 21:57:56 DEBUG ] 21:57:56 DEBUG } 21:57:56 DEBUG "0" 21:57:56 DEBUG } 21:57:56 DEBUG } 21:57:56 DEBUG "0" 21:57:56 DEBUG "5.726200751651007" 21:57:56 DEBUG } 21:57:56 DEBUG } 21:57:56 DEBUG "value": [ 21:57:56 DEBUG } 21:57:56 DEBUG ] 21:57:56 DEBUG ] 21:57:56 DEBUG } 21:57:56 DEBUG } 21:57:56 DEBUG 1713646676, 21:57:56 DEBUG 1713646676, 21:57:56 DEBUG "value": [ 21:57:56 DEBUG "11.850073707131829" 21:57:56 DEBUG } 21:57:56 DEBUG ] 21:57:56 DEBUG 1713646676, 21:57:56 DEBUG ] 21:57:56 DEBUG ] 21:57:56 DEBUG ] 21:57:56 DEBUG ] 21:57:56 DEBUG ] 21:57:56 DEBUG } 21:57:56 DEBUG ] 21:57:56 DEBUG 1713646676, 21:57:56 DEBUG ] 21:57:56 DEBUG } 21:57:56 DEBUG } 21:57:56 DEBUG ] 21:57:56 DEBUG ] 21:57:56 DEBUG "0.07899999999997741" 21:57:56 DEBUG "0" 21:57:56 DEBUG 1713646676, 21:57:56 DEBUG ] 21:57:56 DEBUG } 21:57:56 DEBUG "0" 21:57:56 DEBUG } 21:57:56 DEBUG } 21:57:56 DEBUG } 21:57:56 DEBUG } 21:57:56 DEBUG ] 21:57:56 DEBUG ] 21:57:56 DEBUG } 21:57:56 DEBUG "599700" 21:57:56 DEBUG } 21:57:56 DEBUG } 21:57:56 DEBUG ] 21:57:56 DEBUG } 21:57:56 DEBUG } 21:57:56 DEBUG ] 21:57:56 DEBUG } 21:57:56 DEBUG "0" 21:57:56 DEBUG } 21:57:56 DEBUG } 21:57:56 DEBUG ] 21:57:56 DEBUG } 21:57:56 DEBUG ] 21:57:56 DEBUG } 21:57:56 DEBUG ] 21:57:56 DEBUG } 21:57:56 DEBUG } 21:57:56 DEBUG } 21:57:56 DEBUG ] 21:57:56 DEBUG } 21:57:56 DEBUG } 21:57:56 DEBUG } 21:57:56 DEBUG } 21:57:56 DEBUG ] 21:57:56 DEBUG ] 21:57:56 DEBUG ] 21:57:56 DEBUG ] 21:57:56 DEBUG } 21:57:56 DEBUG } 21:57:56 DEBUG } 21:57:56 DEBUG } 21:57:56 DEBUG } 21:57:56 DEBUG } 21:57:56 DEBUG } 21:57:56 DEBUG } 21:57:56 DEBUG } 21:57:56 DEBUG } 21:57:56 DEBUG } 21:57:56 DEBUG ] 21:57:56 DEBUG } 21:57:56 DEBUG } 21:57:56 DEBUG ] 21:57:56 DEBUG } 21:57:56 DEBUG ] 21:57:56 DEBUG ] 21:57:56 DEBUG } 21:57:56 DEBUG } 21:57:56 DEBUG } 21:57:56 DEBUG } 21:57:56 DEBUG } 21:57:56 DEBUG } 21:57:56 DEBUG } 21:57:56 DEBUG } 21:57:56 DEBUG } 21:58:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646733" 21:58:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713646733" 21:58:53 ERROR [http_cmd]: ERROR 21:58:53 INFO [http_cmd]: http status code OK 21:58:53 ERROR --- status code --- 21:58:53 DEBUG --- status code --- 21:58:53 ERROR http status code is 400 (expected [200, 401, 403, 404]) 21:58:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:58:53 ERROR --- http response --- 21:58:53 DEBUG --- http response --- 21:58:53 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 21:58:53 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 21:58:53 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 21:58:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713646733" 21:58:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646733" 21:58:53 INFO [http_cmd]: http status code OK 21:58:53 DEBUG --- status code --- 21:58:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:58:53 DEBUG --- http response --- 21:58:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:58:53 DEBUG ----- output ----- 21:58:53 DEBUG { 21:58:53 DEBUG "status": "success", 21:58:53 DEBUG "data": { 21:58:53 DEBUG "resultType": "vector", 21:58:53 DEBUG "result": [ 21:58:53 DEBUG { 21:58:53 DEBUG "metric": { 21:58:53 DEBUG "pod": "ds-idrepo-0" 21:58:53 DEBUG }, 21:58:53 DEBUG "value": [ 21:58:53 DEBUG 21:58:53 DEBUG 21:58:53 DEBUG 21:58:53 DEBUG [...] 21:58:53 DEBUG 21:58:53 INFO [http_cmd]: http status code OK 21:58:53 DEBUG 21:58:53 DEBUG --- status code --- 21:58:53 DEBUG 21:58:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:58:53 DEBUG "pod": "ds-cts-2" 21:58:53 DEBUG --- http response --- 21:58:53 DEBUG }, 21:58:53 DEBUG "value": [ 21:58:53 DEBUG 1713646733, 21:58:53 DEBUG "0" 21:58:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:58:53 DEBUG ] 21:58:53 DEBUG ----- output ----- 21:58:53 DEBUG { 21:58:53 DEBUG "status": "success", 21:58:53 DEBUG "data": { 21:58:53 DEBUG "resultType": "vector", 21:58:53 DEBUG } 21:58:53 DEBUG "result": [ 21:58:53 DEBUG ] 21:58:53 DEBUG { 21:58:53 DEBUG } 21:58:53 DEBUG "metric": { 21:58:53 DEBUG } 21:58:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:58:53 DEBUG }, 21:58:53 DEBUG "value": [ 21:58:53 DEBUG 21:58:53 DEBUG 21:58:53 DEBUG 21:58:53 DEBUG [...] 21:58:53 DEBUG 21:58:53 DEBUG 21:58:53 DEBUG 21:58:53 DEBUG "pod": "overseer-0-967d957f7-zk66v" 21:58:53 DEBUG }, 21:58:53 DEBUG "value": [ 21:58:53 DEBUG 1713646733, 21:58:53 DEBUG "57335.56442270562" 21:58:53 DEBUG ] 21:58:53 DEBUG } 21:58:53 DEBUG ] 21:58:53 DEBUG } 21:58:53 DEBUG } 21:58:55 INFO 21:58:55 INFO [loop_until]: kubectl --namespace=xlou top pods 21:58:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:58:55 INFO [loop_until]: OK (rc = 0) 21:58:55 DEBUG --- stdout --- 21:58:55 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 21744m 8769Mi am-767f6b67b5-qcqz2 21928m 8888Mi am-767f6b67b5-vbltz 24201m 8937Mi ds-cts-0 19430m 12291Mi ds-cts-1 19481m 12264Mi ds-cts-2 20908m 12428Mi ds-idrepo-0 9509m 3906Mi ds-idrepo-1 9512m 3975Mi ds-idrepo-2 9639m 3878Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-7dd48676f6-tdxl9 13m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-967d957f7-zk66v 11794m 898Mi 21:58:55 DEBUG --- stderr --- 21:58:55 DEBUG 21:58:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646735" 21:58:55 INFO [http_cmd]: http status code OK 21:58:55 DEBUG --- status code --- 21:58:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:58:55 DEBUG --- http response --- 21:58:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:58:55 DEBUG ----- output ----- 21:58:55 DEBUG { 21:58:55 DEBUG "status": "success", 21:58:55 DEBUG "data": { 21:58:55 DEBUG "resultType": "vector", 21:58:55 DEBUG "result": [ 21:58:55 DEBUG { 21:58:55 DEBUG "metric": { 21:58:55 DEBUG "pod": "am-767f6b67b5-9zw22" 21:58:55 DEBUG }, 21:58:55 DEBUG "value": [ 21:58:55 DEBUG 21:58:55 DEBUG 21:58:55 DEBUG 21:58:55 DEBUG [...] 21:58:55 DEBUG 21:58:55 DEBUG 21:58:55 DEBUG 21:58:55 DEBUG "pod": "am-767f6b67b5-vbltz" 21:58:55 DEBUG }, 21:58:55 DEBUG "value": [ 21:58:55 DEBUG 1713646735, 21:58:55 DEBUG "3324.7666666666664" 21:58:55 DEBUG ] 21:58:55 DEBUG } 21:58:55 DEBUG ] 21:58:55 DEBUG } 21:58:55 DEBUG } 21:58:56 INFO 21:58:56 INFO [loop_until]: kubectl --namespace=xlou top node 21:58:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:58:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646736" 21:58:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646736" 21:58:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646736" 21:58:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646736" 21:58:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646736" 21:58:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713646736" 21:58:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646736" 21:58:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646736" 21:58:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646736" 21:58:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713646736" 21:58:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646736" 21:58:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646736" 21:58:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646736" 21:58:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646736" 21:58:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646736" 21:58:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646736" 21:58:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646736" 21:58:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713646736" 21:58:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646736" 21:58:56 INFO [http_cmd]: http status code OK 21:58:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646736" 21:58:56 INFO [http_cmd]: http status code OK 21:58:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646736" 21:58:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646736" 21:58:56 INFO [loop_until]: OK (rc = 0) 21:58:56 INFO [http_cmd]: http status code OK 21:58:56 INFO [http_cmd]: http status code OK 21:58:56 DEBUG --- status code --- 21:58:56 INFO [http_cmd]: http status code OK 21:58:56 INFO [http_cmd]: http status code OK 21:58:56 INFO [http_cmd]: http status code OK 21:58:56 INFO [http_cmd]: http status code OK 21:58:56 INFO [http_cmd]: http status code OK 21:58:56 INFO [http_cmd]: http status code OK 21:58:56 DEBUG --- status code --- 21:58:56 INFO [http_cmd]: http status code OK 21:58:56 INFO [http_cmd]: http status code OK 21:58:56 INFO [http_cmd]: http status code OK 21:58:56 INFO [http_cmd]: http status code OK 21:58:56 DEBUG --- status code --- 21:58:56 DEBUG --- stdout --- 21:58:56 INFO [http_cmd]: http status code OK 21:58:56 INFO [http_cmd]: http status code OK 21:58:56 DEBUG --- status code --- 21:58:56 INFO [http_cmd]: http status code OK 21:58:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:58:56 INFO [http_cmd]: http status code OK 21:58:56 INFO [http_cmd]: http status code OK 21:58:56 DEBUG --- status code --- 21:58:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:58:56 DEBUG --- status code --- 21:58:56 INFO [http_cmd]: http status code OK 21:58:56 INFO [http_cmd]: http status code OK 21:58:56 DEBUG --- status code --- 21:58:56 DEBUG --- status code --- 21:58:56 DEBUG --- status code --- 21:58:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:58:56 DEBUG --- status code --- 21:58:56 DEBUG --- status code --- 21:58:56 DEBUG --- status code --- 21:58:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:58:56 DEBUG --- status code --- 21:58:56 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 60m 0% 6224Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 9977m 33% 7672Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 21767m 72% 11499Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 79m 0% 3360Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 19903m 66% 15494Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 142m 0% 5248Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 21568m 72% 11788Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19656m 65% 3057Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 24597m 82% 11193Mi 10% gke-xlou-cdm-ds-0f601751-j488 10337m 34% 7288Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19818m 66% 17351Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 10130m 33% 7950Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20487m 68% 15139Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 11641m 73% 8814Mi 15% 21:58:56 DEBUG --- status code --- 21:58:56 DEBUG --- status code --- 21:58:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:58:56 DEBUG --- status code --- 21:58:56 DEBUG --- http response --- 21:58:56 DEBUG --- status code --- 21:58:56 DEBUG --- status code --- 21:58:56 DEBUG --- http response --- 21:58:56 DEBUG --- status code --- 21:58:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:58:56 DEBUG --- status code --- 21:58:56 DEBUG --- status code --- 21:58:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:58:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:58:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:58:56 DEBUG --- http response --- 21:58:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:58:56 INFO [http_cmd]: http status code OK 21:58:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:58:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:58:56 DEBUG --- http response --- 21:58:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:58:56 DEBUG --- stderr --- 21:58:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:58:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:58:56 DEBUG --- http response --- 21:58:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:58:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:58:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:58:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:58:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:58:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:58:56 DEBUG --- http response --- 21:58:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:58:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:58:56 DEBUG --- http response --- 21:58:56 DEBUG --- http response --- 21:58:56 DEBUG --- http response --- 21:58:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:58:56 DEBUG --- http response --- 21:58:56 DEBUG --- status code --- 21:58:56 DEBUG --- http response --- 21:58:56 DEBUG --- http response --- 21:58:56 DEBUG --- http response --- 21:58:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 21:58:56 DEBUG 21:58:56 DEBUG --- http response --- 21:58:56 DEBUG --- http response --- 21:58:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:58:56 DEBUG --- http response --- 21:58:56 DEBUG ----- output ----- 21:58:56 DEBUG --- http response --- 21:58:56 DEBUG --- http response --- 21:58:56 DEBUG ----- output ----- 21:58:56 DEBUG --- http response --- 21:58:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:58:56 DEBUG --- http response --- 21:58:56 DEBUG --- http response --- 21:58:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:58:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:58:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:58:56 DEBUG ----- output ----- 21:58:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:58:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:58:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 21:58:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 21:58:56 DEBUG ----- output ----- 21:58:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:58:56 DEBUG ----- output ----- 21:58:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:58:56 DEBUG ----- output ----- 21:58:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:58:56 DEBUG { 21:58:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:58:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:58:56 DEBUG { 21:58:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:58:56 DEBUG ----- output ----- 21:58:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:58:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:58:56 DEBUG ----- output ----- 21:58:56 DEBUG { 21:58:56 DEBUG ----- output ----- 21:58:56 DEBUG { 21:58:56 DEBUG ----- output ----- 21:58:56 DEBUG --- http response --- 21:58:56 DEBUG ----- output ----- 21:58:56 DEBUG ----- output ----- 21:58:56 DEBUG { 21:58:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:58:56 DEBUG { 21:58:56 DEBUG ----- output ----- 21:58:56 DEBUG { 21:58:56 DEBUG ----- output ----- 21:58:56 DEBUG "status": "success", 21:58:56 DEBUG ----- output ----- 21:58:56 DEBUG ----- output ----- 21:58:56 DEBUG "status": "success", 21:58:56 DEBUG ----- output ----- 21:58:56 DEBUG { 21:58:56 DEBUG ----- output ----- 21:58:56 DEBUG ----- output ----- 21:58:56 DEBUG "status": "success", 21:58:56 DEBUG "data": { 21:58:56 DEBUG { 21:58:56 DEBUG "status": "success", 21:58:56 DEBUG { 21:58:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:58:56 DEBUG { 21:58:56 DEBUG { 21:58:56 DEBUG "status": "success", 21:58:56 DEBUG ----- output ----- 21:58:56 DEBUG "status": "success", 21:58:56 DEBUG { 21:58:56 DEBUG "status": "success", 21:58:56 DEBUG { 21:58:56 DEBUG "data": { 21:58:56 DEBUG { 21:58:56 DEBUG { 21:58:56 DEBUG "data": { 21:58:56 DEBUG { 21:58:56 DEBUG "status": "success", 21:58:56 DEBUG { 21:58:56 DEBUG { 21:58:56 DEBUG "resultType": "vector", 21:58:56 DEBUG ----- output ----- 21:58:56 DEBUG "status": "success", 21:58:56 DEBUG "data": { 21:58:56 DEBUG "status": "success", 21:58:56 DEBUG ----- output ----- 21:58:56 DEBUG "status": "success", 21:58:56 DEBUG "status": "success", 21:58:56 DEBUG "data": { 21:58:56 DEBUG { 21:58:56 DEBUG "status": "success", 21:58:56 DEBUG "status": "success", 21:58:56 DEBUG "data": { 21:58:56 DEBUG "status": "success", 21:58:56 DEBUG "resultType": "vector", 21:58:56 DEBUG "status": "success", 21:58:56 DEBUG "status": "success", 21:58:56 DEBUG "resultType": "vector", 21:58:56 DEBUG "status": "success", 21:58:56 DEBUG "data": { 21:58:56 DEBUG "status": "success", 21:58:56 DEBUG "status": "success", 21:58:56 DEBUG "result": [ 21:58:56 DEBUG { 21:58:56 DEBUG "data": { 21:58:56 DEBUG "resultType": "vector", 21:58:56 DEBUG "data": { 21:58:56 DEBUG { 21:58:56 DEBUG "data": { 21:58:56 DEBUG "data": { 21:58:56 DEBUG "resultType": "vector", 21:58:56 DEBUG "data": { 21:58:56 DEBUG "resultType": "vector", 21:58:56 DEBUG "data": { 21:58:56 DEBUG "resultType": "vector", 21:58:56 DEBUG "data": { 21:58:56 DEBUG "result": [ 21:58:56 DEBUG "data": { 21:58:56 DEBUG "data": { 21:58:56 DEBUG "result": [ 21:58:56 DEBUG "data": { 21:58:56 DEBUG "resultType": "vector", 21:58:56 DEBUG "data": { 21:58:56 DEBUG "data": { 21:58:56 DEBUG { 21:58:56 DEBUG "status": "success", 21:58:56 DEBUG "resultType": "vector", 21:58:56 DEBUG "result": [ 21:58:56 DEBUG "resultType": "vector", 21:58:56 DEBUG "status": "success", 21:58:56 DEBUG "resultType": "vector", 21:58:56 DEBUG "resultType": "vector", 21:58:56 DEBUG "result": [ 21:58:56 DEBUG "result": [ 21:58:56 DEBUG "data": { 21:58:56 DEBUG "resultType": "vector", 21:58:56 DEBUG "result": [ 21:58:56 DEBUG "resultType": "vector", 21:58:56 DEBUG "result": [ 21:58:56 DEBUG "resultType": "vector", 21:58:56 DEBUG "resultType": "vector", 21:58:56 DEBUG { 21:58:56 DEBUG "resultType": "vector", 21:58:56 DEBUG "result": [ 21:58:56 DEBUG "resultType": "vector", 21:58:56 DEBUG "resultType": "vector", 21:58:56 DEBUG "metric": { 21:58:56 DEBUG "data": { 21:58:56 DEBUG "result": [ 21:58:56 DEBUG { 21:58:56 DEBUG "result": [ 21:58:56 DEBUG "data": { 21:58:56 DEBUG "result": [ 21:58:56 DEBUG "result": [ 21:58:56 DEBUG { 21:58:56 DEBUG { 21:58:56 DEBUG "metric": { 21:58:56 DEBUG "result": [ 21:58:56 DEBUG { 21:58:56 DEBUG { 21:58:56 DEBUG "result": [ 21:58:56 DEBUG "result": [ 21:58:56 DEBUG "metric": { 21:58:56 DEBUG "instance": "10.142.0.107:9100" 21:58:56 DEBUG "result": [ 21:58:56 DEBUG { 21:58:56 DEBUG "result": [ 21:58:56 DEBUG "result": [ 21:58:56 DEBUG "instance": "10.142.0.107:9100" 21:58:56 DEBUG "resultType": "vector", 21:58:56 DEBUG { 21:58:56 DEBUG "metric": { 21:58:56 DEBUG { 21:58:56 DEBUG "resultType": "vector", 21:58:56 DEBUG { 21:58:56 DEBUG { 21:58:56 DEBUG "metric": { 21:58:56 DEBUG "instance": "10.142.0.107:9100" 21:58:56 DEBUG "resultType": "vector", 21:58:56 DEBUG { 21:58:56 DEBUG "metric": { 21:58:56 DEBUG "metric": { 21:58:56 DEBUG { 21:58:56 DEBUG { 21:58:56 DEBUG { 21:58:56 DEBUG }, 21:58:56 DEBUG "metric": { 21:58:56 DEBUG { 21:58:56 DEBUG { 21:58:56 DEBUG { 21:58:56 DEBUG "metric": { 21:58:56 DEBUG "result": [ 21:58:56 DEBUG "metric": { 21:58:56 DEBUG "pod": "ds-cts-0" 21:58:56 DEBUG }, 21:58:56 DEBUG "metric": { 21:58:56 DEBUG "metric": { 21:58:56 DEBUG "metric": { 21:58:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:58:56 DEBUG }, 21:58:56 DEBUG "result": [ 21:58:56 DEBUG "metric": { 21:58:56 DEBUG "pod": "am-767f6b67b5-9zw22" 21:58:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 21:58:56 DEBUG "metric": { 21:58:56 DEBUG "metric": { 21:58:56 DEBUG "metric": { 21:58:56 DEBUG "value": [ 21:58:56 DEBUG 21:58:56 DEBUG "metric": { 21:58:56 DEBUG "metric": { 21:58:56 DEBUG "pod": "am-767f6b67b5-9zw22" 21:58:56 DEBUG { 21:58:56 DEBUG "pod": "am-767f6b67b5-9zw22" 21:58:56 DEBUG }, 21:58:56 DEBUG "result": [ 21:58:56 DEBUG "value": [ 21:58:56 DEBUG "pod": "ds-idrepo-0" 21:58:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:58:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:58:56 DEBUG }, 21:58:56 DEBUG "value": [ 21:58:56 DEBUG { 21:58:56 DEBUG "pod": "am-767f6b67b5-9zw22" 21:58:56 DEBUG }, 21:58:56 DEBUG }, 21:58:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 21:58:56 DEBUG "pod": "am-767f6b67b5-9zw22" 21:58:56 DEBUG "instance": "10.142.0.107:9100" 21:58:56 DEBUG 21:58:56 DEBUG "pod": "ds-cts-0" 21:58:56 DEBUG }, 21:58:56 DEBUG "pod": "ds-idrepo-0" 21:58:56 DEBUG }, 21:58:56 DEBUG "metric": { 21:58:56 DEBUG }, 21:58:56 DEBUG "value": [ 21:58:56 DEBUG { 21:58:56 DEBUG "metric": { 21:58:56 DEBUG }, 21:58:56 DEBUG "value": [ 21:58:56 DEBUG }, 21:58:56 DEBUG "value": [ 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG }, 21:58:56 DEBUG "value": [ 21:58:56 DEBUG "value": [ 21:58:56 DEBUG }, 21:58:56 DEBUG }, 21:58:56 DEBUG "value": [ 21:58:56 DEBUG 21:58:56 DEBUG "value": [ 21:58:56 DEBUG "pod": "ds-cts-0" 21:58:56 DEBUG }, 21:58:56 DEBUG "value": [ 21:58:56 DEBUG "pod": "am-767f6b67b5-9zw22" 21:58:56 DEBUG "value": [ 21:58:56 DEBUG 21:58:56 DEBUG "pod": "ds-idrepo-0" 21:58:56 DEBUG 21:58:56 DEBUG }, 21:58:56 DEBUG 21:58:56 DEBUG "value": [ 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG "metric": { 21:58:56 DEBUG "instance": "10.142.0.107:9100" 21:58:56 DEBUG }, 21:58:56 DEBUG 21:58:56 DEBUG "value": [ 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG }, 21:58:56 DEBUG 21:58:56 DEBUG "value": [ 21:58:56 DEBUG }, 21:58:56 DEBUG 21:58:56 DEBUG }, 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG }, 21:58:56 DEBUG 21:58:56 DEBUG "value": [ 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG [...] 21:58:56 DEBUG 21:58:56 DEBUG "value": [ 21:58:56 DEBUG "value": [ 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG "value": [ 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG "value": [ 21:58:56 DEBUG 21:58:56 DEBUG "value": [ 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG "value": [ 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG [...] 21:58:56 DEBUG [...] 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG [...] 21:58:56 DEBUG 21:58:56 DEBUG [...] 21:58:56 DEBUG 21:58:56 DEBUG [...] 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG [...] 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG [...] 21:58:56 DEBUG [...] 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG [...] 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG [...] 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG [...] 21:58:56 DEBUG 21:58:56 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG [...] 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG [...] 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG [...] 21:58:56 DEBUG [...] 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG [...] 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG }, 21:58:56 DEBUG 21:58:56 DEBUG [...] 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG "pod": "am-767f6b67b5-vbltz" 21:58:56 DEBUG 21:58:56 DEBUG [...] 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG [...] 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG [...] 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG "value": [ 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG }, 21:58:56 DEBUG "value": [ 21:58:56 DEBUG "pod": "am-767f6b67b5-vbltz" 21:58:56 DEBUG }, 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG "pod": "ds-cts-2" 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG "instance": "10.142.0.43:9100" 21:58:56 DEBUG "pod": "ds-idrepo-2" 21:58:56 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 21:58:56 DEBUG [...] 21:58:56 DEBUG 1713646736, 21:58:56 DEBUG "instance": "10.142.0.43:9100" 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG 1713646736, 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG "value": [ 21:58:56 DEBUG "pod": "ds-cts-2" 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG "pod": "am-767f6b67b5-vbltz" 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG }, 21:58:56 DEBUG 21:58:56 DEBUG 21:58:56 DEBUG }, 21:58:56 DEBUG }, 21:58:56 DEBUG "value": [ 21:58:56 DEBUG 21:58:56 DEBUG "0" 21:58:56 DEBUG }, 21:58:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 21:58:56 DEBUG "instance": "10.142.0.43:9100" 21:58:56 DEBUG 21:58:56 DEBUG "3342.7999999999997" 21:58:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 21:58:56 DEBUG 21:58:56 DEBUG 1713646736, 21:58:56 DEBUG "599385" 21:58:56 DEBUG "pod": "ds-cts-2" 21:58:56 DEBUG }, 21:58:56 DEBUG }, 21:58:56 DEBUG "pod": "am-767f6b67b5-vbltz" 21:58:56 DEBUG "pod": "am-767f6b67b5-vbltz" 21:58:56 DEBUG "value": [ 21:58:56 DEBUG 21:58:56 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 21:58:56 DEBUG }, 21:58:56 DEBUG 1713646736, 21:58:56 DEBUG }, 21:58:56 DEBUG 21:58:56 DEBUG ] 21:58:56 DEBUG "value": [ 21:58:56 DEBUG }, 21:58:56 DEBUG }, 21:58:56 DEBUG "pod": "am-767f6b67b5-vbltz" 21:58:56 DEBUG ] 21:58:56 DEBUG }, 21:58:56 DEBUG 21:58:56 DEBUG ] 21:58:56 DEBUG 21:58:56 DEBUG }, 21:58:56 DEBUG "value": [ 21:58:56 DEBUG "value": [ 21:58:56 DEBUG }, 21:58:56 DEBUG }, 21:58:56 DEBUG 1713646736, 21:58:56 DEBUG "value": [ 21:58:56 DEBUG "value": [ 21:58:56 DEBUG "0" 21:58:56 DEBUG "pod": "ds-idrepo-2" 21:58:56 DEBUG "value": [ 21:58:56 DEBUG 21:58:56 DEBUG } 21:58:56 DEBUG 1713646736, 21:58:56 DEBUG "value": [ 21:58:56 DEBUG "value": [ 21:58:56 DEBUG }, 21:58:56 DEBUG } 21:58:56 DEBUG "value": [ 21:58:56 DEBUG "instance": "10.142.0.43:9100" 21:58:56 DEBUG } 21:58:56 DEBUG "pod": "ds-idrepo-2" 21:58:56 DEBUG "value": [ 21:58:56 DEBUG 1713646736, 21:58:56 DEBUG 1713646736, 21:58:56 DEBUG "value": [ 21:58:56 DEBUG "value": [ 21:58:56 DEBUG "0.4931333333333337" 21:58:56 DEBUG 1713646736, 21:58:56 DEBUG 1713646736, 21:58:56 DEBUG ] 21:58:56 DEBUG }, 21:58:56 DEBUG 1713646736, 21:58:56 DEBUG "instance": "10.142.0.43:9100" 21:58:56 DEBUG ] 21:58:56 DEBUG "99988973.93333334" 21:58:56 DEBUG 1713646736, 21:58:56 DEBUG 1713646736, 21:58:56 DEBUG "value": [ 21:58:56 DEBUG ] 21:58:56 DEBUG 1713646736, 21:58:56 DEBUG }, 21:58:56 DEBUG ] 21:58:56 DEBUG }, 21:58:56 DEBUG "0" 21:58:56 DEBUG 1713646736, 21:58:56 DEBUG "0" 21:58:56 DEBUG 1713646736, 21:58:56 DEBUG 1713646736, 21:58:56 DEBUG ] 21:58:56 DEBUG } 21:58:56 DEBUG "0.000027567583607827523" 21:58:56 DEBUG } 21:58:56 DEBUG "value": [ 21:58:56 DEBUG "0" 21:58:56 DEBUG }, 21:58:56 DEBUG } 21:58:56 DEBUG ] 21:58:56 DEBUG } 21:58:56 DEBUG "4023939931136" 21:58:56 DEBUG 1713646736, 21:58:56 DEBUG } 21:58:56 DEBUG "1037996032" 21:58:56 DEBUG "value": [ 21:58:56 DEBUG 1713646736, 21:58:56 DEBUG "0.4006000000000341" 21:58:56 DEBUG ] 21:58:56 DEBUG "28.7" 21:58:56 DEBUG ] 21:58:56 DEBUG "3323.2666666666664" 21:58:56 DEBUG "49.377" 21:58:56 DEBUG ] 21:58:56 DEBUG "99990781.66666666" 21:58:56 DEBUG ] 21:58:56 DEBUG ] 21:58:56 DEBUG } 21:58:56 DEBUG } 21:58:56 DEBUG "value": [ 21:58:56 DEBUG ] 21:58:56 DEBUG ] 21:58:56 DEBUG "11.809180393977309" 21:58:56 DEBUG ] 21:58:56 DEBUG "3325.733333333333" 21:58:56 DEBUG } 21:58:56 DEBUG ] 21:58:56 DEBUG ] 21:58:56 DEBUG } 21:58:56 DEBUG "value": [ 21:58:56 DEBUG } 21:58:56 DEBUG ] 21:58:56 DEBUG } 21:58:56 DEBUG ] 21:58:56 DEBUG ] 21:58:56 DEBUG } 21:58:56 DEBUG ] 21:58:56 DEBUG } 21:58:56 DEBUG 1713646736, 21:58:56 DEBUG "0" 21:58:56 DEBUG } 21:58:56 DEBUG } 21:58:56 DEBUG } 21:58:56 DEBUG ] 21:58:56 DEBUG } 21:58:56 DEBUG ] 21:58:56 DEBUG } 21:58:56 DEBUG ] 21:58:56 DEBUG } 21:58:56 DEBUG 1713646736, 21:58:56 DEBUG ] 21:58:56 DEBUG } 21:58:56 DEBUG ] 21:58:56 DEBUG } 21:58:56 DEBUG } 21:58:56 DEBUG } 21:58:56 DEBUG ] 21:58:56 DEBUG } 21:58:56 DEBUG ] 21:58:56 DEBUG 1713646736, 21:58:56 DEBUG ] 21:58:56 DEBUG } 21:58:56 DEBUG } 21:58:56 DEBUG ] 21:58:56 DEBUG } 21:58:56 DEBUG } 21:58:56 DEBUG } 21:58:56 DEBUG "0" 21:58:56 DEBUG } 21:58:56 DEBUG ] 21:58:56 DEBUG } 21:58:56 DEBUG } 21:58:56 DEBUG ] 21:58:56 DEBUG ] 21:58:56 DEBUG ] 21:58:56 DEBUG } 21:58:56 DEBUG "0.08133333333333323" 21:58:56 DEBUG } 21:58:56 DEBUG } 21:58:56 DEBUG } 21:58:56 DEBUG ] 21:58:56 DEBUG } 21:58:56 DEBUG ] 21:58:56 DEBUG ] 21:58:56 DEBUG } 21:58:56 DEBUG } 21:58:56 DEBUG } 21:58:56 DEBUG } 21:58:56 DEBUG } 21:58:56 DEBUG } 21:58:56 DEBUG ] 21:58:56 DEBUG } 21:58:56 DEBUG ] 21:58:56 DEBUG } 21:58:56 DEBUG } 21:58:56 DEBUG } 21:58:56 DEBUG } 21:58:56 DEBUG } 21:58:56 DEBUG } 21:58:56 DEBUG } 21:58:56 DEBUG } 21:58:56 DEBUG ] 21:58:56 DEBUG } 21:58:56 DEBUG } 21:58:56 DEBUG } 21:58:56 DEBUG } 21:58:56 DEBUG ] 21:58:56 DEBUG } 21:58:56 DEBUG } 21:58:56 DEBUG } 21:58:56 DEBUG } 21:58:56 DEBUG ] 21:58:56 DEBUG } 21:58:56 DEBUG } 21:58:56 DEBUG } 21:59:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713646793" 21:59:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646793" 21:59:53 ERROR [http_cmd]: ERROR 21:59:53 INFO [http_cmd]: http status code OK 21:59:53 ERROR --- status code --- 21:59:53 DEBUG --- status code --- 21:59:53 ERROR http status code is 400 (expected [200, 401, 403, 404]) 21:59:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:59:53 ERROR --- http response --- 21:59:53 DEBUG --- http response --- 21:59:53 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 21:59:53 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 21:59:53 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 21:59:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713646794" 21:59:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646794" 21:59:54 INFO [http_cmd]: http status code OK 21:59:54 INFO [http_cmd]: http status code OK 21:59:54 DEBUG --- status code --- 21:59:54 DEBUG --- status code --- 21:59:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:59:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:59:54 DEBUG --- http response --- 21:59:54 DEBUG --- http response --- 21:59:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:59:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 21:59:54 DEBUG ----- output ----- 21:59:54 DEBUG ----- output ----- 21:59:54 DEBUG { 21:59:54 DEBUG { 21:59:54 DEBUG "status": "success", 21:59:54 DEBUG "status": "success", 21:59:54 DEBUG "data": { 21:59:54 DEBUG "data": { 21:59:54 DEBUG "resultType": "vector", 21:59:54 DEBUG "resultType": "vector", 21:59:54 DEBUG "result": [ 21:59:54 DEBUG { 21:59:54 DEBUG "result": [ 21:59:54 DEBUG "metric": { 21:59:54 DEBUG { 21:59:54 DEBUG "pod": "ds-idrepo-0" 21:59:54 DEBUG "metric": { 21:59:54 DEBUG }, 21:59:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:59:54 DEBUG "value": [ 21:59:54 DEBUG }, 21:59:54 DEBUG 21:59:54 DEBUG "value": [ 21:59:54 DEBUG 21:59:54 DEBUG 21:59:54 DEBUG 21:59:54 DEBUG 21:59:54 DEBUG [...] 21:59:54 DEBUG 21:59:54 DEBUG 21:59:54 DEBUG [...] 21:59:54 DEBUG 21:59:54 DEBUG 21:59:54 DEBUG 21:59:54 DEBUG 21:59:54 DEBUG "pod": "ds-cts-2" 21:59:54 DEBUG 21:59:54 DEBUG }, 21:59:54 DEBUG "pod": "overseer-0-967d957f7-zk66v" 21:59:54 DEBUG "value": [ 21:59:54 DEBUG }, 21:59:54 DEBUG 1713646794, 21:59:54 DEBUG "value": [ 21:59:54 DEBUG "0" 21:59:54 DEBUG 1713646794, 21:59:54 DEBUG ] 21:59:54 DEBUG "33524283.458235998" 21:59:54 DEBUG } 21:59:54 DEBUG ] 21:59:54 DEBUG ] 21:59:54 DEBUG } 21:59:54 DEBUG } 21:59:54 DEBUG ] 21:59:54 DEBUG } 21:59:54 DEBUG } 21:59:54 DEBUG } 21:59:55 INFO 21:59:55 INFO [loop_until]: kubectl --namespace=xlou top pods 21:59:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:59:55 INFO [loop_until]: OK (rc = 0) 21:59:55 DEBUG --- stdout --- 21:59:55 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 22339m 8769Mi am-767f6b67b5-qcqz2 22644m 8888Mi am-767f6b67b5-vbltz 24450m 8939Mi ds-cts-0 20245m 12293Mi ds-cts-1 20070m 12259Mi ds-cts-2 20664m 12484Mi ds-idrepo-0 9368m 3908Mi ds-idrepo-1 9560m 3975Mi ds-idrepo-2 9635m 3878Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-7dd48676f6-tdxl9 15m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-967d957f7-zk66v 11879m 900Mi 21:59:55 DEBUG --- stderr --- 21:59:55 DEBUG 21:59:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646795" 21:59:55 INFO [http_cmd]: http status code OK 21:59:55 DEBUG --- status code --- 21:59:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:59:55 DEBUG --- http response --- 21:59:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:59:55 DEBUG ----- output ----- 21:59:55 DEBUG { 21:59:55 DEBUG "status": "success", 21:59:55 DEBUG "data": { 21:59:55 DEBUG "resultType": "vector", 21:59:55 DEBUG "result": [ 21:59:55 DEBUG { 21:59:55 DEBUG "metric": { 21:59:55 DEBUG "pod": "am-767f6b67b5-9zw22" 21:59:55 DEBUG }, 21:59:55 DEBUG "value": [ 21:59:55 DEBUG 21:59:55 DEBUG 21:59:55 DEBUG 21:59:55 DEBUG [...] 21:59:55 DEBUG 21:59:55 DEBUG 21:59:55 DEBUG 21:59:55 DEBUG "pod": "am-767f6b67b5-vbltz" 21:59:55 DEBUG }, 21:59:55 DEBUG "value": [ 21:59:55 DEBUG 1713646795, 21:59:55 DEBUG "3326.4333333333334" 21:59:55 DEBUG ] 21:59:55 DEBUG } 21:59:55 DEBUG ] 21:59:55 DEBUG } 21:59:55 DEBUG } 21:59:56 INFO 21:59:56 INFO [loop_until]: kubectl --namespace=xlou top node 21:59:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 21:59:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646796" 21:59:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646796" 21:59:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646796" 21:59:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646796" 21:59:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646796" 21:59:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646796" 21:59:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646796" 21:59:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646796" 21:59:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713646796" 21:59:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646796" 21:59:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646796" 21:59:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646796" 21:59:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646796" 21:59:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646796" 21:59:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713646796" 21:59:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646796" 21:59:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713646796" 21:59:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646796" 21:59:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646796" 21:59:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646796" 21:59:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646796" 21:59:56 INFO [http_cmd]: http status code OK 21:59:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646796" 21:59:56 INFO [http_cmd]: http status code OK 21:59:56 INFO [http_cmd]: http status code OK 21:59:56 INFO [http_cmd]: http status code OK 21:59:56 INFO [loop_until]: OK (rc = 0) 21:59:56 INFO [http_cmd]: http status code OK 21:59:56 INFO [http_cmd]: http status code OK 21:59:56 INFO [http_cmd]: http status code OK 21:59:56 INFO [http_cmd]: http status code OK 21:59:56 INFO [http_cmd]: http status code OK 21:59:56 INFO [http_cmd]: http status code OK 21:59:56 INFO [http_cmd]: http status code OK 21:59:56 INFO [http_cmd]: http status code OK 21:59:56 INFO [http_cmd]: http status code OK 21:59:56 INFO [http_cmd]: http status code OK 21:59:56 DEBUG --- status code --- 21:59:56 INFO [http_cmd]: http status code OK 21:59:56 INFO [http_cmd]: http status code OK 21:59:56 DEBUG --- status code --- 21:59:56 INFO [http_cmd]: http status code OK 21:59:56 DEBUG --- status code --- 21:59:56 INFO [http_cmd]: http status code OK 21:59:56 DEBUG --- status code --- 21:59:56 DEBUG --- stdout --- 21:59:56 DEBUG --- status code --- 21:59:56 DEBUG --- status code --- 21:59:56 DEBUG --- status code --- 21:59:56 DEBUG --- status code --- 21:59:56 DEBUG --- status code --- 21:59:56 DEBUG --- status code --- 21:59:56 DEBUG --- status code --- 21:59:56 DEBUG --- status code --- 21:59:56 DEBUG --- status code --- 21:59:56 DEBUG --- status code --- 21:59:56 INFO [http_cmd]: http status code OK 21:59:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:59:56 DEBUG --- status code --- 21:59:56 DEBUG --- status code --- 21:59:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:59:56 DEBUG --- status code --- 21:59:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:59:56 DEBUG --- http response --- 21:59:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:59:56 INFO [http_cmd]: http status code OK 21:59:56 DEBUG --- status code --- 21:59:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:59:56 DEBUG --- http response --- 21:59:56 INFO [http_cmd]: http status code OK 21:59:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:59:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:59:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:59:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:59:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:59:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:59:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:59:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:59:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:59:56 DEBUG --- status code --- 21:59:56 DEBUG --- http response --- 21:59:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:59:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:59:56 DEBUG --- http response --- 21:59:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:59:56 DEBUG --- status code --- 21:59:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 21:59:56 DEBUG --- http response --- 21:59:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:59:56 INFO [http_cmd]: http status code OK 21:59:56 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 6220Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 10010m 33% 7668Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 21761m 72% 11497Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 88m 0% 3370Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 19907m 66% 15520Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 160m 0% 5225Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 22254m 74% 11784Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19684m 65% 3052Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 24691m 82% 11190Mi 10% gke-xlou-cdm-ds-0f601751-j488 10336m 34% 7289Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19779m 66% 17347Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 9723m 32% 7956Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20464m 68% 15187Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 12107m 76% 8816Mi 15% 21:59:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:59:56 DEBUG --- status code --- 21:59:56 DEBUG --- http response --- 21:59:56 DEBUG --- http response --- 21:59:56 DEBUG --- http response --- 21:59:56 DEBUG --- http response --- 21:59:56 DEBUG --- http response --- 21:59:56 DEBUG --- http response --- 21:59:56 DEBUG --- http response --- 21:59:56 DEBUG --- http response --- 21:59:56 DEBUG --- http response --- 21:59:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:59:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:59:56 DEBUG --- http response --- 21:59:56 DEBUG --- http response --- 21:59:56 DEBUG --- http response --- 21:59:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:59:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:59:56 DEBUG ----- output ----- 21:59:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:59:56 DEBUG --- http response --- 21:59:56 DEBUG --- status code --- 21:59:56 DEBUG --- stderr --- 21:59:56 DEBUG ----- output ----- 21:59:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:59:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:59:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:59:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 21:59:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:59:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:59:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:59:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:59:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 21:59:56 DEBUG --- http response --- 21:59:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:59:56 DEBUG ----- output ----- 21:59:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:59:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:59:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:59:56 DEBUG ----- output ----- 21:59:56 DEBUG --- http response --- 21:59:56 DEBUG { 21:59:56 DEBUG ----- output ----- 21:59:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:59:56 DEBUG ----- output ----- 21:59:56 DEBUG 21:59:56 DEBUG { 21:59:56 DEBUG --- http response --- 21:59:56 DEBUG ----- output ----- 21:59:56 DEBUG ----- output ----- 21:59:56 DEBUG ----- output ----- 21:59:56 DEBUG ----- output ----- 21:59:56 DEBUG ----- output ----- 21:59:56 DEBUG ----- output ----- 21:59:56 DEBUG ----- output ----- 21:59:56 DEBUG ----- output ----- 21:59:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 21:59:56 DEBUG ----- output ----- 21:59:56 DEBUG { 21:59:56 DEBUG ----- output ----- 21:59:56 DEBUG ----- output ----- 21:59:56 DEBUG { 21:59:56 DEBUG { 21:59:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:59:56 DEBUG "status": "success", 21:59:56 DEBUG { 21:59:56 DEBUG { 21:59:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 21:59:56 DEBUG "status": "success", 21:59:56 DEBUG { 21:59:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 21:59:56 DEBUG { 21:59:56 DEBUG { 21:59:56 DEBUG { 21:59:56 DEBUG { 21:59:56 DEBUG { 21:59:56 DEBUG { 21:59:56 DEBUG { 21:59:56 DEBUG ----- output ----- 21:59:56 DEBUG { 21:59:56 DEBUG "status": "success", 21:59:56 DEBUG { 21:59:56 DEBUG "status": "success", 21:59:56 DEBUG ----- output ----- 21:59:56 DEBUG "status": "success", 21:59:56 DEBUG ----- output ----- 21:59:56 DEBUG "data": { 21:59:56 DEBUG "status": "success", 21:59:56 DEBUG "status": "success", 21:59:56 DEBUG --- http response --- 21:59:56 DEBUG "data": { 21:59:56 DEBUG "status": "success", 21:59:56 DEBUG ----- output ----- 21:59:56 DEBUG "status": "success", 21:59:56 DEBUG "status": "success", 21:59:56 DEBUG "status": "success", 21:59:56 DEBUG "status": "success", 21:59:56 DEBUG "status": "success", 21:59:56 DEBUG "status": "success", 21:59:56 DEBUG "status": "success", 21:59:56 DEBUG { 21:59:56 DEBUG "status": "success", 21:59:56 DEBUG "data": { 21:59:56 DEBUG "status": "success", 21:59:56 DEBUG "data": { 21:59:56 DEBUG { 21:59:56 DEBUG "status": "success", 21:59:56 DEBUG { 21:59:56 DEBUG "resultType": "vector", 21:59:56 DEBUG "result": [ 21:59:56 DEBUG "data": { 21:59:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 21:59:56 DEBUG "resultType": "vector", 21:59:56 DEBUG "result": [ 21:59:56 DEBUG { 21:59:56 DEBUG "data": { 21:59:56 DEBUG "data": { 21:59:56 DEBUG "data": { 21:59:56 DEBUG "data": { 21:59:56 DEBUG "data": { 21:59:56 DEBUG "data": { 21:59:56 DEBUG "data": { 21:59:56 DEBUG "status": "success", 21:59:56 DEBUG "resultType": "vector", 21:59:56 DEBUG "data": { 21:59:56 DEBUG "data": { 21:59:56 DEBUG "resultType": "vector", 21:59:56 DEBUG "data": { 21:59:56 DEBUG "status": "success", 21:59:56 DEBUG "data": { 21:59:56 DEBUG { 21:59:56 DEBUG "resultType": "vector", 21:59:56 DEBUG "data": { 21:59:56 DEBUG ----- output ----- 21:59:56 DEBUG { 21:59:56 DEBUG "data": { 21:59:56 DEBUG "status": "success", 21:59:56 DEBUG "resultType": "vector", 21:59:56 DEBUG "result": [ 21:59:56 DEBUG "resultType": "vector", 21:59:56 DEBUG "resultType": "vector", 21:59:56 DEBUG "resultType": "vector", 21:59:56 DEBUG "resultType": "vector", 21:59:56 DEBUG "resultType": "vector", 21:59:56 DEBUG "data": { 21:59:56 DEBUG "resultType": "vector", 21:59:56 DEBUG "resultType": "vector", 21:59:56 DEBUG "resultType": "vector", 21:59:56 DEBUG "result": [ 21:59:56 DEBUG "resultType": "vector", 21:59:56 DEBUG "data": { 21:59:56 DEBUG "resultType": "vector", 21:59:56 DEBUG "metric": { 21:59:56 DEBUG "result": [ 21:59:56 DEBUG "resultType": "vector", 21:59:56 DEBUG { 21:59:56 DEBUG "metric": { 21:59:56 DEBUG "resultType": "vector", 21:59:56 DEBUG "data": { 21:59:56 DEBUG { 21:59:56 DEBUG "resultType": "vector", 21:59:56 DEBUG "result": [ 21:59:56 DEBUG "result": [ 21:59:56 DEBUG "result": [ 21:59:56 DEBUG "result": [ 21:59:56 DEBUG "result": [ 21:59:56 DEBUG "result": [ 21:59:56 DEBUG "result": [ 21:59:56 DEBUG "result": [ 21:59:56 DEBUG "result": [ 21:59:56 DEBUG { 21:59:56 DEBUG "result": [ 21:59:56 DEBUG "resultType": "vector", 21:59:56 DEBUG "result": [ 21:59:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:59:56 DEBUG { 21:59:56 DEBUG "result": [ 21:59:56 DEBUG "status": "success", 21:59:56 DEBUG "pod": "ds-idrepo-0" 21:59:56 DEBUG "result": [ 21:59:56 DEBUG "resultType": "vector", 21:59:56 DEBUG "metric": { 21:59:56 DEBUG "result": [ 21:59:56 DEBUG { 21:59:56 DEBUG { 21:59:56 DEBUG "metric": { 21:59:56 DEBUG { 21:59:56 DEBUG { 21:59:56 DEBUG { 21:59:56 DEBUG { 21:59:56 DEBUG { 21:59:56 DEBUG { 21:59:56 DEBUG "metric": { 21:59:56 DEBUG { 21:59:56 DEBUG "result": [ 21:59:56 DEBUG { 21:59:56 DEBUG }, 21:59:56 DEBUG "metric": { 21:59:56 DEBUG { 21:59:56 DEBUG "data": { 21:59:56 DEBUG }, 21:59:56 DEBUG { 21:59:56 DEBUG "result": [ 21:59:56 DEBUG "pod": "ds-cts-0" 21:59:56 DEBUG }, 21:59:56 DEBUG "metric": { 21:59:56 DEBUG "pod": "ds-cts-0" 21:59:56 DEBUG { 21:59:56 DEBUG "metric": { 21:59:56 DEBUG "metric": { 21:59:56 DEBUG "metric": { 21:59:56 DEBUG "metric": { 21:59:56 DEBUG "metric": { 21:59:56 DEBUG "metric": { 21:59:56 DEBUG "pod": "am-767f6b67b5-9zw22" 21:59:56 DEBUG "metric": { 21:59:56 DEBUG { 21:59:56 DEBUG "metric": { 21:59:56 DEBUG "value": [ 21:59:56 DEBUG "instance": "10.142.0.107:9100" 21:59:56 DEBUG "metric": { 21:59:56 DEBUG "resultType": "vector", 21:59:56 DEBUG "value": [ 21:59:56 DEBUG "metric": { 21:59:56 DEBUG { 21:59:56 DEBUG "value": [ 21:59:56 DEBUG { 21:59:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 21:59:56 DEBUG }, 21:59:56 DEBUG "metric": { 21:59:56 DEBUG "pod": "am-767f6b67b5-9zw22" 21:59:56 DEBUG "pod": "ds-cts-0" 21:59:56 DEBUG "instance": "10.142.0.107:9100" 21:59:56 DEBUG "instance": "10.142.0.107:9100" 21:59:56 DEBUG "pod": "am-767f6b67b5-9zw22" 21:59:56 DEBUG "instance": "10.142.0.107:9100" 21:59:56 DEBUG }, 21:59:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 21:59:56 DEBUG "metric": { 21:59:56 DEBUG "pod": "am-767f6b67b5-9zw22" 21:59:56 DEBUG 21:59:56 DEBUG }, 21:59:56 DEBUG "pod": "ds-idrepo-0" 21:59:56 DEBUG "result": [ 21:59:56 DEBUG 21:59:56 DEBUG "pod": "am-767f6b67b5-9zw22" 21:59:56 DEBUG "metric": { 21:59:56 DEBUG 21:59:56 DEBUG "metric": { 21:59:56 DEBUG }, 21:59:56 DEBUG "value": [ 21:59:56 DEBUG "instance": "10.142.0.107:9100" 21:59:56 DEBUG }, 21:59:56 DEBUG }, 21:59:56 DEBUG }, 21:59:56 DEBUG }, 21:59:56 DEBUG }, 21:59:56 DEBUG }, 21:59:56 DEBUG "value": [ 21:59:56 DEBUG }, 21:59:56 DEBUG "pod": "am-767f6b67b5-9zw22" 21:59:56 DEBUG }, 21:59:56 DEBUG 21:59:56 DEBUG "value": [ 21:59:56 DEBUG }, 21:59:56 DEBUG { 21:59:56 DEBUG 21:59:56 DEBUG }, 21:59:56 DEBUG "pod": "ds-idrepo-0" 21:59:56 DEBUG 21:59:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:59:56 DEBUG "value": [ 21:59:56 DEBUG 21:59:56 DEBUG }, 21:59:56 DEBUG "value": [ 21:59:56 DEBUG "value": [ 21:59:56 DEBUG "value": [ 21:59:56 DEBUG "value": [ 21:59:56 DEBUG "value": [ 21:59:56 DEBUG "value": [ 21:59:56 DEBUG 21:59:56 DEBUG "value": [ 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG "value": [ 21:59:56 DEBUG "metric": { 21:59:56 DEBUG 21:59:56 DEBUG [...] 21:59:56 DEBUG }, 21:59:56 DEBUG 21:59:56 DEBUG }, 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG "value": [ 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG }, 21:59:56 DEBUG "value": [ 21:59:56 DEBUG 21:59:56 DEBUG [...] 21:59:56 DEBUG 21:59:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 21:59:56 DEBUG "value": [ 21:59:56 DEBUG 21:59:56 DEBUG "value": [ 21:59:56 DEBUG [...] 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG [...] 21:59:56 DEBUG "value": [ 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG }, 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG "value": [ 21:59:56 DEBUG 21:59:56 DEBUG [...] 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG [...] 21:59:56 DEBUG 21:59:56 DEBUG [...] 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG "value": [ 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG "pod": "ds-cts-2" 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG [...] 21:59:56 DEBUG 21:59:56 DEBUG [...] 21:59:56 DEBUG [...] 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG [...] 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG [...] 21:59:56 DEBUG 21:59:56 DEBUG [...] 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG "pod": "ds-idrepo-2" 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG }, 21:59:56 DEBUG 21:59:56 DEBUG [...] 21:59:56 DEBUG 21:59:56 DEBUG [...] 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG [...] 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG [...] 21:59:56 DEBUG [...] 21:59:56 DEBUG 21:59:56 DEBUG "pod": "overseer-0-967d957f7-zk66v" 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG [...] 21:59:56 DEBUG }, 21:59:56 DEBUG [...] 21:59:56 DEBUG 21:59:56 DEBUG "value": [ 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG }, 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG "value": [ 21:59:56 DEBUG 21:59:56 DEBUG [...] 21:59:56 DEBUG 1713646796, 21:59:56 DEBUG "pod": "ds-cts-2" 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG "instance": "10.142.0.43:9100" 21:59:56 DEBUG 21:59:56 DEBUG "instance": "10.142.0.43:9100" 21:59:56 DEBUG 21:59:56 DEBUG "pod": "am-767f6b67b5-vbltz" 21:59:56 DEBUG }, 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG "value": [ 21:59:56 DEBUG 21:59:56 DEBUG [...] 21:59:56 DEBUG 21:59:56 DEBUG 1713646796, 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG "0.5071666666666668" 21:59:56 DEBUG }, 21:59:56 DEBUG 21:59:56 DEBUG "instance": "10.142.0.43:9100" 21:59:56 DEBUG "pod": "am-767f6b67b5-vbltz" 21:59:56 DEBUG 21:59:56 DEBUG }, 21:59:56 DEBUG 21:59:56 DEBUG }, 21:59:56 DEBUG "instance": "10.142.0.43:9100" 21:59:56 DEBUG }, 21:59:56 DEBUG "value": [ 21:59:56 DEBUG 21:59:56 DEBUG "instance": "10.142.0.43:9100" 21:59:56 DEBUG 21:59:56 DEBUG 1713646796, 21:59:56 DEBUG "pod": "ds-idrepo-2" 21:59:56 DEBUG }, 21:59:56 DEBUG 21:59:56 DEBUG "0" 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG ] 21:59:56 DEBUG "value": [ 21:59:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 21:59:56 DEBUG }, 21:59:56 DEBUG }, 21:59:56 DEBUG "pod": "ds-cts-2" 21:59:56 DEBUG "value": [ 21:59:56 DEBUG 21:59:56 DEBUG "value": [ 21:59:56 DEBUG }, 21:59:56 DEBUG "value": [ 21:59:56 DEBUG 1713646796, 21:59:56 DEBUG "pod": "am-767f6b67b5-vbltz" 21:59:56 DEBUG }, 21:59:56 DEBUG "pod": "am-767f6b67b5-vbltz" 21:59:56 DEBUG "0" 21:59:56 DEBUG "value": [ 21:59:56 DEBUG 21:59:56 DEBUG 21:59:56 DEBUG ] 21:59:56 DEBUG "pod": "ds-idrepo-2" 21:59:56 DEBUG 21:59:56 DEBUG } 21:59:56 DEBUG 1713646796, 21:59:56 DEBUG "0" 21:59:56 DEBUG "value": [ 21:59:56 DEBUG "value": [ 21:59:56 DEBUG }, 21:59:56 DEBUG 1713646796, 21:59:56 DEBUG "pod": "am-767f6b67b5-vbltz" 21:59:56 DEBUG 1713646796, 21:59:56 DEBUG "value": [ 21:59:56 DEBUG 1713646796, 21:59:56 DEBUG "11.890780164810627" 21:59:56 DEBUG }, 21:59:56 DEBUG "value": [ 21:59:56 DEBUG }, 21:59:56 DEBUG ] 21:59:56 DEBUG 1713646796, 21:59:56 DEBUG "pod": "am-767f6b67b5-vbltz" 21:59:56 DEBUG 21:59:56 DEBUG } 21:59:56 DEBUG }, 21:59:56 DEBUG "pod": "overseer-0-967d957f7-zk66v" 21:59:56 DEBUG ] 21:59:56 DEBUG ] 21:59:56 DEBUG 1713646796, 21:59:56 DEBUG 1713646796, 21:59:56 DEBUG }, 21:59:56 DEBUG "value": [ 21:59:56 DEBUG "0.07766666666666425" 21:59:56 DEBUG }, 21:59:56 DEBUG "4029144484864" 21:59:56 DEBUG 1713646796, 21:59:56 DEBUG "49.377" 21:59:56 DEBUG ] 21:59:56 DEBUG "value": [ 21:59:56 DEBUG 1713646796, 21:59:56 DEBUG "value": [ 21:59:56 DEBUG } 21:59:56 DEBUG "0" 21:59:56 DEBUG }, 21:59:56 DEBUG "pod": "overseer-0-967d957f7-zk66v" 21:59:56 DEBUG ] 21:59:56 DEBUG "value": [ 21:59:56 DEBUG }, 21:59:56 DEBUG } 21:59:56 DEBUG } 21:59:56 DEBUG "100285316.3" 21:59:56 DEBUG "3326.9666666666667" 21:59:56 DEBUG "value": [ 21:59:56 DEBUG 1713646796, 21:59:56 DEBUG ] 21:59:56 DEBUG "value": [ 21:59:56 DEBUG ] 21:59:56 DEBUG "0.4077999999999046" 21:59:56 DEBUG ] 21:59:56 DEBUG } 21:59:56 DEBUG 1713646796, 21:59:56 DEBUG "100281090.76666667" 21:59:56 DEBUG 1713646796, 21:59:56 DEBUG ] 21:59:56 DEBUG } 21:59:56 DEBUG "value": [ 21:59:56 DEBUG }, 21:59:56 DEBUG } 21:59:56 DEBUG 1713646796, 21:59:56 DEBUG "value": [ 21:59:56 DEBUG } 21:59:56 DEBUG ] 21:59:56 DEBUG } 21:59:56 DEBUG ] 21:59:56 DEBUG 1713646796, 21:59:56 DEBUG "31.2" 21:59:56 DEBUG } 21:59:56 DEBUG 1713646796, 21:59:56 DEBUG } 21:59:56 DEBUG ] 21:59:56 DEBUG } 21:59:56 DEBUG ] 21:59:56 DEBUG "3328.0666666666666" 21:59:56 DEBUG ] 21:59:56 DEBUG "599966" 21:59:56 DEBUG ] 21:59:56 DEBUG } 21:59:56 DEBUG 1713646796, 21:59:56 DEBUG "value": [ 21:59:56 DEBUG } 21:59:56 DEBUG "0" 21:59:56 DEBUG 1713646796, 21:59:56 DEBUG } 21:59:56 DEBUG } 21:59:56 DEBUG "1039585280" 21:59:56 DEBUG ] 21:59:56 DEBUG ] 21:59:56 DEBUG ] 21:59:56 DEBUG "3343.7999999999997" 21:59:56 DEBUG ] 21:59:56 DEBUG } 21:59:56 DEBUG ] 21:59:56 DEBUG } 21:59:56 DEBUG ] 21:59:56 DEBUG } 21:59:56 DEBUG ] 21:59:56 DEBUG } 21:59:56 DEBUG "0" 21:59:56 DEBUG 1713646796, 21:59:56 DEBUG ] 21:59:56 DEBUG "0" 21:59:56 DEBUG ] 21:59:56 DEBUG ] 21:59:56 DEBUG } 21:59:56 DEBUG } 21:59:56 DEBUG } 21:59:56 DEBUG } 21:59:56 DEBUG } 21:59:56 DEBUG ] 21:59:56 DEBUG } 21:59:56 DEBUG } 21:59:56 DEBUG } 21:59:56 DEBUG ] 21:59:56 DEBUG } 21:59:56 DEBUG ] 21:59:56 DEBUG } 21:59:56 DEBUG "5.028773850375131" 21:59:56 DEBUG ] 21:59:56 DEBUG ] 21:59:56 DEBUG } 21:59:56 DEBUG } 21:59:56 DEBUG ] 21:59:56 DEBUG ] 21:59:56 DEBUG ] 21:59:56 DEBUG } 21:59:56 DEBUG } 21:59:56 DEBUG } 21:59:56 DEBUG ] 21:59:56 DEBUG } 21:59:56 DEBUG ] 21:59:56 DEBUG } 21:59:56 DEBUG } 21:59:56 DEBUG ] 21:59:56 DEBUG } 21:59:56 DEBUG } 21:59:56 DEBUG } 21:59:56 DEBUG ] 21:59:56 DEBUG } 21:59:56 DEBUG } 21:59:56 DEBUG } 21:59:56 DEBUG } 21:59:56 DEBUG } 21:59:56 DEBUG } 21:59:56 DEBUG } 21:59:56 DEBUG ] 21:59:56 DEBUG } 21:59:56 DEBUG ] 21:59:56 DEBUG ] 21:59:56 DEBUG } 21:59:56 DEBUG } 21:59:56 DEBUG } 21:59:56 DEBUG } 21:59:56 DEBUG } 21:59:56 DEBUG } 21:59:56 DEBUG ] 21:59:56 DEBUG } 21:59:56 DEBUG } 21:59:56 DEBUG } 21:59:56 DEBUG ] 21:59:56 DEBUG } 21:59:56 DEBUG } 21:59:56 DEBUG } 21:59:56 DEBUG } 21:59:56 DEBUG } 21:59:56 DEBUG } 21:59:56 DEBUG } 22:00:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713646853" 22:00:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646853" 22:00:53 ERROR [http_cmd]: ERROR 22:00:53 ERROR --- status code --- 22:00:53 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:00:53 ERROR --- http response --- 22:00:53 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:00:53 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:00:53 INFO [http_cmd]: http status code OK 22:00:53 DEBUG --- status code --- 22:00:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:00:53 DEBUG --- http response --- 22:00:53 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:00:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713646854" 22:00:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646854" 22:00:54 INFO [http_cmd]: http status code OK 22:00:54 INFO [http_cmd]: http status code OK 22:00:54 DEBUG --- status code --- 22:00:54 DEBUG --- status code --- 22:00:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:00:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:00:54 DEBUG --- http response --- 22:00:54 DEBUG --- http response --- 22:00:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:00:54 DEBUG ----- output ----- 22:00:54 DEBUG { 22:00:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 22:00:54 DEBUG "status": "success", 22:00:54 DEBUG ----- output ----- 22:00:54 DEBUG "data": { 22:00:54 DEBUG { 22:00:54 DEBUG "resultType": "vector", 22:00:54 DEBUG "status": "success", 22:00:54 DEBUG "result": [ 22:00:54 DEBUG "data": { 22:00:54 DEBUG { 22:00:54 DEBUG "resultType": "vector", 22:00:54 DEBUG "metric": { 22:00:54 DEBUG "result": [ 22:00:54 DEBUG "pod": "ds-idrepo-0" 22:00:54 DEBUG { 22:00:54 DEBUG }, 22:00:54 DEBUG "metric": { 22:00:54 DEBUG "value": [ 22:00:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:00:54 DEBUG 22:00:54 DEBUG }, 22:00:54 DEBUG 22:00:54 DEBUG "value": [ 22:00:54 DEBUG 22:00:54 DEBUG 22:00:54 DEBUG [...] 22:00:54 DEBUG 22:00:54 DEBUG 22:00:54 DEBUG 22:00:54 DEBUG 22:00:54 DEBUG [...] 22:00:54 DEBUG 22:00:54 DEBUG 22:00:54 DEBUG "pod": "ds-cts-2" 22:00:54 DEBUG 22:00:54 DEBUG }, 22:00:54 DEBUG 22:00:54 DEBUG "value": [ 22:00:54 DEBUG "pod": "am-767f6b67b5-vbltz" 22:00:54 DEBUG 1713646854, 22:00:54 DEBUG }, 22:00:54 DEBUG "0" 22:00:54 DEBUG "value": [ 22:00:54 DEBUG ] 22:00:54 DEBUG 1713646854, 22:00:54 DEBUG } 22:00:54 DEBUG "298922808.2174302" 22:00:54 DEBUG ] 22:00:54 DEBUG ] 22:00:54 DEBUG } 22:00:54 DEBUG } 22:00:54 DEBUG } 22:00:54 DEBUG ] 22:00:54 DEBUG } 22:00:54 DEBUG } 22:00:55 INFO 22:00:55 INFO [loop_until]: kubectl --namespace=xlou top pods 22:00:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:00:55 INFO [loop_until]: OK (rc = 0) 22:00:55 DEBUG --- stdout --- 22:00:55 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 21781m 8771Mi am-767f6b67b5-qcqz2 22042m 8890Mi am-767f6b67b5-vbltz 24188m 8938Mi ds-cts-0 19702m 12291Mi ds-cts-1 19687m 12266Mi ds-cts-2 20199m 12483Mi ds-idrepo-0 9577m 3908Mi ds-idrepo-1 9576m 3975Mi ds-idrepo-2 9719m 3878Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 4m 1238Mi lodemon-7dd48676f6-tdxl9 15m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-967d957f7-zk66v 11797m 902Mi 22:00:55 DEBUG --- stderr --- 22:00:55 DEBUG 22:00:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646855" 22:00:55 INFO [http_cmd]: http status code OK 22:00:55 DEBUG --- status code --- 22:00:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:00:55 DEBUG --- http response --- 22:00:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:00:55 DEBUG ----- output ----- 22:00:55 DEBUG { 22:00:55 DEBUG "status": "success", 22:00:55 DEBUG "data": { 22:00:55 DEBUG "resultType": "vector", 22:00:55 DEBUG "result": [ 22:00:55 DEBUG { 22:00:55 DEBUG "metric": { 22:00:55 DEBUG "pod": "am-767f6b67b5-9zw22" 22:00:55 DEBUG }, 22:00:55 DEBUG "value": [ 22:00:55 DEBUG 22:00:55 DEBUG 22:00:55 DEBUG 22:00:55 DEBUG [...] 22:00:55 DEBUG 22:00:55 DEBUG 22:00:55 DEBUG 22:00:55 DEBUG "pod": "am-767f6b67b5-vbltz" 22:00:55 DEBUG }, 22:00:55 DEBUG "value": [ 22:00:55 DEBUG 1713646855, 22:00:55 DEBUG "3320.4" 22:00:55 DEBUG ] 22:00:55 DEBUG } 22:00:55 DEBUG ] 22:00:55 DEBUG } 22:00:55 DEBUG } 22:00:56 INFO 22:00:56 INFO [loop_until]: kubectl --namespace=xlou top node 22:00:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:00:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646856" 22:00:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646856" 22:00:56 INFO [http_cmd]: http status code OK 22:00:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646856" 22:00:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646856" 22:00:56 DEBUG --- status code --- 22:00:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646856" 22:00:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:00:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713646856" 22:00:56 DEBUG --- http response --- 22:00:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646856" 22:00:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646856" 22:00:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646856" 22:00:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646856" 22:00:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646856" 22:00:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646856" 22:00:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646856" 22:00:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646856" 22:00:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646856" 22:00:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713646856" 22:00:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:00:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646856" 22:00:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646856" 22:00:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646856" 22:00:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713646856" 22:00:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646856" 22:00:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646856" 22:00:56 INFO [http_cmd]: http status code OK 22:00:56 INFO [http_cmd]: http status code OK 22:00:56 INFO [http_cmd]: http status code OK 22:00:56 DEBUG --- status code --- 22:00:56 INFO [loop_until]: OK (rc = 0) 22:00:56 INFO [http_cmd]: http status code OK 22:00:56 INFO [http_cmd]: http status code OK 22:00:56 INFO [http_cmd]: http status code OK 22:00:56 INFO [http_cmd]: http status code OK 22:00:56 INFO [http_cmd]: http status code OK 22:00:56 DEBUG --- status code --- 22:00:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:00:56 DEBUG --- http response --- 22:00:56 INFO [http_cmd]: http status code OK 22:00:56 DEBUG --- status code --- 22:00:56 DEBUG --- status code --- 22:00:56 INFO [http_cmd]: http status code OK 22:00:56 INFO [http_cmd]: http status code OK 22:00:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:00:56 DEBUG --- stdout --- 22:00:56 INFO [http_cmd]: http status code OK 22:00:56 DEBUG --- status code --- 22:00:56 INFO [http_cmd]: http status code OK 22:00:56 DEBUG --- status code --- 22:00:56 INFO [http_cmd]: http status code OK 22:00:56 DEBUG --- status code --- 22:00:56 INFO [http_cmd]: http status code OK 22:00:56 INFO [http_cmd]: http status code OK 22:00:56 INFO [http_cmd]: http status code OK 22:00:56 DEBUG --- status code --- 22:00:56 INFO [http_cmd]: http status code OK 22:00:56 INFO [http_cmd]: http status code OK 22:00:56 DEBUG --- status code --- 22:00:56 DEBUG --- status code --- 22:00:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:00:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:00:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:00:56 DEBUG ----- output ----- 22:00:56 DEBUG --- status code --- 22:00:56 DEBUG --- status code --- 22:00:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:00:56 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 61m 0% 6208Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 10014m 33% 7667Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 21708m 72% 11491Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 84m 0% 3364Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 19618m 65% 15520Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 282m 0% 5166Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 22291m 74% 11783Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19048m 63% 3064Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 24627m 82% 11184Mi 10% gke-xlou-cdm-ds-0f601751-j488 10333m 34% 7288Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19890m 66% 17340Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 10170m 34% 7943Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20499m 68% 15181Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 12059m 75% 8817Mi 15% 22:00:56 DEBUG --- status code --- 22:00:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:00:56 DEBUG --- http response --- 22:00:56 DEBUG --- status code --- 22:00:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:00:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:00:56 DEBUG --- http response --- 22:00:56 DEBUG --- status code --- 22:00:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:00:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:00:56 DEBUG --- status code --- 22:00:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:00:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:00:56 DEBUG --- http response --- 22:00:56 INFO [http_cmd]: http status code OK 22:00:56 DEBUG --- http response --- 22:00:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:00:56 DEBUG --- http response --- 22:00:56 DEBUG { 22:00:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:00:56 DEBUG --- http response --- 22:00:56 DEBUG --- http response --- 22:00:56 DEBUG --- stderr --- 22:00:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:00:56 DEBUG --- status code --- 22:00:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:00:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:00:56 DEBUG --- http response --- 22:00:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:00:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:00:56 DEBUG --- http response --- 22:00:56 DEBUG --- status code --- 22:00:56 DEBUG --- http response --- 22:00:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:00:56 DEBUG --- http response --- 22:00:56 DEBUG ----- output ----- 22:00:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:00:56 DEBUG --- status code --- 22:00:56 DEBUG ----- output ----- 22:00:56 INFO [http_cmd]: http status code OK 22:00:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:00:56 DEBUG "status": "success", 22:00:56 DEBUG --- http response --- 22:00:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:00:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:00:56 DEBUG --- http response --- 22:00:56 DEBUG 22:00:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:00:56 DEBUG ----- output ----- 22:00:56 DEBUG --- http response --- 22:00:56 DEBUG ----- output ----- 22:00:56 DEBUG --- status code --- 22:00:56 DEBUG ----- output ----- 22:00:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:00:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:00:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:00:56 DEBUG --- http response --- 22:00:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:00:56 DEBUG { 22:00:56 DEBUG ----- output ----- 22:00:56 DEBUG { 22:00:56 DEBUG { 22:00:56 DEBUG --- status code --- 22:00:56 DEBUG ----- output ----- 22:00:56 DEBUG "data": { 22:00:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:00:56 DEBUG ----- output ----- 22:00:56 DEBUG { 22:00:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:00:56 DEBUG --- http response --- 22:00:56 DEBUG { 22:00:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:00:56 DEBUG { 22:00:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:00:56 DEBUG { 22:00:56 DEBUG "status": "success", 22:00:56 DEBUG --- http response --- 22:00:56 DEBUG ----- output ----- 22:00:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:00:56 DEBUG ----- output ----- 22:00:56 DEBUG "status": "success", 22:00:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:00:56 DEBUG "status": "success", 22:00:56 DEBUG "status": "success", 22:00:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:00:56 DEBUG --- http response --- 22:00:56 DEBUG "resultType": "vector", 22:00:56 DEBUG ----- output ----- 22:00:56 DEBUG "status": "success", 22:00:56 DEBUG ----- output ----- 22:00:56 DEBUG ----- output ----- 22:00:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:00:56 DEBUG "status": "success", 22:00:56 DEBUG "data": { 22:00:56 DEBUG "status": "success", 22:00:56 DEBUG --- http response --- 22:00:56 DEBUG "data": { 22:00:56 DEBUG ----- output ----- 22:00:56 DEBUG { 22:00:56 DEBUG "status": "success", 22:00:56 DEBUG ----- output ----- 22:00:56 DEBUG { 22:00:56 DEBUG "data": { 22:00:56 DEBUG --- http response --- 22:00:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:00:56 DEBUG "data": { 22:00:56 DEBUG { 22:00:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:00:56 DEBUG ----- output ----- 22:00:56 DEBUG { 22:00:56 DEBUG "data": { 22:00:56 DEBUG { 22:00:56 DEBUG { 22:00:56 DEBUG ----- output ----- 22:00:56 DEBUG "resultType": "vector", 22:00:56 DEBUG "data": { 22:00:56 DEBUG ----- output ----- 22:00:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:00:56 DEBUG "resultType": "vector", 22:00:56 DEBUG "data": { 22:00:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:00:56 DEBUG ----- output ----- 22:00:56 DEBUG { 22:00:56 DEBUG "status": "success", 22:00:56 DEBUG { 22:00:56 DEBUG "resultType": "vector", 22:00:56 DEBUG "data": { 22:00:56 DEBUG "resultType": "vector", 22:00:56 DEBUG "status": "success", 22:00:56 DEBUG { 22:00:56 DEBUG "result": [ 22:00:56 DEBUG "status": "success", 22:00:56 DEBUG "resultType": "vector", 22:00:56 DEBUG "result": [ 22:00:56 DEBUG "status": "success", 22:00:56 DEBUG { 22:00:56 DEBUG "result": [ 22:00:56 DEBUG "resultType": "vector", 22:00:56 DEBUG "result": [ 22:00:56 DEBUG { 22:00:56 DEBUG "result": [ 22:00:56 DEBUG "resultType": "vector", 22:00:56 DEBUG "result": [ 22:00:56 DEBUG { 22:00:56 DEBUG "metric": { 22:00:56 DEBUG ----- output ----- 22:00:56 DEBUG "status": "success", 22:00:56 DEBUG "data": { 22:00:56 DEBUG "resultType": "vector", 22:00:56 DEBUG "result": [ 22:00:56 DEBUG "data": { 22:00:56 DEBUG "status": "success", 22:00:56 DEBUG { 22:00:56 DEBUG "data": { 22:00:56 DEBUG { 22:00:56 DEBUG "data": { 22:00:56 DEBUG "status": "success", 22:00:56 DEBUG "data": { 22:00:56 DEBUG { 22:00:56 DEBUG { 22:00:56 DEBUG ----- output ----- 22:00:56 DEBUG "status": "success", 22:00:56 DEBUG { 22:00:56 DEBUG "pod": "am-767f6b67b5-9zw22" 22:00:56 DEBUG }, 22:00:56 DEBUG "value": [ 22:00:56 DEBUG { 22:00:56 DEBUG "resultType": "vector", 22:00:56 DEBUG "result": [ 22:00:56 DEBUG { 22:00:56 DEBUG "result": [ 22:00:56 DEBUG { 22:00:56 DEBUG "metric": { 22:00:56 DEBUG "data": { 22:00:56 DEBUG "metric": { 22:00:56 DEBUG "resultType": "vector", 22:00:56 DEBUG "metric": { 22:00:56 DEBUG "resultType": "vector", 22:00:56 DEBUG "resultType": "vector", 22:00:56 DEBUG "status": "success", 22:00:56 DEBUG "metric": { 22:00:56 DEBUG "metric": { 22:00:56 DEBUG { 22:00:56 DEBUG "data": { 22:00:56 DEBUG "metric": { 22:00:56 DEBUG "data": { 22:00:56 DEBUG 22:00:56 DEBUG { 22:00:56 DEBUG "status": "success", 22:00:56 DEBUG "result": [ 22:00:56 DEBUG "status": "success", 22:00:56 DEBUG "metric": { 22:00:56 DEBUG { 22:00:56 DEBUG "resultType": "vector", 22:00:56 DEBUG "pod": "ds-cts-0" 22:00:56 DEBUG "resultType": "vector", 22:00:56 DEBUG "pod": "ds-cts-0" 22:00:56 DEBUG "result": [ 22:00:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:00:56 DEBUG }, 22:00:56 DEBUG "result": [ 22:00:56 DEBUG "data": { 22:00:56 DEBUG "instance": "10.142.0.107:9100" 22:00:56 DEBUG "instance": "10.142.0.107:9100" 22:00:56 DEBUG "status": "success", 22:00:56 DEBUG "resultType": "vector", 22:00:56 DEBUG "pod": "ds-idrepo-0" 22:00:56 DEBUG "resultType": "vector", 22:00:56 DEBUG 22:00:56 DEBUG "status": "success", 22:00:56 DEBUG "data": { 22:00:56 DEBUG "resultType": "vector", 22:00:56 DEBUG "data": { 22:00:56 DEBUG "resultType": "vector", 22:00:56 DEBUG "metric": { 22:00:56 DEBUG "result": [ 22:00:56 DEBUG { 22:00:56 DEBUG "result": [ 22:00:56 DEBUG { 22:00:56 DEBUG { 22:00:56 DEBUG "value": [ 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG }, 22:00:56 DEBUG "result": [ 22:00:56 DEBUG }, 22:00:56 DEBUG "data": { 22:00:56 DEBUG "result": [ 22:00:56 DEBUG }, 22:00:56 DEBUG "result": [ 22:00:56 DEBUG 22:00:56 DEBUG "data": { 22:00:56 DEBUG "result": [ 22:00:56 DEBUG { 22:00:56 DEBUG "result": [ 22:00:56 DEBUG "pod": "am-767f6b67b5-9zw22" 22:00:56 DEBUG "pod": "ds-cts-0" 22:00:56 DEBUG "metric": { 22:00:56 DEBUG }, 22:00:56 DEBUG "metric": { 22:00:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:00:56 DEBUG }, 22:00:56 DEBUG "value": [ 22:00:56 DEBUG "resultType": "vector", 22:00:56 DEBUG "value": [ 22:00:56 DEBUG { 22:00:56 DEBUG "metric": { 22:00:56 DEBUG "value": [ 22:00:56 DEBUG "resultType": "vector", 22:00:56 DEBUG { 22:00:56 DEBUG "value": [ 22:00:56 DEBUG { 22:00:56 DEBUG "metric": { 22:00:56 DEBUG "resultType": "vector", 22:00:56 DEBUG "metric": { 22:00:56 DEBUG { 22:00:56 DEBUG { 22:00:56 DEBUG }, 22:00:56 DEBUG }, 22:00:56 DEBUG "pod": "ds-idrepo-0" 22:00:56 DEBUG "value": [ 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG "result": [ 22:00:56 DEBUG 22:00:56 DEBUG { 22:00:56 DEBUG "pod": "am-767f6b67b5-9zw22" 22:00:56 DEBUG 22:00:56 DEBUG "result": [ 22:00:56 DEBUG "metric": { 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:00:56 DEBUG "result": [ 22:00:56 DEBUG [...] 22:00:56 DEBUG "metric": { 22:00:56 DEBUG "metric": { 22:00:56 DEBUG "value": [ 22:00:56 DEBUG "value": [ 22:00:56 DEBUG }, 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG [...] 22:00:56 DEBUG 22:00:56 DEBUG }, 22:00:56 DEBUG { 22:00:56 DEBUG 22:00:56 DEBUG "metric": { 22:00:56 DEBUG }, 22:00:56 DEBUG 22:00:56 DEBUG { 22:00:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:00:56 DEBUG }, 22:00:56 DEBUG }, 22:00:56 DEBUG { 22:00:56 DEBUG "instance": "10.142.0.107:9100" 22:00:56 DEBUG 22:00:56 DEBUG "instance": "10.142.0.107:9100" 22:00:56 DEBUG }, 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG "value": [ 22:00:56 DEBUG [...] 22:00:56 DEBUG "metric": { 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG "value": [ 22:00:56 DEBUG "metric": { 22:00:56 DEBUG 22:00:56 DEBUG "pod": "am-767f6b67b5-9zw22" 22:00:56 DEBUG "value": [ 22:00:56 DEBUG 22:00:56 DEBUG [...] 22:00:56 DEBUG "value": [ 22:00:56 DEBUG "value": [ 22:00:56 DEBUG "metric": { 22:00:56 DEBUG 22:00:56 DEBUG }, 22:00:56 DEBUG 22:00:56 DEBUG "value": [ 22:00:56 DEBUG "pod": "am-767f6b67b5-9zw22" 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG "instance": "10.142.0.107:9100" 22:00:56 DEBUG [...] 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG "pod": "am-767f6b67b5-9zw22" 22:00:56 DEBUG [...] 22:00:56 DEBUG }, 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG "instance": "10.142.0.43:9100" 22:00:56 DEBUG "pod": "ds-idrepo-0" 22:00:56 DEBUG [...] 22:00:56 DEBUG "value": [ 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG }, 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG [...] 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG }, 22:00:56 DEBUG 22:00:56 DEBUG "value": [ 22:00:56 DEBUG 22:00:56 DEBUG "metric": { 22:00:56 DEBUG 22:00:56 DEBUG }, 22:00:56 DEBUG }, 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG "pod": "am-767f6b67b5-vbltz" 22:00:56 DEBUG 22:00:56 DEBUG "value": [ 22:00:56 DEBUG [...] 22:00:56 DEBUG }, 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG [...] 22:00:56 DEBUG 22:00:56 DEBUG "value": [ 22:00:56 DEBUG 22:00:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:00:56 DEBUG 22:00:56 DEBUG "value": [ 22:00:56 DEBUG "value": [ 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG }, 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG "value": [ 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG [...] 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG }, 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG [...] 22:00:56 DEBUG 22:00:56 DEBUG 1713646856, 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG "value": [ 22:00:56 DEBUG 22:00:56 DEBUG [...] 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG "pod": "ds-cts-2" 22:00:56 DEBUG 22:00:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:00:56 DEBUG }, 22:00:56 DEBUG "value": [ 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG "value": [ 22:00:56 DEBUG 22:00:56 DEBUG }, 22:00:56 DEBUG [...] 22:00:56 DEBUG [...] 22:00:56 DEBUG "100226793.33333334" 22:00:56 DEBUG ] 22:00:56 DEBUG "pod": "ds-idrepo-2" 22:00:56 DEBUG [...] 22:00:56 DEBUG 22:00:56 DEBUG [...] 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG "instance": "10.142.0.43:9100" 22:00:56 DEBUG }, 22:00:56 DEBUG 1713646856, 22:00:56 DEBUG 1713646856, 22:00:56 DEBUG "11.83948788211679" 22:00:56 DEBUG ] 22:00:56 DEBUG } 22:00:56 DEBUG ] 22:00:56 DEBUG 1713646856, 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG "pod": "overseer-0-967d957f7-zk66v" 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG "value": [ 22:00:56 DEBUG 1713646856, 22:00:56 DEBUG "0.39500000000005003" 22:00:56 DEBUG ] 22:00:56 DEBUG } 22:00:56 DEBUG ] 22:00:56 DEBUG "3320.4" 22:00:56 DEBUG ] 22:00:56 DEBUG } 22:00:56 DEBUG ] 22:00:56 DEBUG "instance": "10.142.0.43:9100" 22:00:56 DEBUG }, 22:00:56 DEBUG "value": [ 22:00:56 DEBUG "value": [ 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG "pod": "am-767f6b67b5-vbltz" 22:00:56 DEBUG }, 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG [...] 22:00:56 DEBUG } 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG }, 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG [...] 22:00:56 DEBUG "pod": "ds-cts-2" 22:00:56 DEBUG 1713646856, 22:00:56 DEBUG } 22:00:56 DEBUG 22:00:56 DEBUG "pod": "am-767f6b67b5-vbltz" 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG }, 22:00:56 DEBUG }, 22:00:56 DEBUG } 22:00:56 DEBUG "value": [ 22:00:56 DEBUG [...] 22:00:56 DEBUG "pod": "idm-b85f9d978-wnhxb" 22:00:56 DEBUG "pod": "am-767f6b67b5-vbltz" 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG [...] 22:00:56 DEBUG "pod": "ds-idrepo-2" 22:00:56 DEBUG "value": [ 22:00:56 DEBUG 22:00:56 DEBUG } 22:00:56 DEBUG }, 22:00:56 DEBUG "1043402752" 22:00:56 DEBUG "4034202936320" 22:00:56 DEBUG } 22:00:56 DEBUG 22:00:56 DEBUG }, 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG "value": [ 22:00:56 DEBUG "value": [ 22:00:56 DEBUG ] 22:00:56 DEBUG 1713646856, 22:00:56 DEBUG 22:00:56 DEBUG }, 22:00:56 DEBUG }, 22:00:56 DEBUG "value": [ 22:00:56 DEBUG 22:00:56 DEBUG 22:00:56 DEBUG }, 22:00:56 DEBUG "value": [ 22:00:56 DEBUG 1713646856, 22:00:56 DEBUG } 22:00:56 DEBUG "value": [ 22:00:56 DEBUG 1713646856, 22:00:56 DEBUG "0.5145333333333335" 22:00:56 DEBUG 22:00:56 DEBUG "value": [ 22:00:56 DEBUG 22:00:56 DEBUG "instance": "10.142.0.43:9100" 22:00:56 DEBUG 1713646856, 22:00:56 DEBUG 1713646856, 22:00:56 DEBUG } 22:00:56 DEBUG "0" 22:00:56 DEBUG 22:00:56 DEBUG "value": [ 22:00:56 DEBUG 1713646856, 22:00:56 DEBUG } 22:00:56 DEBUG "pod": "am-767f6b67b5-vbltz" 22:00:56 DEBUG }, 22:00:56 DEBUG "value": [ 22:00:56 DEBUG 1713646856, 22:00:56 DEBUG "0" 22:00:56 DEBUG ] 22:00:56 DEBUG ] 22:00:56 DEBUG ] 22:00:56 DEBUG "pod": "ds-cts-2" 22:00:56 DEBUG 1713646856, 22:00:56 DEBUG [...] 22:00:56 DEBUG }, 22:00:56 DEBUG "3320.3333333333335" 22:00:56 DEBUG ] 22:00:56 DEBUG } 22:00:56 DEBUG ] 22:00:56 DEBUG 22:00:56 DEBUG "pod": "ds-idrepo-2" 22:00:56 DEBUG "3333.233333333333" 22:00:56 DEBUG ] 22:00:56 DEBUG "0" 22:00:56 DEBUG 1713646856, 22:00:56 DEBUG 22:00:56 DEBUG ] 22:00:56 DEBUG } 22:00:56 DEBUG } 22:00:56 DEBUG } 22:00:56 DEBUG }, 22:00:56 DEBUG "49.377" 22:00:56 DEBUG 22:00:56 DEBUG "value": [ 22:00:56 DEBUG "0" 22:00:56 DEBUG } 22:00:56 DEBUG } 22:00:56 DEBUG }, 22:00:56 DEBUG 1713646856, 22:00:56 DEBUG 22:00:56 DEBUG } 22:00:56 DEBUG ] 22:00:56 DEBUG "29.566666666666666" 22:00:56 DEBUG 22:00:56 DEBUG } 22:00:56 DEBUG ] 22:00:56 DEBUG ] 22:00:56 DEBUG ] 22:00:56 DEBUG "value": [ 22:00:56 DEBUG ] 22:00:56 DEBUG 22:00:56 DEBUG 1713646856, 22:00:56 DEBUG ] 22:00:56 DEBUG } 22:00:56 DEBUG ] 22:00:56 DEBUG } 22:00:56 DEBUG "0" 22:00:56 DEBUG 22:00:56 DEBUG "pod": "am-767f6b67b5-vbltz" 22:00:56 DEBUG } 22:00:56 DEBUG ] 22:00:56 DEBUG "instance": "10.142.0.43:9100" 22:00:56 DEBUG }, 22:00:56 DEBUG "value": [ 22:00:56 DEBUG } 22:00:56 DEBUG } 22:00:56 DEBUG 1713646856, 22:00:56 DEBUG } 22:00:56 DEBUG 22:00:56 DEBUG "pod": "idm-b85f9d978-wnhxb" 22:00:56 DEBUG ] 22:00:56 DEBUG ] 22:00:56 DEBUG } 22:00:56 DEBUG "value": [ 22:00:56 DEBUG ] 22:00:56 DEBUG ] 22:00:56 DEBUG }, 22:00:56 DEBUG ] 22:00:56 DEBUG } 22:00:56 DEBUG } 22:00:56 DEBUG 1713646856, 22:00:56 DEBUG } 22:00:56 DEBUG } 22:00:56 DEBUG ] 22:00:56 DEBUG "0" 22:00:56 DEBUG ] 22:00:56 DEBUG }, 22:00:56 DEBUG "100222605.69999999" 22:00:56 DEBUG } 22:00:56 DEBUG } 22:00:56 DEBUG 1713646856, 22:00:56 DEBUG } 22:00:56 DEBUG } 22:00:56 DEBUG "value": [ 22:00:56 DEBUG } 22:00:56 DEBUG ] 22:00:56 DEBUG } 22:00:56 DEBUG "0.07133333333334424" 22:00:56 DEBUG } 22:00:56 DEBUG ] 22:00:56 DEBUG } 22:00:56 DEBUG "value": [ 22:00:56 DEBUG ] 22:00:56 DEBUG } 22:00:56 DEBUG } 22:00:56 DEBUG "0" 22:00:56 DEBUG ] 22:00:56 DEBUG } 22:00:56 DEBUG 1713646856, 22:00:56 DEBUG } 22:00:56 DEBUG } 22:00:56 DEBUG ] 22:00:56 DEBUG } 22:00:56 DEBUG } 22:00:56 DEBUG } 22:00:56 DEBUG 1713646856, 22:00:56 DEBUG } 22:00:56 DEBUG ] 22:00:56 DEBUG } 22:00:56 DEBUG "599283.5" 22:00:56 DEBUG } 22:00:56 DEBUG } 22:00:56 DEBUG ] 22:00:56 DEBUG ] 22:00:56 DEBUG "5000.204076173449" 22:00:56 DEBUG ] 22:00:56 DEBUG } 22:00:56 DEBUG ] 22:00:56 DEBUG ] 22:00:56 DEBUG } 22:00:56 DEBUG } 22:00:56 DEBUG ] 22:00:56 DEBUG } 22:00:56 DEBUG } 22:00:56 DEBUG } 22:00:56 DEBUG } 22:00:56 DEBUG ] 22:00:56 DEBUG } 22:00:56 DEBUG } 22:00:56 DEBUG } 22:00:56 DEBUG } 22:00:56 DEBUG } 22:00:56 DEBUG } 22:00:56 DEBUG ] 22:00:56 DEBUG } 22:00:56 DEBUG } 22:00:56 DEBUG } 22:01:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713646913" 22:01:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646914" 22:01:54 ERROR [http_cmd]: ERROR 22:01:54 ERROR --- status code --- 22:01:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:01:54 ERROR --- http response --- 22:01:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:01:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:01:54 INFO [http_cmd]: http status code OK 22:01:54 DEBUG --- status code --- 22:01:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:01:54 DEBUG --- http response --- 22:01:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:01:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646914" 22:01:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713646914" 22:01:54 INFO [http_cmd]: http status code OK 22:01:54 DEBUG --- status code --- 22:01:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:01:54 DEBUG --- http response --- 22:01:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:01:54 DEBUG ----- output ----- 22:01:54 DEBUG { 22:01:54 DEBUG "status": "success", 22:01:54 DEBUG "data": { 22:01:54 DEBUG "resultType": "vector", 22:01:54 DEBUG "result": [ 22:01:54 DEBUG { 22:01:54 DEBUG "metric": { 22:01:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:01:54 DEBUG }, 22:01:54 DEBUG "value": [ 22:01:54 DEBUG 22:01:54 DEBUG 22:01:54 DEBUG 22:01:54 DEBUG [...] 22:01:54 DEBUG 22:01:54 DEBUG 22:01:54 DEBUG 22:01:54 DEBUG "pod": "am-767f6b67b5-vbltz" 22:01:54 DEBUG }, 22:01:54 DEBUG "value": [ 22:01:54 DEBUG 1713646914, 22:01:54 DEBUG "19614.2911234149" 22:01:54 DEBUG ] 22:01:54 DEBUG } 22:01:54 DEBUG ] 22:01:54 DEBUG } 22:01:54 DEBUG } 22:01:54 INFO [http_cmd]: http status code OK 22:01:54 DEBUG --- status code --- 22:01:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:01:54 DEBUG --- http response --- 22:01:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:01:54 DEBUG ----- output ----- 22:01:54 DEBUG { 22:01:54 DEBUG "status": "success", 22:01:54 DEBUG "data": { 22:01:54 DEBUG "resultType": "vector", 22:01:54 DEBUG "result": [ 22:01:54 DEBUG { 22:01:54 DEBUG "metric": { 22:01:54 DEBUG "pod": "ds-idrepo-0" 22:01:54 DEBUG }, 22:01:54 DEBUG "value": [ 22:01:54 DEBUG 22:01:54 DEBUG 22:01:54 DEBUG 22:01:54 DEBUG [...] 22:01:54 DEBUG 22:01:54 DEBUG 22:01:54 DEBUG 22:01:54 DEBUG "pod": "ds-cts-2" 22:01:54 DEBUG }, 22:01:54 DEBUG "value": [ 22:01:54 DEBUG 1713646914, 22:01:54 DEBUG "0" 22:01:54 DEBUG ] 22:01:54 DEBUG } 22:01:54 DEBUG ] 22:01:54 DEBUG } 22:01:54 DEBUG } 22:01:55 INFO 22:01:55 INFO [loop_until]: kubectl --namespace=xlou top pods 22:01:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:01:55 INFO [loop_until]: OK (rc = 0) 22:01:55 DEBUG --- stdout --- 22:01:55 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 21866m 8769Mi am-767f6b67b5-qcqz2 22265m 8889Mi am-767f6b67b5-vbltz 24017m 8942Mi ds-cts-0 19623m 12289Mi ds-cts-1 20087m 12264Mi ds-cts-2 20630m 12482Mi ds-idrepo-0 9693m 3908Mi ds-idrepo-1 9449m 3975Mi ds-idrepo-2 9700m 3878Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-7dd48676f6-tdxl9 18m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-967d957f7-zk66v 11872m 906Mi 22:01:55 DEBUG --- stderr --- 22:01:55 DEBUG 22:01:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646915" 22:01:55 INFO [http_cmd]: http status code OK 22:01:55 DEBUG --- status code --- 22:01:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:01:55 DEBUG --- http response --- 22:01:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:01:55 DEBUG ----- output ----- 22:01:55 DEBUG { 22:01:55 DEBUG "status": "success", 22:01:55 DEBUG "data": { 22:01:55 DEBUG "resultType": "vector", 22:01:55 DEBUG "result": [ 22:01:55 DEBUG { 22:01:55 DEBUG "metric": { 22:01:55 DEBUG "pod": "am-767f6b67b5-9zw22" 22:01:55 DEBUG }, 22:01:55 DEBUG "value": [ 22:01:55 DEBUG 22:01:55 DEBUG 22:01:55 DEBUG 22:01:55 DEBUG [...] 22:01:55 DEBUG 22:01:55 DEBUG 22:01:55 DEBUG 22:01:55 DEBUG "pod": "am-767f6b67b5-vbltz" 22:01:55 DEBUG }, 22:01:55 DEBUG "value": [ 22:01:55 DEBUG 1713646915, 22:01:55 DEBUG "3336" 22:01:55 DEBUG ] 22:01:55 DEBUG } 22:01:55 DEBUG ] 22:01:55 DEBUG } 22:01:55 DEBUG } 22:01:56 INFO 22:01:56 INFO [loop_until]: kubectl --namespace=xlou top node 22:01:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:01:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646916" 22:01:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713646916" 22:01:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646916" 22:01:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646916" 22:01:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646916" 22:01:56 INFO [http_cmd]: http status code OK 22:01:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646916" 22:01:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713646916" 22:01:56 DEBUG --- status code --- 22:01:56 INFO [http_cmd]: http status code OK 22:01:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646916" 22:01:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646916" 22:01:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646916" 22:01:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:01:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646916" 22:01:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713646916" 22:01:56 INFO [http_cmd]: http status code OK 22:01:56 DEBUG --- status code --- 22:01:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646916" 22:01:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646916" 22:01:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646916" 22:01:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646916" 22:01:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646916" 22:01:56 INFO [http_cmd]: http status code OK 22:01:56 DEBUG --- http response --- 22:01:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646916" 22:01:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646916" 22:01:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646916" 22:01:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646916" 22:01:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646916" 22:01:56 INFO [http_cmd]: http status code OK 22:01:57 DEBUG --- status code --- 22:01:57 INFO [http_cmd]: http status code OK 22:01:57 INFO [http_cmd]: http status code OK 22:01:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:01:57 INFO [http_cmd]: http status code OK 22:01:57 INFO [http_cmd]: http status code OK 22:01:57 INFO [http_cmd]: http status code OK 22:01:57 INFO [loop_until]: OK (rc = 0) 22:01:57 INFO [http_cmd]: http status code OK 22:01:57 INFO [http_cmd]: http status code OK 22:01:57 DEBUG --- status code --- 22:01:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:01:57 INFO [http_cmd]: http status code OK 22:01:57 INFO [http_cmd]: http status code OK 22:01:57 INFO [http_cmd]: http status code OK 22:01:57 INFO [http_cmd]: http status code OK 22:01:57 DEBUG --- status code --- 22:01:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:01:57 DEBUG --- status code --- 22:01:57 DEBUG --- status code --- 22:01:57 DEBUG --- http response --- 22:01:57 DEBUG --- status code --- 22:01:57 INFO [http_cmd]: http status code OK 22:01:57 INFO [http_cmd]: http status code OK 22:01:57 DEBUG --- status code --- 22:01:57 DEBUG --- status code --- 22:01:57 DEBUG --- stdout --- 22:01:57 DEBUG --- status code --- 22:01:57 DEBUG --- status code --- 22:01:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:01:57 DEBUG ----- output ----- 22:01:57 INFO [http_cmd]: http status code OK 22:01:57 INFO [http_cmd]: http status code OK 22:01:57 DEBUG --- status code --- 22:01:57 DEBUG --- status code --- 22:01:57 DEBUG --- status code --- 22:01:57 DEBUG --- status code --- 22:01:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:01:57 DEBUG --- http response --- 22:01:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:01:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:01:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:01:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:01:57 DEBUG --- status code --- 22:01:57 DEBUG --- status code --- 22:01:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:01:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:01:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:01:57 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 61m 0% 6214Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 9997m 33% 7673Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 21803m 73% 11486Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 87m 0% 3363Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 20088m 67% 15523Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 162m 0% 5213Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 22509m 75% 11777Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19687m 65% 3061Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 23444m 78% 11182Mi 10% gke-xlou-cdm-ds-0f601751-j488 9989m 33% 7293Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19598m 65% 17347Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 10211m 34% 7942Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20411m 68% 15186Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 12075m 75% 8818Mi 15% 22:01:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:01:57 DEBUG --- http response --- 22:01:57 DEBUG { 22:01:57 DEBUG "status": "success", 22:01:57 DEBUG --- status code --- 22:01:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:01:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:01:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:01:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:01:57 DEBUG --- http response --- 22:01:57 DEBUG --- http response --- 22:01:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:01:57 DEBUG --- http response --- 22:01:57 DEBUG ----- output ----- 22:01:57 DEBUG --- http response --- 22:01:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:01:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:01:57 DEBUG --- http response --- 22:01:57 INFO [http_cmd]: http status code OK 22:01:57 DEBUG --- http response --- 22:01:57 INFO [http_cmd]: http status code OK 22:01:57 DEBUG --- status code --- 22:01:57 DEBUG --- stderr --- 22:01:57 DEBUG 22:01:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:01:57 DEBUG --- status code --- 22:01:57 DEBUG "data": { 22:01:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:01:57 DEBUG --- http response --- 22:01:57 DEBUG --- http response --- 22:01:57 DEBUG --- http response --- 22:01:57 DEBUG --- http response --- 22:01:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:01:57 DEBUG ----- output ----- 22:01:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:01:57 DEBUG { 22:01:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:01:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:01:57 DEBUG --- http response --- 22:01:57 DEBUG --- http response --- 22:01:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:01:57 DEBUG --- status code --- 22:01:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 22:01:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:01:57 DEBUG --- http response --- 22:01:57 DEBUG --- http response --- 22:01:57 DEBUG ----- output ----- 22:01:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:01:57 DEBUG "resultType": "vector", 22:01:57 DEBUG --- http response --- 22:01:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 22:01:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:01:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:01:57 DEBUG ----- output ----- 22:01:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:01:57 DEBUG { 22:01:57 DEBUG ----- output ----- 22:01:57 DEBUG "status": "success", 22:01:57 DEBUG ----- output ----- 22:01:57 DEBUG ----- output ----- 22:01:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:01:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:01:57 DEBUG ----- output ----- 22:01:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:01:57 DEBUG --- http response --- 22:01:57 DEBUG --- http response --- 22:01:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:01:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:01:57 DEBUG { 22:01:57 DEBUG --- http response --- 22:01:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:01:57 DEBUG ----- output ----- 22:01:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:01:57 DEBUG ----- output ----- 22:01:57 DEBUG ----- output ----- 22:01:57 DEBUG { 22:01:57 DEBUG ----- output ----- 22:01:57 DEBUG { 22:01:57 DEBUG "status": "success", 22:01:57 DEBUG "data": { 22:01:57 DEBUG { 22:01:57 DEBUG { 22:01:57 DEBUG ----- output ----- 22:01:57 DEBUG { 22:01:57 DEBUG { 22:01:57 DEBUG ----- output ----- 22:01:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:01:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 22:01:57 DEBUG ----- output ----- 22:01:57 DEBUG ----- output ----- 22:01:57 DEBUG "status": "success", 22:01:57 DEBUG "result": [ 22:01:57 DEBUG ----- output ----- 22:01:57 DEBUG { 22:01:57 DEBUG ----- output ----- 22:01:57 DEBUG { 22:01:57 DEBUG { 22:01:57 DEBUG "status": "success", 22:01:57 DEBUG "status": "success", 22:01:57 DEBUG "data": { 22:01:57 DEBUG "resultType": "vector", 22:01:57 DEBUG "status": "success", 22:01:57 DEBUG "status": "success", 22:01:57 DEBUG ----- output ----- 22:01:57 DEBUG "status": "success", 22:01:57 DEBUG { 22:01:57 DEBUG "status": "success", 22:01:57 DEBUG { 22:01:57 DEBUG ----- output ----- 22:01:57 DEBUG ----- output ----- 22:01:57 DEBUG { 22:01:57 DEBUG { 22:01:57 DEBUG "data": { 22:01:57 DEBUG { 22:01:57 DEBUG { 22:01:57 DEBUG "status": "success", 22:01:57 DEBUG { 22:01:57 DEBUG "status": "success", 22:01:57 DEBUG "status": "success", 22:01:57 DEBUG "data": { 22:01:57 DEBUG "data": { 22:01:57 DEBUG "resultType": "vector", 22:01:57 DEBUG "result": [ 22:01:57 DEBUG "data": { 22:01:57 DEBUG "resultType": "vector", 22:01:57 DEBUG { 22:01:57 DEBUG "data": { 22:01:57 DEBUG "status": "success", 22:01:57 DEBUG "data": { 22:01:57 DEBUG "status": "success", 22:01:57 DEBUG { 22:01:57 DEBUG { 22:01:57 DEBUG "status": "success", 22:01:57 DEBUG "status": "success", 22:01:57 DEBUG "resultType": "vector", 22:01:57 DEBUG "metric": { 22:01:57 DEBUG "status": "success", 22:01:57 DEBUG "data": { 22:01:57 DEBUG "resultType": "vector", 22:01:57 DEBUG "data": { 22:01:57 DEBUG "data": { 22:01:57 DEBUG "resultType": "vector", 22:01:57 DEBUG "resultType": "vector", 22:01:57 DEBUG "result": [ 22:01:57 DEBUG { 22:01:57 DEBUG "result": [ 22:01:57 DEBUG "data": { 22:01:57 DEBUG "status": "success", 22:01:57 DEBUG "resultType": "vector", 22:01:57 DEBUG "result": [ 22:01:57 DEBUG "resultType": "vector", 22:01:57 DEBUG "data": { 22:01:57 DEBUG "status": "success", 22:01:57 DEBUG "status": "success", 22:01:57 DEBUG "data": { 22:01:57 DEBUG "data": { 22:01:57 DEBUG "result": [ 22:01:57 DEBUG { 22:01:57 DEBUG "data": { 22:01:57 DEBUG "result": [ 22:01:57 DEBUG "status": "success", 22:01:57 DEBUG "resultType": "vector", 22:01:57 DEBUG "resultType": "vector", 22:01:57 DEBUG "result": [ 22:01:57 DEBUG "result": [ 22:01:57 DEBUG { 22:01:57 DEBUG "metric": { 22:01:57 DEBUG { 22:01:57 DEBUG "resultType": "vector", 22:01:57 DEBUG "data": { 22:01:57 DEBUG { 22:01:57 DEBUG "data": { 22:01:57 DEBUG "result": [ 22:01:57 DEBUG "resultType": "vector", 22:01:57 DEBUG "data": { 22:01:57 DEBUG "data": { 22:01:57 DEBUG "resultType": "vector", 22:01:57 DEBUG "resultType": "vector", 22:01:57 DEBUG "pod": "am-767f6b67b5-9zw22" 22:01:57 DEBUG "metric": { 22:01:57 DEBUG "resultType": "vector", 22:01:57 DEBUG { 22:01:57 DEBUG "data": { 22:01:57 DEBUG "result": [ 22:01:57 DEBUG "result": [ 22:01:57 DEBUG { 22:01:57 DEBUG { 22:01:57 DEBUG "metric": { 22:01:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:01:57 DEBUG "metric": { 22:01:57 DEBUG "result": [ 22:01:57 DEBUG "resultType": "vector", 22:01:57 DEBUG "metric": { 22:01:57 DEBUG "resultType": "vector", 22:01:57 DEBUG { 22:01:57 DEBUG "result": [ 22:01:57 DEBUG "resultType": "vector", 22:01:57 DEBUG "resultType": "vector", 22:01:57 DEBUG "result": [ 22:01:57 DEBUG "result": [ 22:01:57 DEBUG }, 22:01:57 DEBUG "instance": "10.142.0.107:9100" 22:01:57 DEBUG "result": [ 22:01:57 DEBUG "metric": { 22:01:57 DEBUG "resultType": "vector", 22:01:57 DEBUG { 22:01:57 DEBUG { 22:01:57 DEBUG "metric": { 22:01:57 DEBUG "metric": { 22:01:57 DEBUG "pod": "am-767f6b67b5-9zw22" 22:01:57 DEBUG }, 22:01:57 DEBUG "instance": "10.142.0.107:9100" 22:01:57 DEBUG { 22:01:57 DEBUG "result": [ 22:01:57 DEBUG "pod": "ds-cts-0" 22:01:57 DEBUG "result": [ 22:01:57 DEBUG "metric": { 22:01:57 DEBUG { 22:01:57 DEBUG "result": [ 22:01:57 DEBUG "result": [ 22:01:57 DEBUG { 22:01:57 DEBUG { 22:01:57 DEBUG "value": [ 22:01:57 DEBUG }, 22:01:57 DEBUG { 22:01:57 DEBUG "pod": "ds-idrepo-0" 22:01:57 DEBUG "result": [ 22:01:57 DEBUG "metric": { 22:01:57 DEBUG "metric": { 22:01:57 DEBUG "pod": "ds-idrepo-0" 22:01:57 DEBUG "instance": "10.142.0.107:9100" 22:01:57 DEBUG }, 22:01:57 DEBUG "value": [ 22:01:57 DEBUG }, 22:01:57 DEBUG "metric": { 22:01:57 DEBUG { 22:01:57 DEBUG }, 22:01:57 DEBUG { 22:01:57 DEBUG "pod": "ds-cts-0" 22:01:57 DEBUG "metric": { 22:01:57 DEBUG { 22:01:57 DEBUG { 22:01:57 DEBUG "metric": { 22:01:57 DEBUG "metric": { 22:01:57 DEBUG 22:01:57 DEBUG "value": [ 22:01:57 DEBUG "metric": { 22:01:57 DEBUG }, 22:01:57 DEBUG { 22:01:57 DEBUG "pod": "ds-cts-0" 22:01:57 DEBUG "pod": "am-767f6b67b5-9zw22" 22:01:57 DEBUG }, 22:01:57 DEBUG }, 22:01:57 DEBUG "value": [ 22:01:57 DEBUG 22:01:57 DEBUG "value": [ 22:01:57 DEBUG "pod": "am-767f6b67b5-9zw22" 22:01:57 DEBUG "metric": { 22:01:57 DEBUG "value": [ 22:01:57 DEBUG "metric": { 22:01:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:01:57 DEBUG }, 22:01:57 DEBUG "metric": { 22:01:57 DEBUG "metric": { 22:01:57 DEBUG "pod": "am-767f6b67b5-9zw22" 22:01:57 DEBUG "pod": "ds-idrepo-0" 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:01:57 DEBUG "value": [ 22:01:57 DEBUG "metric": { 22:01:57 DEBUG }, 22:01:57 DEBUG }, 22:01:57 DEBUG "value": [ 22:01:57 DEBUG "value": [ 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG }, 22:01:57 DEBUG "pod": "am-767f6b67b5-9zw22" 22:01:57 DEBUG 22:01:57 DEBUG }, 22:01:57 DEBUG "value": [ 22:01:57 DEBUG "instance": "10.142.0.107:9100" 22:01:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:01:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:01:57 DEBUG }, 22:01:57 DEBUG }, 22:01:57 DEBUG "value": [ 22:01:57 DEBUG 22:01:57 DEBUG }, 22:01:57 DEBUG 22:01:57 DEBUG "instance": "10.142.0.107:9100" 22:01:57 DEBUG "value": [ 22:01:57 DEBUG "value": [ 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG "value": [ 22:01:57 DEBUG }, 22:01:57 DEBUG 22:01:57 DEBUG "value": [ 22:01:57 DEBUG 22:01:57 DEBUG }, 22:01:57 DEBUG }, 22:01:57 DEBUG }, 22:01:57 DEBUG "value": [ 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG "value": [ 22:01:57 DEBUG 22:01:57 DEBUG }, 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG [...] 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG "value": [ 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG "value": [ 22:01:57 DEBUG "value": [ 22:01:57 DEBUG "value": [ 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG [...] 22:01:57 DEBUG [...] 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG [...] 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG [...] 22:01:57 DEBUG 22:01:57 DEBUG [...] 22:01:57 DEBUG 22:01:57 DEBUG [...] 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG [...] 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG "value": [ 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG [...] 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG [...] 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG "pod": "am-767f6b67b5-vbltz" 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG [...] 22:01:57 DEBUG [...] 22:01:57 DEBUG 22:01:57 DEBUG [...] 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG [...] 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG [...] 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG [...] 22:01:57 DEBUG }, 22:01:57 DEBUG 22:01:57 DEBUG [...] 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG [...] 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG [...] 22:01:57 DEBUG [...] 22:01:57 DEBUG [...] 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG "value": [ 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG "pod": "ds-idrepo-2" 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG "pod": "ds-idrepo-2" 22:01:57 DEBUG 22:01:57 DEBUG "pod": "am-767f6b67b5-vbltz" 22:01:57 DEBUG }, 22:01:57 DEBUG "instance": "10.142.0.43:9100" 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG "pod": "ds-cts-2" 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG 1713646916, 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG }, 22:01:57 DEBUG [...] 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG }, 22:01:57 DEBUG 22:01:57 DEBUG }, 22:01:57 DEBUG "value": [ 22:01:57 DEBUG }, 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG }, 22:01:57 DEBUG 22:01:57 DEBUG "pod": "ds-cts-2" 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG "pod": "am-767f6b67b5-vbltz" 22:01:57 DEBUG 22:01:57 DEBUG "3317.2" 22:01:57 DEBUG "instance": "10.142.0.43:9100" 22:01:57 DEBUG 22:01:57 DEBUG "value": [ 22:01:57 DEBUG 22:01:57 DEBUG "pod": "ds-cts-2" 22:01:57 DEBUG "pod": "am-767f6b67b5-vbltz" 22:01:57 DEBUG "value": [ 22:01:57 DEBUG "instance": "10.142.0.43:9100" 22:01:57 DEBUG "value": [ 22:01:57 DEBUG 1713646916, 22:01:57 DEBUG "value": [ 22:01:57 DEBUG "pod": "am-767f6b67b5-vbltz" 22:01:57 DEBUG 22:01:57 DEBUG "value": [ 22:01:57 DEBUG 1713646916, 22:01:57 DEBUG }, 22:01:57 DEBUG "instance": "10.142.0.43:9100" 22:01:57 DEBUG 22:01:57 DEBUG 22:01:57 DEBUG }, 22:01:57 DEBUG "pod": "ds-idrepo-2" 22:01:57 DEBUG ] 22:01:57 DEBUG }, 22:01:57 DEBUG "pod": "overseer-0-967d957f7-zk66v" 22:01:57 DEBUG 1713646916, 22:01:57 DEBUG 22:01:57 DEBUG }, 22:01:57 DEBUG }, 22:01:57 DEBUG 1713646916, 22:01:57 DEBUG "0" 22:01:57 DEBUG 1713646916, 22:01:57 DEBUG "11.822436177913216" 22:01:57 DEBUG ] 22:01:57 DEBUG }, 22:01:57 DEBUG "pod": "am-767f6b67b5-vbltz" 22:01:57 DEBUG "0" 22:01:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:01:57 DEBUG "value": [ 22:01:57 DEBUG }, 22:01:57 DEBUG 22:01:57 DEBUG "pod": "overseer-0-967d957f7-zk66v" 22:01:57 DEBUG "value": [ 22:01:57 DEBUG }, 22:01:57 DEBUG } 22:01:57 DEBUG "value": [ 22:01:57 DEBUG }, 22:01:57 DEBUG "0" 22:01:57 DEBUG 22:01:57 DEBUG "value": [ 22:01:57 DEBUG "value": [ 22:01:57 DEBUG 1713646916, 22:01:57 DEBUG }, 22:01:57 DEBUG "3332" 22:01:57 DEBUG 1713646916, 22:01:57 DEBUG } 22:01:57 DEBUG "value": [ 22:01:57 DEBUG }, 22:01:57 DEBUG ] 22:01:57 DEBUG } 22:01:57 DEBUG 1713646916, 22:01:57 DEBUG "0.5557000000000016" 22:01:57 DEBUG ] 22:01:57 DEBUG }, 22:01:57 DEBUG 1713646916, 22:01:57 DEBUG "value": [ 22:01:57 DEBUG 1713646916, 22:01:57 DEBUG 1713646916, 22:01:57 DEBUG "100495027.53333333" 22:01:57 DEBUG ] 22:01:57 DEBUG "instance": "10.142.0.43:9100" 22:01:57 DEBUG 1713646916, 22:01:57 DEBUG "0" 22:01:57 DEBUG ] 22:01:57 DEBUG "value": [ 22:01:57 DEBUG ] 22:01:57 DEBUG "4039397268480" 22:01:57 DEBUG ] 22:01:57 DEBUG 1713646916, 22:01:57 DEBUG "value": [ 22:01:57 DEBUG ] 22:01:57 DEBUG } 22:01:57 DEBUG "pod": "overseer-0-967d957f7-zk66v" 22:01:57 DEBUG } 22:01:57 DEBUG "value": [ 22:01:57 DEBUG "value": [ 22:01:57 DEBUG "49.377" 22:01:57 DEBUG ] 22:01:57 DEBUG "0" 22:01:57 DEBUG "value": [ 22:01:57 DEBUG ] 22:01:57 DEBUG } 22:01:57 DEBUG }, 22:01:57 DEBUG "28.599999999999998" 22:01:57 DEBUG ] 22:01:57 DEBUG } 22:01:57 DEBUG 1713646916, 22:01:57 DEBUG } 22:01:57 DEBUG ] 22:01:57 DEBUG } 22:01:57 DEBUG } 22:01:57 DEBUG 1713646916, 22:01:57 DEBUG "600025.5" 22:01:57 DEBUG } 22:01:57 DEBUG }, 22:01:57 DEBUG ] 22:01:57 DEBUG 1713646916, 22:01:57 DEBUG 1713646916, 22:01:57 DEBUG ] 22:01:57 DEBUG } 22:01:57 DEBUG ] 22:01:57 DEBUG 1713646916, 22:01:57 DEBUG } 22:01:57 DEBUG ] 22:01:57 DEBUG "value": [ 22:01:57 DEBUG ] 22:01:57 DEBUG } 22:01:57 DEBUG ] 22:01:57 DEBUG "0.3959333333333054" 22:01:57 DEBUG ] 22:01:57 DEBUG } 22:01:57 DEBUG "3337.0333333333333" 22:01:57 DEBUG }, 22:01:57 DEBUG ] 22:01:57 DEBUG "value": [ 22:01:57 DEBUG } 22:01:57 DEBUG "0.07733333333334255" 22:01:57 DEBUG "19.059606502189723" 22:01:57 DEBUG } 22:01:57 DEBUG } 22:01:57 DEBUG } 22:01:57 DEBUG "0" 22:01:57 DEBUG ] 22:01:57 DEBUG } 22:01:57 DEBUG 1713646916, 22:01:57 DEBUG } 22:01:57 DEBUG ] 22:01:57 DEBUG } 22:01:57 DEBUG } 22:01:57 DEBUG } 22:01:57 DEBUG ] 22:01:57 DEBUG ] 22:01:57 DEBUG "value": [ 22:01:57 DEBUG } 22:01:57 DEBUG 1713646916, 22:01:57 DEBUG } 22:01:57 DEBUG ] 22:01:57 DEBUG ] 22:01:57 DEBUG ] 22:01:57 DEBUG ] 22:01:57 DEBUG } 22:01:57 DEBUG ] 22:01:57 DEBUG } 22:01:57 DEBUG "100490259.83333333" 22:01:57 DEBUG ] 22:01:57 DEBUG } 22:01:57 DEBUG ] 22:01:57 DEBUG } 22:01:57 DEBUG } 22:01:57 DEBUG } 22:01:57 DEBUG 1713646916, 22:01:57 DEBUG ] 22:01:57 DEBUG "0" 22:01:57 DEBUG } 22:01:57 DEBUG } 22:01:57 DEBUG } 22:01:57 DEBUG } 22:01:57 DEBUG } 22:01:57 DEBUG } 22:01:57 DEBUG ] 22:01:57 DEBUG } 22:01:57 DEBUG } 22:01:57 DEBUG } 22:01:57 DEBUG } 22:01:57 DEBUG ] 22:01:57 DEBUG ] 22:01:57 DEBUG "1045446656" 22:01:57 DEBUG } 22:01:57 DEBUG ] 22:01:57 DEBUG ] 22:01:57 DEBUG ] 22:01:57 DEBUG } 22:01:57 DEBUG } 22:01:57 DEBUG } 22:01:57 DEBUG } 22:01:57 DEBUG ] 22:01:57 DEBUG } 22:01:57 DEBUG } 22:01:57 DEBUG ] 22:01:57 DEBUG } 22:01:57 DEBUG } 22:01:57 DEBUG } 22:01:57 DEBUG } 22:01:57 DEBUG ] 22:01:57 DEBUG } 22:01:57 DEBUG } 22:01:57 DEBUG } 22:01:57 DEBUG } 22:01:57 DEBUG ] 22:01:57 DEBUG } 22:01:57 DEBUG } 22:01:57 DEBUG } 22:01:57 DEBUG } 22:01:57 DEBUG ] 22:01:57 DEBUG } 22:01:57 DEBUG } 22:01:57 DEBUG } 22:01:57 DEBUG } 22:01:57 DEBUG } 22:02:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713646974" 22:02:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646974" 22:02:54 ERROR [http_cmd]: ERROR 22:02:54 ERROR --- status code --- 22:02:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:02:54 ERROR --- http response --- 22:02:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:02:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:02:54 INFO [http_cmd]: http status code OK 22:02:54 DEBUG --- status code --- 22:02:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:02:54 DEBUG --- http response --- 22:02:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:02:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646974" 22:02:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713646974" 22:02:54 INFO [http_cmd]: http status code OK 22:02:54 DEBUG --- status code --- 22:02:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:02:54 DEBUG --- http response --- 22:02:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 22:02:54 DEBUG ----- output ----- 22:02:54 DEBUG { 22:02:54 DEBUG "status": "success", 22:02:54 DEBUG "data": { 22:02:54 DEBUG "resultType": "vector", 22:02:54 DEBUG "result": [ 22:02:54 DEBUG { 22:02:54 DEBUG "metric": { 22:02:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:02:54 DEBUG }, 22:02:54 DEBUG "value": [ 22:02:54 DEBUG 22:02:54 DEBUG 22:02:54 DEBUG 22:02:54 DEBUG [...] 22:02:54 DEBUG 22:02:54 DEBUG 22:02:54 DEBUG 22:02:54 DEBUG "pod": "overseer-0-967d957f7-zk66v" 22:02:54 DEBUG }, 22:02:54 DEBUG "value": [ 22:02:54 DEBUG 1713646974, 22:02:54 DEBUG "0" 22:02:54 DEBUG ] 22:02:54 DEBUG } 22:02:54 DEBUG ] 22:02:54 DEBUG } 22:02:54 DEBUG } 22:02:54 INFO [http_cmd]: http status code OK 22:02:54 DEBUG --- status code --- 22:02:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:02:54 DEBUG --- http response --- 22:02:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:02:54 DEBUG ----- output ----- 22:02:54 DEBUG { 22:02:54 DEBUG "status": "success", 22:02:54 DEBUG "data": { 22:02:54 DEBUG "resultType": "vector", 22:02:54 DEBUG "result": [ 22:02:54 DEBUG { 22:02:54 DEBUG "metric": { 22:02:54 DEBUG "pod": "ds-idrepo-0" 22:02:54 DEBUG }, 22:02:54 DEBUG "value": [ 22:02:54 DEBUG 22:02:54 DEBUG 22:02:54 DEBUG 22:02:54 DEBUG [...] 22:02:54 DEBUG 22:02:54 DEBUG 22:02:54 DEBUG 22:02:54 DEBUG "pod": "ds-cts-2" 22:02:54 DEBUG }, 22:02:54 DEBUG "value": [ 22:02:54 DEBUG 1713646974, 22:02:54 DEBUG "0" 22:02:54 DEBUG ] 22:02:54 DEBUG } 22:02:54 DEBUG ] 22:02:54 DEBUG } 22:02:54 DEBUG } 22:02:55 INFO 22:02:55 INFO [loop_until]: kubectl --namespace=xlou top pods 22:02:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:02:55 INFO [loop_until]: OK (rc = 0) 22:02:55 DEBUG --- stdout --- 22:02:55 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 21774m 8768Mi am-767f6b67b5-qcqz2 22255m 8889Mi am-767f6b67b5-vbltz 24576m 8951Mi ds-cts-0 20105m 12284Mi ds-cts-1 20204m 12266Mi ds-cts-2 20869m 12486Mi ds-idrepo-0 9429m 3912Mi ds-idrepo-1 9478m 3975Mi ds-idrepo-2 9520m 3877Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-7dd48676f6-tdxl9 16m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-967d957f7-zk66v 11720m 907Mi 22:02:55 DEBUG --- stderr --- 22:02:55 DEBUG 22:02:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646975" 22:02:55 INFO [http_cmd]: http status code OK 22:02:55 DEBUG --- status code --- 22:02:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:02:55 DEBUG --- http response --- 22:02:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:02:55 DEBUG ----- output ----- 22:02:55 DEBUG { 22:02:55 DEBUG "status": "success", 22:02:55 DEBUG "data": { 22:02:55 DEBUG "resultType": "vector", 22:02:55 DEBUG "result": [ 22:02:55 DEBUG { 22:02:55 DEBUG "metric": { 22:02:55 DEBUG "pod": "am-767f6b67b5-9zw22" 22:02:55 DEBUG }, 22:02:55 DEBUG "value": [ 22:02:55 DEBUG 22:02:55 DEBUG 22:02:55 DEBUG 22:02:55 DEBUG [...] 22:02:55 DEBUG 22:02:55 DEBUG 22:02:55 DEBUG 22:02:55 DEBUG "pod": "am-767f6b67b5-vbltz" 22:02:55 DEBUG }, 22:02:55 DEBUG "value": [ 22:02:55 DEBUG 1713646975, 22:02:55 DEBUG "3339.7666666666664" 22:02:55 DEBUG ] 22:02:55 DEBUG } 22:02:55 DEBUG ] 22:02:55 DEBUG } 22:02:55 DEBUG } 22:02:57 INFO 22:02:57 INFO [loop_until]: kubectl --namespace=xlou top node 22:02:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:02:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713646977" 22:02:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646977" 22:02:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646977" 22:02:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646977" 22:02:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646977" 22:02:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646977" 22:02:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646977" 22:02:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646977" 22:02:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646977" 22:02:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646977" 22:02:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713646977" 22:02:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646977" 22:02:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646977" 22:02:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646977" 22:02:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646977" 22:02:57 INFO [http_cmd]: http status code OK 22:02:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646977" 22:02:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646977" 22:02:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646977" 22:02:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713646977" 22:02:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646977" 22:02:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713646977" 22:02:57 INFO [http_cmd]: http status code OK 22:02:57 INFO [http_cmd]: http status code OK 22:02:57 INFO [http_cmd]: http status code OK 22:02:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713646977" 22:02:57 INFO [loop_until]: OK (rc = 0) 22:02:57 DEBUG --- stdout --- 22:02:57 INFO [http_cmd]: http status code OK 22:02:57 INFO [http_cmd]: http status code OK 22:02:57 INFO [http_cmd]: http status code OK 22:02:57 DEBUG --- status code --- 22:02:57 INFO [http_cmd]: http status code OK 22:02:57 INFO [http_cmd]: http status code OK 22:02:57 INFO [http_cmd]: http status code OK 22:02:57 INFO [http_cmd]: http status code OK 22:02:57 INFO [http_cmd]: http status code OK 22:02:57 INFO [http_cmd]: http status code OK 22:02:57 DEBUG --- status code --- 22:02:57 INFO [http_cmd]: http status code OK 22:02:57 DEBUG --- status code --- 22:02:57 DEBUG --- status code --- 22:02:57 INFO [http_cmd]: http status code OK 22:02:57 INFO [http_cmd]: http status code OK 22:02:57 DEBUG --- status code --- 22:02:57 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 6223Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 9972m 33% 7660Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 21787m 72% 11489Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 79m 0% 3373Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 19921m 66% 15521Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 166m 0% 5191Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 22309m 74% 11792Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19728m 66% 3058Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 24702m 82% 11196Mi 10% gke-xlou-cdm-ds-0f601751-j488 10373m 34% 7291Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19929m 66% 17348Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 10207m 34% 7957Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20422m 68% 15189Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 12094m 76% 8817Mi 15% 22:02:57 INFO [http_cmd]: http status code OK 22:02:57 DEBUG --- status code --- 22:02:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:02:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:02:57 DEBUG --- status code --- 22:02:57 DEBUG --- status code --- 22:02:57 DEBUG --- status code --- 22:02:57 DEBUG --- status code --- 22:02:57 DEBUG --- status code --- 22:02:57 DEBUG --- status code --- 22:02:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:02:57 DEBUG --- status code --- 22:02:57 INFO [http_cmd]: http status code OK 22:02:57 INFO [http_cmd]: http status code OK 22:02:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:02:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:02:57 DEBUG --- status code --- 22:02:57 DEBUG --- status code --- 22:02:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:02:57 INFO [http_cmd]: http status code OK 22:02:57 DEBUG --- stderr --- 22:02:57 DEBUG --- status code --- 22:02:57 DEBUG --- http response --- 22:02:57 DEBUG --- status code --- 22:02:57 DEBUG --- http response --- 22:02:57 INFO [http_cmd]: http status code OK 22:02:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:02:57 DEBUG --- http response --- 22:02:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:02:57 INFO [http_cmd]: http status code OK 22:02:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:02:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:02:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:02:57 DEBUG --- http response --- 22:02:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:02:57 DEBUG --- status code --- 22:02:57 DEBUG --- status code --- 22:02:57 DEBUG --- http response --- 22:02:57 DEBUG --- http response --- 22:02:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:02:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:02:57 DEBUG --- http response --- 22:02:57 DEBUG --- status code --- 22:02:57 DEBUG 22:02:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:02:57 DEBUG --- http response --- 22:02:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:02:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:02:57 DEBUG --- status code --- 22:02:57 DEBUG ----- output ----- 22:02:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:02:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:02:57 DEBUG --- status code --- 22:02:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:02:57 DEBUG --- http response --- 22:02:57 DEBUG --- http response --- 22:02:57 DEBUG --- http response --- 22:02:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:02:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:02:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:02:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:02:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:02:57 DEBUG --- http response --- 22:02:57 DEBUG --- http response --- 22:02:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:02:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:02:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:02:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:02:57 DEBUG --- http response --- 22:02:57 DEBUG ----- output ----- 22:02:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:02:57 DEBUG { 22:02:57 DEBUG --- http response --- 22:02:57 DEBUG --- http response --- 22:02:57 DEBUG --- http response --- 22:02:57 DEBUG --- http response --- 22:02:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:02:57 DEBUG ----- output ----- 22:02:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:02:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:02:57 DEBUG --- http response --- 22:02:57 DEBUG --- http response --- 22:02:57 DEBUG ----- output ----- 22:02:57 DEBUG ----- output ----- 22:02:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:02:57 DEBUG ----- output ----- 22:02:57 DEBUG ----- output ----- 22:02:57 DEBUG { 22:02:57 DEBUG ----- output ----- 22:02:57 DEBUG ----- output ----- 22:02:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:02:57 DEBUG { 22:02:57 DEBUG --- http response --- 22:02:57 DEBUG "status": "success", 22:02:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:02:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:02:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:02:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:02:57 DEBUG ----- output ----- 22:02:57 DEBUG { 22:02:57 DEBUG ----- output ----- 22:02:57 DEBUG ----- output ----- 22:02:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:02:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:02:57 DEBUG ----- output ----- 22:02:57 DEBUG { 22:02:57 DEBUG { 22:02:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:02:57 DEBUG "status": "success", 22:02:57 DEBUG --- http response --- 22:02:57 DEBUG { 22:02:57 DEBUG { 22:02:57 DEBUG ----- output ----- 22:02:57 DEBUG "status": "success", 22:02:57 DEBUG "data": { 22:02:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:02:57 DEBUG ----- output ----- 22:02:57 DEBUG ----- output ----- 22:02:57 DEBUG ----- output ----- 22:02:57 DEBUG ----- output ----- 22:02:57 DEBUG { 22:02:57 DEBUG "status": "success", 22:02:57 DEBUG { 22:02:57 DEBUG { 22:02:57 DEBUG ----- output ----- 22:02:57 DEBUG { 22:02:57 DEBUG { 22:02:57 DEBUG "status": "success", 22:02:57 DEBUG "status": "success", 22:02:57 DEBUG "data": { 22:02:57 DEBUG "data": { 22:02:57 DEBUG "resultType": "vector", 22:02:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:02:57 DEBUG "status": "success", 22:02:57 DEBUG { 22:02:57 DEBUG "data": { 22:02:57 DEBUG "resultType": "vector", 22:02:57 DEBUG ----- output ----- 22:02:57 DEBUG { 22:02:57 DEBUG { 22:02:57 DEBUG { 22:02:57 DEBUG { 22:02:57 DEBUG "status": "success", 22:02:57 DEBUG "status": "success", 22:02:57 DEBUG "data": { 22:02:57 DEBUG "status": "success", 22:02:57 DEBUG { 22:02:57 DEBUG "status": "success", 22:02:57 DEBUG "status": "success", 22:02:57 DEBUG "data": { 22:02:57 DEBUG "resultType": "vector", 22:02:57 DEBUG ----- output ----- 22:02:57 DEBUG "result": [ 22:02:57 DEBUG "status": "success", 22:02:57 DEBUG ----- output ----- 22:02:57 DEBUG "data": { 22:02:57 DEBUG "status": "success", 22:02:57 DEBUG "resultType": "vector", 22:02:57 DEBUG "result": [ 22:02:57 DEBUG { 22:02:57 DEBUG "status": "success", 22:02:57 DEBUG "status": "success", 22:02:57 DEBUG "status": "success", 22:02:57 DEBUG "data": { 22:02:57 DEBUG "resultType": "vector", 22:02:57 DEBUG "status": "success", 22:02:57 DEBUG "resultType": "vector", 22:02:57 DEBUG "data": { 22:02:57 DEBUG "status": "success", 22:02:57 DEBUG "data": { 22:02:57 DEBUG "data": { 22:02:57 DEBUG "resultType": "vector", 22:02:57 DEBUG "result": [ 22:02:57 DEBUG { 22:02:57 DEBUG { 22:02:57 DEBUG "data": { 22:02:57 DEBUG { 22:02:57 DEBUG "resultType": "vector", 22:02:57 DEBUG "data": { 22:02:57 DEBUG "result": [ 22:02:57 DEBUG { 22:02:57 DEBUG "status": "success", 22:02:57 DEBUG "data": { 22:02:57 DEBUG "data": { 22:02:57 DEBUG "data": { 22:02:57 DEBUG "result": [ 22:02:57 DEBUG "data": { 22:02:57 DEBUG "data": { 22:02:57 DEBUG "result": [ 22:02:57 DEBUG "resultType": "vector", 22:02:57 DEBUG "data": { 22:02:57 DEBUG "resultType": "vector", 22:02:57 DEBUG "resultType": "vector", 22:02:57 DEBUG "result": [ 22:02:57 DEBUG { 22:02:57 DEBUG "status": "success", 22:02:57 DEBUG "metric": { 22:02:57 DEBUG "resultType": "vector", 22:02:57 DEBUG "status": "success", 22:02:57 DEBUG "result": [ 22:02:57 DEBUG "resultType": "vector", 22:02:57 DEBUG { 22:02:57 DEBUG "metric": { 22:02:57 DEBUG "data": { 22:02:57 DEBUG "resultType": "vector", 22:02:57 DEBUG "resultType": "vector", 22:02:57 DEBUG "resultType": "vector", 22:02:57 DEBUG { 22:02:57 DEBUG "resultType": "vector", 22:02:57 DEBUG "resultType": "vector", 22:02:57 DEBUG { 22:02:57 DEBUG "result": [ 22:02:57 DEBUG "resultType": "vector", 22:02:57 DEBUG "result": [ 22:02:57 DEBUG "result": [ 22:02:57 DEBUG { 22:02:57 DEBUG "metric": { 22:02:57 DEBUG "data": { 22:02:57 DEBUG "pod": "am-767f6b67b5-9zw22" 22:02:57 DEBUG "result": [ 22:02:57 DEBUG "data": { 22:02:57 DEBUG { 22:02:57 DEBUG "result": [ 22:02:57 DEBUG "metric": { 22:02:57 DEBUG "pod": "am-767f6b67b5-9zw22" 22:02:57 DEBUG "resultType": "vector", 22:02:57 DEBUG "result": [ 22:02:57 DEBUG "result": [ 22:02:57 DEBUG "result": [ 22:02:57 DEBUG "metric": { 22:02:57 DEBUG "result": [ 22:02:57 DEBUG "result": [ 22:02:57 DEBUG "metric": { 22:02:57 DEBUG { 22:02:57 DEBUG "result": [ 22:02:57 DEBUG { 22:02:57 DEBUG { 22:02:57 DEBUG "metric": { 22:02:57 DEBUG "pod": "ds-idrepo-0" 22:02:57 DEBUG "resultType": "vector", 22:02:57 DEBUG }, 22:02:57 DEBUG { 22:02:57 DEBUG "resultType": "vector", 22:02:57 DEBUG "metric": { 22:02:57 DEBUG { 22:02:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:02:57 DEBUG }, 22:02:57 DEBUG "value": [ 22:02:57 DEBUG { 22:02:57 DEBUG { 22:02:57 DEBUG { 22:02:57 DEBUG "pod": "ds-cts-0" 22:02:57 DEBUG { 22:02:57 DEBUG { 22:02:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:02:57 DEBUG "metric": { 22:02:57 DEBUG { 22:02:57 DEBUG "metric": { 22:02:57 DEBUG "metric": { 22:02:57 DEBUG "pod": "am-767f6b67b5-9zw22" 22:02:57 DEBUG }, 22:02:57 DEBUG "result": [ 22:02:57 DEBUG "value": [ 22:02:57 DEBUG "metric": { 22:02:57 DEBUG "result": [ 22:02:57 DEBUG { 22:02:57 DEBUG "metric": { 22:02:57 DEBUG }, 22:02:57 DEBUG "result": [ 22:02:57 DEBUG 22:02:57 DEBUG "metric": { 22:02:57 DEBUG "metric": { 22:02:57 DEBUG "metric": { 22:02:57 DEBUG }, 22:02:57 DEBUG "metric": { 22:02:57 DEBUG "metric": { 22:02:57 DEBUG }, 22:02:57 DEBUG "pod": "ds-cts-0" 22:02:57 DEBUG "metric": { 22:02:57 DEBUG "instance": "10.142.0.107:9100" 22:02:57 DEBUG "pod": "ds-cts-0" 22:02:57 DEBUG }, 22:02:57 DEBUG "value": [ 22:02:57 DEBUG { 22:02:57 DEBUG 22:02:57 DEBUG "pod": "ds-idrepo-0" 22:02:57 DEBUG "metric": { 22:02:57 DEBUG "pod": "am-767f6b67b5-9zw22" 22:02:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:02:57 DEBUG "value": [ 22:02:57 DEBUG { 22:02:57 DEBUG 22:02:57 DEBUG "pod": "am-767f6b67b5-9zw22" 22:02:57 DEBUG "instance": "10.142.0.107:9100" 22:02:57 DEBUG "pod": "am-767f6b67b5-9zw22" 22:02:57 DEBUG "value": [ 22:02:57 DEBUG "pod": "ds-idrepo-0" 22:02:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:02:57 DEBUG "value": [ 22:02:57 DEBUG }, 22:02:57 DEBUG "value": [ 22:02:57 DEBUG 22:02:57 DEBUG }, 22:02:57 DEBUG "value": [ 22:02:57 DEBUG 22:02:57 DEBUG "metric": { 22:02:57 DEBUG 22:02:57 DEBUG }, 22:02:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:02:57 DEBUG }, 22:02:57 DEBUG }, 22:02:57 DEBUG 22:02:57 DEBUG "metric": { 22:02:57 DEBUG 22:02:57 DEBUG }, 22:02:57 DEBUG }, 22:02:57 DEBUG }, 22:02:57 DEBUG 22:02:57 DEBUG }, 22:02:57 DEBUG }, 22:02:57 DEBUG 22:02:57 DEBUG "instance": "10.142.0.107:9100" 22:02:57 DEBUG 22:02:57 DEBUG }, 22:02:57 DEBUG "value": [ 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG "instance": "10.142.0.107:9100" 22:02:57 DEBUG 22:02:57 DEBUG "value": [ 22:02:57 DEBUG }, 22:02:57 DEBUG "value": [ 22:02:57 DEBUG "value": [ 22:02:57 DEBUG 22:02:57 DEBUG "instance": "10.142.0.107:9100" 22:02:57 DEBUG [...] 22:02:57 DEBUG "value": [ 22:02:57 DEBUG "value": [ 22:02:57 DEBUG "value": [ 22:02:57 DEBUG 22:02:57 DEBUG "value": [ 22:02:57 DEBUG "value": [ 22:02:57 DEBUG 22:02:57 DEBUG }, 22:02:57 DEBUG 22:02:57 DEBUG "value": [ 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG }, 22:02:57 DEBUG [...] 22:02:57 DEBUG 22:02:57 DEBUG "value": [ 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG }, 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG "value": [ 22:02:57 DEBUG [...] 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG [...] 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG [...] 22:02:57 DEBUG "value": [ 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG [...] 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG [...] 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG [...] 22:02:57 DEBUG "value": [ 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG [...] 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG [...] 22:02:57 DEBUG 22:02:57 DEBUG [...] 22:02:57 DEBUG [...] 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG "pod": "am-767f6b67b5-vbltz" 22:02:57 DEBUG [...] 22:02:57 DEBUG [...] 22:02:57 DEBUG [...] 22:02:57 DEBUG 22:02:57 DEBUG [...] 22:02:57 DEBUG [...] 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG [...] 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG "pod": "ds-idrepo-2" 22:02:57 DEBUG "pod": "am-767f6b67b5-vbltz" 22:02:57 DEBUG 22:02:57 DEBUG [...] 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG }, 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG [...] 22:02:57 DEBUG "pod": "ds-cts-2" 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG }, 22:02:57 DEBUG }, 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:02:57 DEBUG [...] 22:02:57 DEBUG "value": [ 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG "pod": "ds-cts-2" 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG "pod": "overseer-0-967d957f7-zk66v" 22:02:57 DEBUG 22:02:57 DEBUG }, 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG "pod": "am-767f6b67b5-vbltz" 22:02:57 DEBUG [...] 22:02:57 DEBUG "value": [ 22:02:57 DEBUG "value": [ 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG }, 22:02:57 DEBUG 22:02:57 DEBUG 1713646977, 22:02:57 DEBUG 22:02:57 DEBUG "instance": "10.142.0.43:9100" 22:02:57 DEBUG 22:02:57 DEBUG }, 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG }, 22:02:57 DEBUG 22:02:57 DEBUG "value": [ 22:02:57 DEBUG "instance": "10.142.0.43:9100" 22:02:57 DEBUG 22:02:57 DEBUG }, 22:02:57 DEBUG 22:02:57 DEBUG 1713646977, 22:02:57 DEBUG 1713646977, 22:02:57 DEBUG "pod": "ds-idrepo-2" 22:02:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:02:57 DEBUG 22:02:57 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 22:02:57 DEBUG "value": [ 22:02:57 DEBUG 22:02:57 DEBUG 22:02:57 DEBUG "0" 22:02:57 DEBUG ] 22:02:57 DEBUG 22:02:57 DEBUG "value": [ 22:02:57 DEBUG }, 22:02:57 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 22:02:57 DEBUG "value": [ 22:02:57 DEBUG 22:02:57 DEBUG 1713646977, 22:02:57 DEBUG }, 22:02:57 DEBUG "pod": "ds-cts-2" 22:02:57 DEBUG "value": [ 22:02:57 DEBUG 22:02:57 DEBUG "0" 22:02:57 DEBUG "3339.233333333333" 22:02:57 DEBUG ] 22:02:57 DEBUG }, 22:02:57 DEBUG "pod": "am-767f6b67b5-vbltz" 22:02:57 DEBUG }, 22:02:57 DEBUG 1713646977, 22:02:57 DEBUG "pod": "am-767f6b67b5-vbltz" 22:02:57 DEBUG "instance": "10.142.0.43:9100" 22:02:57 DEBUG } 22:02:57 DEBUG "pod": "am-767f6b67b5-vbltz" 22:02:57 DEBUG }, 22:02:57 DEBUG "pod": "ds-idrepo-2" 22:02:57 DEBUG "value": [ 22:02:57 DEBUG }, 22:02:57 DEBUG 1713646977, 22:02:57 DEBUG "instance": "10.142.0.43:9100" 22:02:57 DEBUG }, 22:02:57 DEBUG "value": [ 22:02:57 DEBUG }, 22:02:57 DEBUG 1713646977, 22:02:57 DEBUG 22:02:57 DEBUG ] 22:02:57 DEBUG }, 22:02:57 DEBUG } 22:02:57 DEBUG "value": [ 22:02:57 DEBUG }, 22:02:57 DEBUG "value": [ 22:02:57 DEBUG "11.777432963847517" 22:02:57 DEBUG }, 22:02:57 DEBUG }, 22:02:57 DEBUG ] 22:02:57 DEBUG 1713646977, 22:02:57 DEBUG "value": [ 22:02:57 DEBUG }, 22:02:57 DEBUG 1713646977, 22:02:57 DEBUG "value": [ 22:02:57 DEBUG 1713646977, 22:02:57 DEBUG "0" 22:02:57 DEBUG "32.96666666666667" 22:02:57 DEBUG 1713646977, 22:02:57 DEBUG "value": [ 22:02:57 DEBUG "3330.5" 22:02:57 DEBUG "instance": "10.142.0.43:9100" 22:02:57 DEBUG } 22:02:57 DEBUG "value": [ 22:02:57 DEBUG ] 22:02:57 DEBUG 1713646977, 22:02:57 DEBUG "value": [ 22:02:57 DEBUG 1713646977, 22:02:57 DEBUG ] 22:02:57 DEBUG "value": [ 22:02:57 DEBUG "value": [ 22:02:57 DEBUG } 22:02:57 DEBUG "0.5415333333333326" 22:02:57 DEBUG 1713646977, 22:02:57 DEBUG "value": [ 22:02:57 DEBUG "4044679988224" 22:02:57 DEBUG ] 22:02:57 DEBUG "value": [ 22:02:57 DEBUG 1713646977, 22:02:57 DEBUG "99960847.53333333" 22:02:57 DEBUG 1713646977, 22:02:57 DEBUG "0" 22:02:57 DEBUG ] 22:02:57 DEBUG }, 22:02:57 DEBUG ] 22:02:57 DEBUG 1713646977, 22:02:57 DEBUG "0" 22:02:57 DEBUG "1047199744" 22:02:57 DEBUG 1713646977, 22:02:57 DEBUG "3341.0666666666666" 22:02:57 DEBUG } 22:02:57 DEBUG 1713646977, 22:02:57 DEBUG 1713646977, 22:02:57 DEBUG } 22:02:57 DEBUG ] 22:02:57 DEBUG "49.377" 22:02:57 DEBUG 1713646977, 22:02:57 DEBUG ] 22:02:57 DEBUG } 22:02:57 DEBUG ] 22:02:57 DEBUG ] 22:02:57 DEBUG "0.4054333333333678" 22:02:57 DEBUG "0" 22:02:57 DEBUG ] 22:02:57 DEBUG } 22:02:57 DEBUG "value": [ 22:02:57 DEBUG } 22:02:57 DEBUG ] 22:02:57 DEBUG } 22:02:57 DEBUG ] 22:02:57 DEBUG "0" 22:02:57 DEBUG ] 22:02:57 DEBUG ] 22:02:57 DEBUG "599792.5" 22:02:57 DEBUG ] 22:02:57 DEBUG } 22:02:57 DEBUG ] 22:02:57 DEBUG "0" 22:02:57 DEBUG } 22:02:57 DEBUG ] 22:02:57 DEBUG } 22:02:57 DEBUG } 22:02:57 DEBUG ] 22:02:57 DEBUG ] 22:02:57 DEBUG } 22:02:57 DEBUG ] 22:02:57 DEBUG 1713646977, 22:02:57 DEBUG } 22:02:57 DEBUG } 22:02:57 DEBUG } 22:02:57 DEBUG } 22:02:57 DEBUG ] 22:02:57 DEBUG } 22:02:57 DEBUG } 22:02:57 DEBUG } 22:02:57 DEBUG ] 22:02:57 DEBUG ] 22:02:57 DEBUG } 22:02:57 DEBUG ] 22:02:57 DEBUG ] 22:02:57 DEBUG } 22:02:57 DEBUG ] 22:02:57 DEBUG ] 22:02:57 DEBUG } 22:02:57 DEBUG } 22:02:57 DEBUG ] 22:02:57 DEBUG } 22:02:57 DEBUG } 22:02:57 DEBUG ] 22:02:57 DEBUG ] 22:02:57 DEBUG ] 22:02:57 DEBUG } 22:02:57 DEBUG } 22:02:57 DEBUG } 22:02:57 DEBUG "0.07733333333331414" 22:02:57 DEBUG } 22:02:57 DEBUG } 22:02:57 DEBUG } 22:02:57 DEBUG } 22:02:57 DEBUG } 22:02:57 DEBUG } 22:02:57 DEBUG } 22:02:57 DEBUG ] 22:02:57 DEBUG ] 22:02:57 DEBUG } 22:02:57 DEBUG } 22:02:57 DEBUG "99964126.1" 22:02:57 DEBUG } 22:02:57 DEBUG } 22:02:57 DEBUG ] 22:02:57 DEBUG } 22:02:57 DEBUG ] 22:02:57 DEBUG ] 22:02:57 DEBUG ] 22:02:57 DEBUG } 22:02:57 DEBUG } 22:02:57 DEBUG } 22:02:57 DEBUG } 22:02:57 DEBUG } 22:02:57 DEBUG } 22:02:57 DEBUG } 22:02:57 DEBUG ] 22:02:57 DEBUG } 22:02:57 DEBUG } 22:02:57 DEBUG } 22:02:57 DEBUG } 22:02:57 DEBUG } 22:02:57 DEBUG } 22:02:57 DEBUG } 22:02:57 DEBUG } 22:02:57 DEBUG } 22:02:57 DEBUG ] 22:02:57 DEBUG } 22:02:57 DEBUG } 22:02:57 DEBUG } 22:02:57 DEBUG ] 22:02:57 DEBUG } 22:02:57 DEBUG } 22:02:57 DEBUG } 22:02:57 DEBUG } 22:03:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713647034" 22:03:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647034" 22:03:54 INFO [http_cmd]: http status code OK 22:03:54 DEBUG --- status code --- 22:03:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:03:54 DEBUG --- http response --- 22:03:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:03:54 ERROR [http_cmd]: ERROR 22:03:54 ERROR --- status code --- 22:03:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:03:54 ERROR --- http response --- 22:03:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:03:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:03:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647034" 22:03:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713647034" 22:03:54 INFO [http_cmd]: http status code OK 22:03:54 DEBUG --- status code --- 22:03:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:03:54 DEBUG --- http response --- 22:03:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:03:54 DEBUG ----- output ----- 22:03:54 DEBUG { 22:03:54 DEBUG "status": "success", 22:03:54 DEBUG "data": { 22:03:54 DEBUG "resultType": "vector", 22:03:54 DEBUG "result": [ 22:03:54 DEBUG { 22:03:54 DEBUG "metric": { 22:03:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:03:54 DEBUG }, 22:03:54 DEBUG "value": [ 22:03:54 DEBUG 22:03:54 DEBUG 22:03:54 DEBUG 22:03:54 DEBUG [...] 22:03:54 DEBUG 22:03:54 DEBUG 22:03:54 DEBUG 22:03:54 DEBUG "pod": "am-767f6b67b5-9zw22" 22:03:54 DEBUG }, 22:03:54 DEBUG "value": [ 22:03:54 DEBUG 1713647034, 22:03:54 DEBUG "0" 22:03:54 DEBUG ] 22:03:54 DEBUG } 22:03:54 DEBUG ] 22:03:54 DEBUG } 22:03:54 DEBUG } 22:03:54 INFO [http_cmd]: http status code OK 22:03:54 DEBUG --- status code --- 22:03:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:03:54 DEBUG --- http response --- 22:03:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:03:54 DEBUG ----- output ----- 22:03:54 DEBUG { 22:03:54 DEBUG "status": "success", 22:03:54 DEBUG "data": { 22:03:54 DEBUG "resultType": "vector", 22:03:54 DEBUG "result": [ 22:03:54 DEBUG { 22:03:54 DEBUG "metric": { 22:03:54 DEBUG "pod": "ds-idrepo-0" 22:03:54 DEBUG }, 22:03:54 DEBUG "value": [ 22:03:54 DEBUG 22:03:54 DEBUG 22:03:54 DEBUG 22:03:54 DEBUG [...] 22:03:54 DEBUG 22:03:54 DEBUG 22:03:54 DEBUG 22:03:54 DEBUG "pod": "ds-cts-2" 22:03:54 DEBUG }, 22:03:54 DEBUG "value": [ 22:03:54 DEBUG 1713647034, 22:03:54 DEBUG "0" 22:03:54 DEBUG ] 22:03:54 DEBUG } 22:03:54 DEBUG ] 22:03:54 DEBUG } 22:03:54 DEBUG } 22:03:55 INFO 22:03:55 INFO [loop_until]: kubectl --namespace=xlou top pods 22:03:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:03:55 INFO [loop_until]: OK (rc = 0) 22:03:55 DEBUG --- stdout --- 22:03:55 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 21747m 8770Mi am-767f6b67b5-qcqz2 21960m 8892Mi am-767f6b67b5-vbltz 24254m 8947Mi ds-cts-0 20299m 12284Mi ds-cts-1 19246m 12278Mi ds-cts-2 20955m 12476Mi ds-idrepo-0 9472m 3912Mi ds-idrepo-1 9661m 3975Mi ds-idrepo-2 9864m 3878Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-7dd48676f6-tdxl9 15m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-967d957f7-zk66v 11566m 910Mi 22:03:55 DEBUG --- stderr --- 22:03:55 DEBUG 22:03:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647036" 22:03:56 INFO [http_cmd]: http status code OK 22:03:56 DEBUG --- status code --- 22:03:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:03:56 DEBUG --- http response --- 22:03:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:03:56 DEBUG ----- output ----- 22:03:56 DEBUG { 22:03:56 DEBUG "status": "success", 22:03:56 DEBUG "data": { 22:03:56 DEBUG "resultType": "vector", 22:03:56 DEBUG "result": [ 22:03:56 DEBUG { 22:03:56 DEBUG "metric": { 22:03:56 DEBUG "pod": "am-767f6b67b5-9zw22" 22:03:56 DEBUG }, 22:03:56 DEBUG "value": [ 22:03:56 DEBUG 22:03:56 DEBUG 22:03:56 DEBUG 22:03:56 DEBUG [...] 22:03:56 DEBUG 22:03:56 DEBUG 22:03:56 DEBUG 22:03:56 DEBUG "pod": "am-767f6b67b5-vbltz" 22:03:56 DEBUG }, 22:03:56 DEBUG "value": [ 22:03:56 DEBUG 1713647036, 22:03:56 DEBUG "3331.8333333333335" 22:03:56 DEBUG ] 22:03:56 DEBUG } 22:03:56 DEBUG ] 22:03:56 DEBUG } 22:03:56 DEBUG } 22:03:57 INFO 22:03:57 INFO [loop_until]: kubectl --namespace=xlou top node 22:03:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:03:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647037" 22:03:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647037" 22:03:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647037" 22:03:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647037" 22:03:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713647037" 22:03:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647037" 22:03:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647037" 22:03:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647037" 22:03:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647037" 22:03:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647037" 22:03:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647037" 22:03:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647037" 22:03:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647037" 22:03:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713647037" 22:03:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647037" 22:03:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647037" 22:03:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647037" 22:03:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713647037" 22:03:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647037" 22:03:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647037" 22:03:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647037" 22:03:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647037" 22:03:57 INFO [loop_until]: OK (rc = 0) 22:03:57 INFO [http_cmd]: http status code OK 22:03:57 INFO [http_cmd]: http status code OK 22:03:57 INFO [http_cmd]: http status code OK 22:03:57 INFO [http_cmd]: http status code OK 22:03:57 INFO [http_cmd]: http status code OK 22:03:57 INFO [http_cmd]: http status code OK 22:03:57 DEBUG --- status code --- 22:03:57 INFO [http_cmd]: http status code OK 22:03:57 INFO [http_cmd]: http status code OK 22:03:57 DEBUG --- status code --- 22:03:57 INFO [http_cmd]: http status code OK 22:03:57 INFO [http_cmd]: http status code OK 22:03:57 INFO [http_cmd]: http status code OK 22:03:57 INFO [http_cmd]: http status code OK 22:03:57 DEBUG --- status code --- 22:03:57 INFO [http_cmd]: http status code OK 22:03:57 INFO [http_cmd]: http status code OK 22:03:57 INFO [http_cmd]: http status code OK 22:03:57 DEBUG --- status code --- 22:03:57 DEBUG --- status code --- 22:03:57 INFO [http_cmd]: http status code OK 22:03:57 INFO [http_cmd]: http status code OK 22:03:57 DEBUG --- status code --- 22:03:57 DEBUG --- status code --- 22:03:57 DEBUG --- status code --- 22:03:57 INFO [http_cmd]: http status code OK 22:03:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:03:57 DEBUG --- status code --- 22:03:57 INFO [http_cmd]: http status code OK 22:03:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:03:57 INFO [http_cmd]: http status code OK 22:03:57 DEBUG --- status code --- 22:03:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:03:57 INFO [http_cmd]: http status code OK 22:03:57 DEBUG --- status code --- 22:03:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:03:57 DEBUG --- stdout --- 22:03:57 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6234Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 9984m 33% 7672Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 21761m 72% 11491Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 81m 0% 3363Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 20204m 67% 15521Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 160m 0% 5188Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 22267m 74% 11791Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19698m 65% 3044Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 24577m 82% 11196Mi 10% gke-xlou-cdm-ds-0f601751-j488 10377m 34% 7290Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19748m 66% 17337Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 10130m 33% 7951Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20561m 68% 15171Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 12055m 75% 8817Mi 15% 22:03:57 DEBUG --- stderr --- 22:03:57 DEBUG --- status code --- 22:03:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:03:57 DEBUG --- status code --- 22:03:57 INFO [http_cmd]: http status code OK 22:03:57 DEBUG --- status code --- 22:03:57 DEBUG --- status code --- 22:03:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:03:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:03:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:03:57 DEBUG --- status code --- 22:03:57 DEBUG --- http response --- 22:03:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:03:57 DEBUG --- status code --- 22:03:57 DEBUG --- http response --- 22:03:57 DEBUG --- status code --- 22:03:57 DEBUG --- http response --- 22:03:57 DEBUG --- status code --- 22:03:57 DEBUG --- status code --- 22:03:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:03:57 DEBUG --- http response --- 22:03:57 DEBUG --- status code --- 22:03:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:03:57 DEBUG --- http response --- 22:03:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:03:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:03:57 DEBUG 22:03:57 DEBUG --- status code --- 22:03:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:03:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:03:57 DEBUG --- http response --- 22:03:57 DEBUG --- http response --- 22:03:57 DEBUG --- http response --- 22:03:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:03:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:03:57 DEBUG --- http response --- 22:03:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:03:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:03:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:03:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:03:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:03:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:03:57 DEBUG --- http response --- 22:03:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:03:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:03:57 DEBUG --- http response --- 22:03:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:03:57 DEBUG --- http response --- 22:03:57 DEBUG --- http response --- 22:03:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:03:57 DEBUG --- http response --- 22:03:57 DEBUG --- http response --- 22:03:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:03:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:03:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:03:57 DEBUG --- http response --- 22:03:57 DEBUG ----- output ----- 22:03:57 DEBUG --- http response --- 22:03:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:03:57 DEBUG ----- output ----- 22:03:57 DEBUG --- http response --- 22:03:57 DEBUG ----- output ----- 22:03:57 DEBUG --- http response --- 22:03:57 DEBUG --- http response --- 22:03:57 DEBUG ----- output ----- 22:03:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:03:57 DEBUG --- http response --- 22:03:57 DEBUG ----- output ----- 22:03:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:03:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:03:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:03:57 DEBUG --- http response --- 22:03:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:03:57 DEBUG ----- output ----- 22:03:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:03:57 DEBUG ----- output ----- 22:03:57 DEBUG ----- output ----- 22:03:57 DEBUG { 22:03:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:03:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:03:57 DEBUG ----- output ----- 22:03:57 DEBUG { 22:03:57 DEBUG { 22:03:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:03:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:03:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:03:57 DEBUG { 22:03:57 DEBUG ----- output ----- 22:03:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:03:57 DEBUG { 22:03:57 DEBUG ----- output ----- 22:03:57 DEBUG ----- output ----- 22:03:57 DEBUG ----- output ----- 22:03:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:03:57 DEBUG ----- output ----- 22:03:57 DEBUG { 22:03:57 DEBUG ----- output ----- 22:03:57 DEBUG { 22:03:57 DEBUG { 22:03:57 DEBUG "status": "success", 22:03:57 DEBUG ----- output ----- 22:03:57 DEBUG ----- output ----- 22:03:57 DEBUG { 22:03:57 DEBUG "status": "success", 22:03:57 DEBUG "status": "success", 22:03:57 DEBUG ----- output ----- 22:03:57 DEBUG ----- output ----- 22:03:57 DEBUG { 22:03:57 DEBUG "status": "success", 22:03:57 DEBUG { 22:03:57 DEBUG ----- output ----- 22:03:57 DEBUG "status": "success", 22:03:57 DEBUG "data": { 22:03:57 DEBUG { 22:03:57 DEBUG { 22:03:57 DEBUG ----- output ----- 22:03:57 DEBUG { 22:03:57 DEBUG "status": "success", 22:03:57 DEBUG { 22:03:57 DEBUG "status": "success", 22:03:57 DEBUG "status": "success", 22:03:57 DEBUG "data": { 22:03:57 DEBUG "resultType": "vector", 22:03:57 DEBUG { 22:03:57 DEBUG "status": "success", 22:03:57 DEBUG "data": { 22:03:57 DEBUG "data": { 22:03:57 DEBUG { 22:03:57 DEBUG "status": "success", 22:03:57 DEBUG ----- output ----- 22:03:57 DEBUG "data": { 22:03:57 DEBUG "status": "success", 22:03:57 DEBUG { 22:03:57 DEBUG { 22:03:57 DEBUG "resultType": "vector", 22:03:57 DEBUG "status": "success", 22:03:57 DEBUG "status": "success", 22:03:57 DEBUG { 22:03:57 DEBUG "status": "success", 22:03:57 DEBUG "data": { 22:03:57 DEBUG "resultType": "vector", 22:03:57 DEBUG "result": [ 22:03:57 DEBUG { 22:03:57 DEBUG "result": [ 22:03:57 DEBUG "status": "success", 22:03:57 DEBUG { 22:03:57 DEBUG "data": { 22:03:57 DEBUG "resultType": "vector", 22:03:57 DEBUG "resultType": "vector", 22:03:57 DEBUG "result": [ 22:03:57 DEBUG "data": { 22:03:57 DEBUG { 22:03:57 DEBUG "resultType": "vector", 22:03:57 DEBUG "data": { 22:03:57 DEBUG "status": "success", 22:03:57 DEBUG "status": "success", 22:03:57 DEBUG "result": [ 22:03:57 DEBUG "data": { 22:03:57 DEBUG "data": { 22:03:57 DEBUG "status": "success", 22:03:57 DEBUG "data": { 22:03:57 DEBUG "metric": { 22:03:57 DEBUG "data": { 22:03:57 DEBUG "status": "success", 22:03:57 DEBUG { 22:03:57 DEBUG "data": { 22:03:57 DEBUG "status": "success", 22:03:57 DEBUG "data": { 22:03:57 DEBUG "resultType": "vector", 22:03:57 DEBUG "result": [ 22:03:57 DEBUG { 22:03:57 DEBUG "status": "success", 22:03:57 DEBUG "resultType": "vector", 22:03:57 DEBUG "result": [ 22:03:57 DEBUG "result": [ 22:03:57 DEBUG "resultType": "vector", 22:03:57 DEBUG "data": { 22:03:57 DEBUG "data": { 22:03:57 DEBUG { 22:03:57 DEBUG "resultType": "vector", 22:03:57 DEBUG "resultType": "vector", 22:03:57 DEBUG "data": { 22:03:57 DEBUG "resultType": "vector", 22:03:57 DEBUG "pod": "am-767f6b67b5-9zw22" 22:03:57 DEBUG "resultType": "vector", 22:03:57 DEBUG "data": { 22:03:57 DEBUG "resultType": "vector", 22:03:57 DEBUG "resultType": "vector", 22:03:57 DEBUG "data": { 22:03:57 DEBUG "resultType": "vector", 22:03:57 DEBUG "result": [ 22:03:57 DEBUG { 22:03:57 DEBUG "metric": { 22:03:57 DEBUG "data": { 22:03:57 DEBUG { 22:03:57 DEBUG "status": "success", 22:03:57 DEBUG { 22:03:57 DEBUG "result": [ 22:03:57 DEBUG "resultType": "vector", 22:03:57 DEBUG "resultType": "vector", 22:03:57 DEBUG "metric": { 22:03:57 DEBUG "result": [ 22:03:57 DEBUG "result": [ 22:03:57 DEBUG "resultType": "vector", 22:03:57 DEBUG "result": [ 22:03:57 DEBUG }, 22:03:57 DEBUG "value": [ 22:03:57 DEBUG "result": [ 22:03:57 DEBUG { 22:03:57 DEBUG "result": [ 22:03:57 DEBUG "result": [ 22:03:57 DEBUG { 22:03:57 DEBUG "metric": { 22:03:57 DEBUG "pod": "am-767f6b67b5-9zw22" 22:03:57 DEBUG "resultType": "vector", 22:03:57 DEBUG "resultType": "vector", 22:03:57 DEBUG "metric": { 22:03:57 DEBUG "data": { 22:03:57 DEBUG "metric": { 22:03:57 DEBUG { 22:03:57 DEBUG "result": [ 22:03:57 DEBUG "result": [ 22:03:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:03:57 DEBUG { 22:03:57 DEBUG { 22:03:57 DEBUG "result": [ 22:03:57 DEBUG { 22:03:57 DEBUG 22:03:57 DEBUG "result": [ 22:03:57 DEBUG "metric": { 22:03:57 DEBUG "metric": { 22:03:57 DEBUG { 22:03:57 DEBUG { 22:03:57 DEBUG "metric": { 22:03:57 DEBUG "instance": "10.142.0.107:9100" 22:03:57 DEBUG }, 22:03:57 DEBUG "result": [ 22:03:57 DEBUG "result": [ 22:03:57 DEBUG "instance": "10.142.0.107:9100" 22:03:57 DEBUG "resultType": "vector", 22:03:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:03:57 DEBUG "metric": { 22:03:57 DEBUG { 22:03:57 DEBUG { 22:03:57 DEBUG }, 22:03:57 DEBUG "metric": { 22:03:57 DEBUG "metric": { 22:03:57 DEBUG { 22:03:57 DEBUG "metric": { 22:03:57 DEBUG "pod": "am-767f6b67b5-9zw22" 22:03:57 DEBUG { 22:03:57 DEBUG "pod": "ds-cts-0" 22:03:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:03:57 DEBUG "metric": { 22:03:57 DEBUG "metric": { 22:03:57 DEBUG "pod": "am-767f6b67b5-9zw22" 22:03:57 DEBUG }, 22:03:57 DEBUG "value": [ 22:03:57 DEBUG { 22:03:57 DEBUG { 22:03:57 DEBUG }, 22:03:57 DEBUG "result": [ 22:03:57 DEBUG }, 22:03:57 DEBUG "pod": "ds-idrepo-0" 22:03:57 DEBUG "metric": { 22:03:57 DEBUG "metric": { 22:03:57 DEBUG "value": [ 22:03:57 DEBUG "pod": "am-767f6b67b5-9zw22" 22:03:57 DEBUG "pod": "ds-idrepo-0" 22:03:57 DEBUG "metric": { 22:03:57 DEBUG }, 22:03:57 DEBUG 22:03:57 DEBUG "metric": { 22:03:57 DEBUG }, 22:03:57 DEBUG }, 22:03:57 DEBUG "pod": "ds-cts-0" 22:03:57 DEBUG "pod": "ds-cts-0" 22:03:57 DEBUG }, 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG "metric": { 22:03:57 DEBUG "value": [ 22:03:57 DEBUG "value": [ 22:03:57 DEBUG { 22:03:57 DEBUG "value": [ 22:03:57 DEBUG }, 22:03:57 DEBUG "pod": "am-767f6b67b5-9zw22" 22:03:57 DEBUG }, 22:03:57 DEBUG 22:03:57 DEBUG }, 22:03:57 DEBUG }, 22:03:57 DEBUG "instance": "10.142.0.107:9100" 22:03:57 DEBUG "value": [ 22:03:57 DEBUG 22:03:57 DEBUG "pod": "ds-idrepo-0" 22:03:57 DEBUG }, 22:03:57 DEBUG "value": [ 22:03:57 DEBUG }, 22:03:57 DEBUG }, 22:03:57 DEBUG "value": [ 22:03:57 DEBUG 22:03:57 DEBUG "metric": { 22:03:57 DEBUG "instance": "10.142.0.107:9100" 22:03:57 DEBUG }, 22:03:57 DEBUG 22:03:57 DEBUG "metric": { 22:03:57 DEBUG 22:03:57 DEBUG "value": [ 22:03:57 DEBUG "value": [ 22:03:57 DEBUG 22:03:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:03:57 DEBUG "value": [ 22:03:57 DEBUG "value": [ 22:03:57 DEBUG }, 22:03:57 DEBUG 22:03:57 DEBUG [...] 22:03:57 DEBUG "value": [ 22:03:57 DEBUG "value": [ 22:03:57 DEBUG 22:03:57 DEBUG "value": [ 22:03:57 DEBUG "value": [ 22:03:57 DEBUG 22:03:57 DEBUG [...] 22:03:57 DEBUG "instance": "10.142.0.107:9100" 22:03:57 DEBUG "value": [ 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG }, 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG "value": [ 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG }, 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG }, 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG [...] 22:03:57 DEBUG "value": [ 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG "value": [ 22:03:57 DEBUG 22:03:57 DEBUG [...] 22:03:57 DEBUG 22:03:57 DEBUG "value": [ 22:03:57 DEBUG [...] 22:03:57 DEBUG [...] 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG [...] 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG [...] 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG [...] 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG [...] 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG [...] 22:03:57 DEBUG [...] 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG "pod": "am-767f6b67b5-vbltz" 22:03:57 DEBUG [...] 22:03:57 DEBUG [...] 22:03:57 DEBUG 22:03:57 DEBUG [...] 22:03:57 DEBUG [...] 22:03:57 DEBUG 22:03:57 DEBUG "instance": "10.142.0.43:9100" 22:03:57 DEBUG 22:03:57 DEBUG [...] 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG [...] 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG [...] 22:03:57 DEBUG 22:03:57 DEBUG }, 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG }, 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG "pod": "overseer-0-967d957f7-zk66v" 22:03:57 DEBUG [...] 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG "value": [ 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG "pod": "overseer-0-967d957f7-zk66v" 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG "value": [ 22:03:57 DEBUG [...] 22:03:57 DEBUG 22:03:57 DEBUG "instance": "10.142.0.43:9100" 22:03:57 DEBUG 22:03:57 DEBUG [...] 22:03:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:03:57 DEBUG "pod": "ds-idrepo-2" 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG "pod": "am-767f6b67b5-vbltz" 22:03:57 DEBUG }, 22:03:57 DEBUG 22:03:57 DEBUG "pod": "ds-idrepo-2" 22:03:57 DEBUG "pod": "am-767f6b67b5-vbltz" 22:03:57 DEBUG 1713647037, 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG }, 22:03:57 DEBUG 22:03:57 DEBUG "pod": "am-767f6b67b5-vbltz" 22:03:57 DEBUG 1713647037, 22:03:57 DEBUG "4049717015552" 22:03:57 DEBUG 22:03:57 DEBUG }, 22:03:57 DEBUG "pod": "am-767f6b67b5-vbltz" 22:03:57 DEBUG 22:03:57 DEBUG }, 22:03:57 DEBUG }, 22:03:57 DEBUG "value": [ 22:03:57 DEBUG 1713647037, 22:03:57 DEBUG "3332.2666666666664" 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG }, 22:03:57 DEBUG }, 22:03:57 DEBUG "49.377" 22:03:57 DEBUG "pod": "ds-cts-2" 22:03:57 DEBUG "pod": "ds-idrepo-2" 22:03:57 DEBUG 22:03:57 DEBUG "value": [ 22:03:57 DEBUG "pod": "ds-cts-2" 22:03:57 DEBUG }, 22:03:57 DEBUG 22:03:57 DEBUG ] 22:03:57 DEBUG "instance": "10.142.0.43:9100" 22:03:57 DEBUG "value": [ 22:03:57 DEBUG }, 22:03:57 DEBUG 22:03:57 DEBUG "value": [ 22:03:57 DEBUG "value": [ 22:03:57 DEBUG }, 22:03:57 DEBUG ] 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG 22:03:57 DEBUG "value": [ 22:03:57 DEBUG "value": [ 22:03:57 DEBUG ] 22:03:57 DEBUG }, 22:03:57 DEBUG }, 22:03:57 DEBUG "pod": "ds-cts-2" 22:03:57 DEBUG 1713647037, 22:03:57 DEBUG }, 22:03:57 DEBUG "value": [ 22:03:57 DEBUG 22:03:57 DEBUG } 22:03:57 DEBUG }, 22:03:57 DEBUG 1713647037, 22:03:57 DEBUG "value": [ 22:03:57 DEBUG 22:03:57 DEBUG 1713647037, 22:03:57 DEBUG 1713647037, 22:03:57 DEBUG "value": [ 22:03:57 DEBUG } 22:03:57 DEBUG ] 22:03:57 DEBUG 22:03:57 DEBUG "instance": "10.142.0.43:9100" 22:03:57 DEBUG 1713647037, 22:03:57 DEBUG 1713647037, 22:03:57 DEBUG "0" 22:03:57 DEBUG } 22:03:57 DEBUG "value": [ 22:03:57 DEBUG }, 22:03:57 DEBUG "0" 22:03:57 DEBUG "value": [ 22:03:57 DEBUG 1713647037, 22:03:57 DEBUG 22:03:57 DEBUG ] 22:03:57 DEBUG "value": [ 22:03:57 DEBUG "100539815.8" 22:03:57 DEBUG 1713647037, 22:03:57 DEBUG "pod": "overseer-0-967d957f7-zk66v" 22:03:57 DEBUG "1050353664" 22:03:57 DEBUG "0" 22:03:57 DEBUG 1713647037, 22:03:57 DEBUG } 22:03:57 DEBUG "pod": "am-767f6b67b5-vbltz" 22:03:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:03:57 DEBUG }, 22:03:57 DEBUG "0" 22:03:57 DEBUG "value": [ 22:03:57 DEBUG ] 22:03:57 DEBUG ] 22:03:57 DEBUG 1713647037, 22:03:57 DEBUG "value": [ 22:03:57 DEBUG ] 22:03:57 DEBUG 1713647037, 22:03:57 DEBUG "3321.6666666666665" 22:03:57 DEBUG "instance": "10.142.0.43:9100" 22:03:57 DEBUG } 22:03:57 DEBUG 1713647037, 22:03:57 DEBUG ] 22:03:57 DEBUG "600020.5" 22:03:57 DEBUG }, 22:03:57 DEBUG ] 22:03:57 DEBUG ] 22:03:57 DEBUG "11.779217784903766" 22:03:57 DEBUG } 22:03:57 DEBUG }, 22:03:57 DEBUG }, 22:03:57 DEBUG "value": [ 22:03:57 DEBUG ] 22:03:57 DEBUG 1713647037, 22:03:57 DEBUG } 22:03:57 DEBUG } 22:03:57 DEBUG "0" 22:03:57 DEBUG 1713647037, 22:03:57 DEBUG } 22:03:57 DEBUG "0.4790333333333346" 22:03:57 DEBUG ] 22:03:57 DEBUG }, 22:03:57 DEBUG } 22:03:57 DEBUG "0.40819999999999934" 22:03:57 DEBUG } 22:03:57 DEBUG ] 22:03:57 DEBUG "value": [ 22:03:57 DEBUG } 22:03:57 DEBUG } 22:03:57 DEBUG ] 22:03:57 DEBUG "value": [ 22:03:57 DEBUG "value": [ 22:03:57 DEBUG 1713647037, 22:03:57 DEBUG } 22:03:57 DEBUG "0" 22:03:57 DEBUG ] 22:03:57 DEBUG } 22:03:57 DEBUG ] 22:03:57 DEBUG "30.266666666666666" 22:03:57 DEBUG ] 22:03:57 DEBUG ] 22:03:57 DEBUG } 22:03:57 DEBUG ] 22:03:57 DEBUG ] 22:03:57 DEBUG } 22:03:57 DEBUG } 22:03:57 DEBUG 1713647037, 22:03:57 DEBUG ] 22:03:57 DEBUG ] 22:03:57 DEBUG } 22:03:57 DEBUG 1713647037, 22:03:57 DEBUG 1713647037, 22:03:57 DEBUG "100542925.80000001" 22:03:57 DEBUG ] 22:03:57 DEBUG ] 22:03:57 DEBUG } 22:03:57 DEBUG } 22:03:57 DEBUG ] 22:03:57 DEBUG } 22:03:57 DEBUG } 22:03:57 DEBUG "value": [ 22:03:57 DEBUG } 22:03:57 DEBUG ] 22:03:57 DEBUG ] 22:03:57 DEBUG "0" 22:03:57 DEBUG } 22:03:57 DEBUG ] 22:03:57 DEBUG } 22:03:57 DEBUG ] 22:03:57 DEBUG "3327.233333333333" 22:03:57 DEBUG "11.818081182172424" 22:03:57 DEBUG ] 22:03:57 DEBUG } 22:03:57 DEBUG } 22:03:57 DEBUG } 22:03:57 DEBUG ] 22:03:57 DEBUG } 22:03:57 DEBUG } 22:03:57 DEBUG ] 22:03:57 DEBUG 1713647037, 22:03:57 DEBUG } 22:03:57 DEBUG } 22:03:57 DEBUG } 22:03:57 DEBUG ] 22:03:57 DEBUG } 22:03:57 DEBUG } 22:03:57 DEBUG } 22:03:57 DEBUG } 22:03:57 DEBUG ] 22:03:57 DEBUG ] 22:03:57 DEBUG } 22:03:57 DEBUG } 22:03:57 DEBUG ] 22:03:57 DEBUG } 22:03:57 DEBUG ] 22:03:57 DEBUG } 22:03:57 DEBUG "0.08033333333333798" 22:03:57 DEBUG } 22:03:57 DEBUG } 22:03:57 DEBUG } 22:03:57 DEBUG } 22:03:57 DEBUG } 22:03:57 DEBUG } 22:03:57 DEBUG } 22:03:57 DEBUG ] 22:03:57 DEBUG } 22:03:57 DEBUG } 22:03:57 DEBUG } 22:03:57 DEBUG } 22:03:57 DEBUG ] 22:03:57 DEBUG ] 22:03:57 DEBUG ] 22:03:57 DEBUG ] 22:03:57 DEBUG } 22:03:57 DEBUG } 22:03:57 DEBUG } 22:03:57 DEBUG } 22:03:57 DEBUG } 22:03:57 DEBUG } 22:03:57 DEBUG } 22:03:57 DEBUG } 22:03:57 DEBUG ] 22:03:57 DEBUG } 22:03:57 DEBUG } 22:03:57 DEBUG } 22:03:57 DEBUG } 22:03:57 DEBUG } 22:04:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647094" 22:04:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713647094" 22:04:54 ERROR [http_cmd]: ERROR 22:04:54 INFO [http_cmd]: http status code OK 22:04:54 ERROR --- status code --- 22:04:54 DEBUG --- status code --- 22:04:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:04:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:04:54 ERROR --- http response --- 22:04:54 DEBUG --- http response --- 22:04:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:04:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:04:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:04:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647094" 22:04:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713647094" 22:04:54 INFO [http_cmd]: http status code OK 22:04:54 DEBUG --- status code --- 22:04:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:04:54 DEBUG --- http response --- 22:04:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:04:54 DEBUG ----- output ----- 22:04:54 DEBUG { 22:04:54 DEBUG "status": "success", 22:04:54 DEBUG "data": { 22:04:54 DEBUG "resultType": "vector", 22:04:54 DEBUG "result": [ 22:04:54 DEBUG { 22:04:54 DEBUG "metric": { 22:04:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:04:54 DEBUG }, 22:04:54 DEBUG "value": [ 22:04:54 DEBUG 22:04:54 DEBUG 22:04:54 DEBUG 22:04:54 DEBUG [...] 22:04:54 DEBUG 22:04:54 DEBUG 22:04:54 INFO [http_cmd]: http status code OK 22:04:54 DEBUG 22:04:54 DEBUG --- status code --- 22:04:54 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 22:04:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:04:54 DEBUG }, 22:04:54 DEBUG --- http response --- 22:04:54 DEBUG "value": [ 22:04:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:04:54 DEBUG 1713647094, 22:04:54 DEBUG ----- output ----- 22:04:54 DEBUG "0" 22:04:54 DEBUG { 22:04:54 DEBUG ] 22:04:54 DEBUG "status": "success", 22:04:54 DEBUG } 22:04:54 DEBUG "data": { 22:04:54 DEBUG ] 22:04:54 DEBUG "resultType": "vector", 22:04:54 DEBUG } 22:04:54 DEBUG "result": [ 22:04:54 DEBUG } 22:04:54 DEBUG { 22:04:54 DEBUG "metric": { 22:04:54 DEBUG "pod": "ds-idrepo-0" 22:04:54 DEBUG }, 22:04:54 DEBUG "value": [ 22:04:54 DEBUG 22:04:54 DEBUG 22:04:54 DEBUG 22:04:54 DEBUG [...] 22:04:54 DEBUG 22:04:54 DEBUG 22:04:54 DEBUG 22:04:54 DEBUG "pod": "ds-cts-2" 22:04:54 DEBUG }, 22:04:54 DEBUG "value": [ 22:04:54 DEBUG 1713647094, 22:04:54 DEBUG "0" 22:04:54 DEBUG ] 22:04:54 DEBUG } 22:04:54 DEBUG ] 22:04:54 DEBUG } 22:04:54 DEBUG } 22:04:55 INFO 22:04:55 INFO [loop_until]: kubectl --namespace=xlou top pods 22:04:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:04:56 INFO [loop_until]: OK (rc = 0) 22:04:56 DEBUG --- stdout --- 22:04:56 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 21879m 8769Mi am-767f6b67b5-qcqz2 22351m 8891Mi am-767f6b67b5-vbltz 24649m 8942Mi ds-cts-0 20171m 12282Mi ds-cts-1 19818m 12277Mi ds-cts-2 20640m 12478Mi ds-idrepo-0 9533m 3912Mi ds-idrepo-1 9460m 3975Mi ds-idrepo-2 9501m 3878Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-7dd48676f6-tdxl9 14m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-967d957f7-zk66v 11887m 913Mi 22:04:56 DEBUG --- stderr --- 22:04:56 DEBUG 22:04:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647096" 22:04:56 INFO [http_cmd]: http status code OK 22:04:56 DEBUG --- status code --- 22:04:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:04:56 DEBUG --- http response --- 22:04:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:04:56 DEBUG ----- output ----- 22:04:56 DEBUG { 22:04:56 DEBUG "status": "success", 22:04:56 DEBUG "data": { 22:04:56 DEBUG "resultType": "vector", 22:04:56 DEBUG "result": [ 22:04:56 DEBUG { 22:04:56 DEBUG "metric": { 22:04:56 DEBUG "pod": "am-767f6b67b5-9zw22" 22:04:56 DEBUG }, 22:04:56 DEBUG "value": [ 22:04:56 DEBUG 22:04:56 DEBUG 22:04:56 DEBUG 22:04:56 DEBUG [...] 22:04:56 DEBUG 22:04:56 DEBUG 22:04:56 DEBUG 22:04:56 DEBUG "pod": "am-767f6b67b5-vbltz" 22:04:56 DEBUG }, 22:04:56 DEBUG "value": [ 22:04:56 DEBUG 1713647096, 22:04:56 DEBUG "3340.2" 22:04:56 DEBUG ] 22:04:56 DEBUG } 22:04:56 DEBUG ] 22:04:56 DEBUG } 22:04:56 DEBUG } 22:04:57 INFO 22:04:57 INFO [loop_until]: kubectl --namespace=xlou top node 22:04:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:04:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647097" 22:04:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647097" 22:04:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713647097" 22:04:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647097" 22:04:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647097" 22:04:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647097" 22:04:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713647097" 22:04:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647097" 22:04:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647097" 22:04:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647097" 22:04:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647097" 22:04:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647097" 22:04:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647097" 22:04:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647097" 22:04:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647097" 22:04:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647097" 22:04:57 INFO [http_cmd]: http status code OK 22:04:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647097" 22:04:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647097" 22:04:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647097" 22:04:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647097" 22:04:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647097" 22:04:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713647097" 22:04:57 INFO [http_cmd]: http status code OK 22:04:57 INFO [loop_until]: OK (rc = 0) 22:04:57 INFO [http_cmd]: http status code OK 22:04:57 INFO [http_cmd]: http status code OK 22:04:57 INFO [http_cmd]: http status code OK 22:04:57 INFO [http_cmd]: http status code OK 22:04:57 INFO [http_cmd]: http status code OK 22:04:57 INFO [http_cmd]: http status code OK 22:04:57 DEBUG --- status code --- 22:04:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:04:57 INFO [http_cmd]: http status code OK 22:04:57 INFO [http_cmd]: http status code OK 22:04:57 INFO [http_cmd]: http status code OK 22:04:57 INFO [http_cmd]: http status code OK 22:04:57 INFO [http_cmd]: http status code OK 22:04:57 INFO [http_cmd]: http status code OK 22:04:57 INFO [http_cmd]: http status code OK 22:04:57 DEBUG --- status code --- 22:04:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:04:57 INFO [http_cmd]: http status code OK 22:04:57 INFO [http_cmd]: http status code OK 22:04:57 DEBUG --- stdout --- 22:04:57 DEBUG --- status code --- 22:04:57 DEBUG --- status code --- 22:04:57 DEBUG --- status code --- 22:04:57 DEBUG --- status code --- 22:04:57 DEBUG --- status code --- 22:04:57 DEBUG --- status code --- 22:04:57 INFO [http_cmd]: http status code OK 22:04:57 DEBUG --- http response --- 22:04:57 DEBUG --- status code --- 22:04:57 DEBUG --- status code --- 22:04:57 INFO [http_cmd]: http status code OK 22:04:57 DEBUG --- status code --- 22:04:57 DEBUG --- status code --- 22:04:57 DEBUG --- status code --- 22:04:57 DEBUG --- status code --- 22:04:57 DEBUG --- status code --- 22:04:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:04:57 DEBUG --- http response --- 22:04:57 DEBUG --- status code --- 22:04:57 DEBUG --- status code --- 22:04:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:04:57 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 67m 0% 6217Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 9485m 31% 7667Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 21923m 73% 11518Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 80m 0% 3366Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 20083m 67% 15529Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 154m 0% 5171Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 22051m 73% 11832Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19801m 66% 3053Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 24961m 83% 11191Mi 10% gke-xlou-cdm-ds-0f601751-j488 10308m 34% 7284Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19742m 66% 17343Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 10137m 33% 7941Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20555m 68% 15184Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 12106m 76% 8817Mi 15% 22:04:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:04:57 INFO [http_cmd]: http status code OK 22:04:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:04:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:04:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:04:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:04:57 DEBUG --- status code --- 22:04:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:04:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:04:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:04:57 DEBUG --- status code --- 22:04:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:04:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:04:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:04:57 DEBUG --- http response --- 22:04:57 INFO [http_cmd]: http status code OK 22:04:57 DEBUG --- http response --- 22:04:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:04:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:04:57 DEBUG ----- output ----- 22:04:57 DEBUG --- http response --- 22:04:57 DEBUG --- stderr --- 22:04:57 DEBUG --- http response --- 22:04:57 DEBUG --- status code --- 22:04:57 DEBUG --- http response --- 22:04:57 DEBUG --- http response --- 22:04:57 DEBUG --- http response --- 22:04:57 DEBUG --- http response --- 22:04:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:04:57 DEBUG ----- output ----- 22:04:57 INFO [http_cmd]: http status code OK 22:04:57 DEBUG --- http response --- 22:04:57 DEBUG --- http response --- 22:04:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:04:57 DEBUG --- http response --- 22:04:57 DEBUG --- http response --- 22:04:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:04:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:04:57 DEBUG --- status code --- 22:04:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:04:57 DEBUG --- http response --- 22:04:57 DEBUG --- http response --- 22:04:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:04:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:04:57 DEBUG 22:04:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:04:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:04:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:04:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:04:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:04:57 DEBUG --- http response --- 22:04:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:04:57 DEBUG ----- output ----- 22:04:57 DEBUG --- status code --- 22:04:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:04:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:04:57 DEBUG --- http response --- 22:04:57 DEBUG --- http response --- 22:04:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:04:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:04:57 DEBUG ----- output ----- 22:04:57 DEBUG { 22:04:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:04:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:04:57 DEBUG --- http response --- 22:04:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:04:57 DEBUG ----- output ----- 22:04:57 DEBUG ----- output ----- 22:04:57 DEBUG --- http response --- 22:04:57 DEBUG ----- output ----- 22:04:57 DEBUG ----- output ----- 22:04:57 DEBUG ----- output ----- 22:04:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:04:57 DEBUG { 22:04:57 DEBUG { 22:04:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:04:57 DEBUG ----- output ----- 22:04:57 DEBUG { 22:04:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:04:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:04:57 DEBUG ----- output ----- 22:04:57 DEBUG ----- output ----- 22:04:57 DEBUG { 22:04:57 DEBUG "status": "success", 22:04:57 DEBUG ----- output ----- 22:04:57 DEBUG ----- output ----- 22:04:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:04:57 DEBUG ----- output ----- 22:04:57 DEBUG { 22:04:57 DEBUG { 22:04:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:04:57 DEBUG { 22:04:57 DEBUG { 22:04:57 DEBUG { 22:04:57 DEBUG ----- output ----- 22:04:57 DEBUG "status": "success", 22:04:57 DEBUG "status": "success", 22:04:57 DEBUG "data": { 22:04:57 DEBUG --- http response --- 22:04:57 DEBUG ----- output ----- 22:04:57 DEBUG { 22:04:57 DEBUG ----- output ----- 22:04:57 DEBUG { 22:04:57 DEBUG { 22:04:57 DEBUG "status": "success", 22:04:57 DEBUG "data": { 22:04:57 DEBUG { 22:04:57 DEBUG { 22:04:57 DEBUG ----- output ----- 22:04:57 DEBUG { 22:04:57 DEBUG "status": "success", 22:04:57 DEBUG "status": "success", 22:04:57 DEBUG ----- output ----- 22:04:57 DEBUG "status": "success", 22:04:57 DEBUG "status": "success", 22:04:57 DEBUG "status": "success", 22:04:57 DEBUG "data": { 22:04:57 DEBUG "data": { 22:04:57 DEBUG "resultType": "vector", 22:04:57 DEBUG "status": "success", 22:04:57 DEBUG ----- output ----- 22:04:57 DEBUG "status": "success", 22:04:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:04:57 DEBUG { 22:04:57 DEBUG "status": "success", 22:04:57 DEBUG "status": "success", 22:04:57 DEBUG "data": { 22:04:57 DEBUG "resultType": "vector", 22:04:57 DEBUG "status": "success", 22:04:57 DEBUG "status": "success", 22:04:57 DEBUG { 22:04:57 DEBUG "status": "success", 22:04:57 DEBUG "data": { 22:04:57 DEBUG "data": { 22:04:57 DEBUG { 22:04:57 DEBUG "data": { 22:04:57 DEBUG "data": { 22:04:57 DEBUG { 22:04:57 DEBUG "resultType": "vector", 22:04:57 DEBUG "resultType": "vector", 22:04:57 DEBUG "result": [ 22:04:57 DEBUG "data": { 22:04:57 DEBUG { 22:04:57 DEBUG "data": { 22:04:57 DEBUG ----- output ----- 22:04:57 DEBUG "status": "success", 22:04:57 DEBUG "data": { 22:04:57 DEBUG "data": { 22:04:57 DEBUG "resultType": "vector", 22:04:57 DEBUG "result": [ 22:04:57 DEBUG "data": { 22:04:57 DEBUG "data": { 22:04:57 DEBUG "status": "success", 22:04:57 DEBUG "data": { 22:04:57 DEBUG "resultType": "vector", 22:04:57 DEBUG "resultType": "vector", 22:04:57 DEBUG "status": "success", 22:04:57 DEBUG "resultType": "vector", 22:04:57 DEBUG "resultType": "vector", 22:04:57 DEBUG "status": "success", 22:04:57 DEBUG "result": [ 22:04:57 DEBUG "result": [ 22:04:57 DEBUG { 22:04:57 DEBUG "resultType": "vector", 22:04:57 DEBUG "status": "success", 22:04:57 DEBUG "resultType": "vector", 22:04:57 DEBUG { 22:04:57 DEBUG "data": { 22:04:57 DEBUG "resultType": "vector", 22:04:57 DEBUG "resultType": "vector", 22:04:57 DEBUG "result": [ 22:04:57 DEBUG { 22:04:57 DEBUG "resultType": "vector", 22:04:57 DEBUG "resultType": "vector", 22:04:57 DEBUG "data": { 22:04:57 DEBUG "resultType": "vector", 22:04:57 DEBUG "result": [ 22:04:57 DEBUG "result": [ 22:04:57 DEBUG "data": { 22:04:57 DEBUG "result": [ 22:04:57 DEBUG "result": [ 22:04:57 DEBUG "data": { 22:04:57 DEBUG { 22:04:57 DEBUG { 22:04:57 DEBUG "metric": { 22:04:57 DEBUG "result": [ 22:04:57 DEBUG "data": { 22:04:57 DEBUG "result": [ 22:04:57 DEBUG "status": "success", 22:04:57 DEBUG "resultType": "vector", 22:04:57 DEBUG "result": [ 22:04:57 DEBUG "result": [ 22:04:57 DEBUG { 22:04:57 DEBUG "metric": { 22:04:57 DEBUG "result": [ 22:04:57 DEBUG "result": [ 22:04:57 DEBUG "resultType": "vector", 22:04:57 DEBUG "result": [ 22:04:57 DEBUG { 22:04:57 DEBUG { 22:04:57 DEBUG "resultType": "vector", 22:04:57 DEBUG { 22:04:57 DEBUG { 22:04:57 DEBUG "resultType": "vector", 22:04:57 DEBUG "metric": { 22:04:57 DEBUG "metric": { 22:04:57 DEBUG "pod": "am-767f6b67b5-9zw22" 22:04:57 DEBUG }, 22:04:57 DEBUG "resultType": "vector", 22:04:57 DEBUG { 22:04:57 DEBUG "data": { 22:04:57 DEBUG "result": [ 22:04:57 DEBUG { 22:04:57 DEBUG { 22:04:57 DEBUG "metric": { 22:04:57 DEBUG "instance": "10.142.0.107:9100" 22:04:57 DEBUG { 22:04:57 DEBUG { 22:04:57 DEBUG "result": [ 22:04:57 DEBUG { 22:04:57 DEBUG "metric": { 22:04:57 DEBUG "metric": { 22:04:57 DEBUG "result": [ 22:04:57 DEBUG "metric": { 22:04:57 DEBUG "metric": { 22:04:57 DEBUG "result": [ 22:04:57 DEBUG "pod": "ds-idrepo-0" 22:04:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:04:57 DEBUG { 22:04:57 DEBUG "value": [ 22:04:57 DEBUG "result": [ 22:04:57 DEBUG "metric": { 22:04:57 DEBUG "resultType": "vector", 22:04:57 DEBUG { 22:04:57 DEBUG "metric": { 22:04:57 DEBUG "metric": { 22:04:57 DEBUG "pod": "ds-idrepo-0" 22:04:57 DEBUG }, 22:04:57 DEBUG "metric": { 22:04:57 DEBUG "metric": { 22:04:57 DEBUG { 22:04:57 DEBUG "metric": { 22:04:57 DEBUG "pod": "am-767f6b67b5-9zw22" 22:04:57 DEBUG "pod": "am-767f6b67b5-9zw22" 22:04:57 DEBUG { 22:04:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:04:57 DEBUG "pod": "am-767f6b67b5-9zw22" 22:04:57 DEBUG { 22:04:57 DEBUG }, 22:04:57 DEBUG }, 22:04:57 DEBUG "metric": { 22:04:57 DEBUG 22:04:57 DEBUG { 22:04:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:04:57 DEBUG "result": [ 22:04:57 DEBUG "metric": { 22:04:57 DEBUG "instance": "10.142.0.107:9100" 22:04:57 DEBUG "pod": "ds-cts-0" 22:04:57 DEBUG }, 22:04:57 DEBUG "value": [ 22:04:57 DEBUG "instance": "10.142.0.107:9100" 22:04:57 DEBUG "pod": "ds-cts-0" 22:04:57 DEBUG "metric": { 22:04:57 DEBUG "pod": "am-767f6b67b5-9zw22" 22:04:57 DEBUG }, 22:04:57 DEBUG }, 22:04:57 DEBUG "metric": { 22:04:57 DEBUG }, 22:04:57 DEBUG }, 22:04:57 DEBUG "metric": { 22:04:57 DEBUG "value": [ 22:04:57 DEBUG "value": [ 22:04:57 DEBUG "pod": "ds-idrepo-0" 22:04:57 DEBUG 22:04:57 DEBUG "metric": { 22:04:57 DEBUG }, 22:04:57 DEBUG { 22:04:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:04:57 DEBUG }, 22:04:57 DEBUG }, 22:04:57 DEBUG "value": [ 22:04:57 DEBUG 22:04:57 DEBUG }, 22:04:57 DEBUG }, 22:04:57 DEBUG "instance": "10.142.0.107:9100" 22:04:57 DEBUG }, 22:04:57 DEBUG "value": [ 22:04:57 DEBUG "value": [ 22:04:57 DEBUG "pod": "ds-cts-0" 22:04:57 DEBUG "value": [ 22:04:57 DEBUG "value": [ 22:04:57 DEBUG "pod": "am-767f6b67b5-9zw22" 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG }, 22:04:57 DEBUG 22:04:57 DEBUG "instance": "10.142.0.107:9100" 22:04:57 DEBUG "value": [ 22:04:57 DEBUG "metric": { 22:04:57 DEBUG }, 22:04:57 DEBUG "value": [ 22:04:57 DEBUG "value": [ 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG "value": [ 22:04:57 DEBUG "value": [ 22:04:57 DEBUG }, 22:04:57 DEBUG "value": [ 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG }, 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG }, 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG "value": [ 22:04:57 DEBUG [...] 22:04:57 DEBUG }, 22:04:57 DEBUG 22:04:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:04:57 DEBUG "value": [ 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG "value": [ 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG "value": [ 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG "value": [ 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG "value": [ 22:04:57 DEBUG 22:04:57 DEBUG }, 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG [...] 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG [...] 22:04:57 DEBUG [...] 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG "value": [ 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG [...] 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG [...] 22:04:57 DEBUG [...] 22:04:57 DEBUG 22:04:57 DEBUG [...] 22:04:57 DEBUG [...] 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG [...] 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG [...] 22:04:57 DEBUG [...] 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG [...] 22:04:57 DEBUG [...] 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG [...] 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG [...] 22:04:57 DEBUG "pod": "am-767f6b67b5-vbltz" 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG [...] 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG [...] 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG [...] 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG }, 22:04:57 DEBUG [...] 22:04:57 DEBUG 22:04:57 DEBUG [...] 22:04:57 DEBUG [...] 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG "pod": "ds-idrepo-2" 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG "pod": "ds-idrepo-2" 22:04:57 DEBUG }, 22:04:57 DEBUG 22:04:57 DEBUG "value": [ 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG "instance": "10.142.0.43:9100" 22:04:57 DEBUG }, 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG "pod": "am-767f6b67b5-vbltz" 22:04:57 DEBUG "pod": "am-767f6b67b5-vbltz" 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG "pod": "ds-idrepo-2" 22:04:57 DEBUG "pod": "am-767f6b67b5-vbltz" 22:04:57 DEBUG 22:04:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:04:57 DEBUG "value": [ 22:04:57 DEBUG 22:04:57 DEBUG 1713647097, 22:04:57 DEBUG 22:04:57 DEBUG "pod": "ds-idrepo-2" 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG "instance": "10.142.0.43:9100" 22:04:57 DEBUG "pod": "ds-cts-2" 22:04:57 DEBUG }, 22:04:57 DEBUG "value": [ 22:04:57 DEBUG "instance": "10.142.0.43:9100" 22:04:57 DEBUG "pod": "ds-cts-2" 22:04:57 DEBUG 22:04:57 DEBUG }, 22:04:57 DEBUG }, 22:04:57 DEBUG 22:04:57 DEBUG "pod": "am-767f6b67b5-vbltz" 22:04:57 DEBUG }, 22:04:57 DEBUG }, 22:04:57 DEBUG 22:04:57 DEBUG }, 22:04:57 DEBUG 1713647097, 22:04:57 DEBUG "pod": "ds-idrepo-2" 22:04:57 DEBUG "3341.733333333333" 22:04:57 DEBUG 22:04:57 DEBUG }, 22:04:57 DEBUG 22:04:57 DEBUG 22:04:57 DEBUG }, 22:04:57 DEBUG "value": [ 22:04:57 DEBUG "value": [ 22:04:57 DEBUG 1713647097, 22:04:57 DEBUG }, 22:04:57 DEBUG }, 22:04:57 DEBUG "instance": "10.142.0.43:9100" 22:04:57 DEBUG "value": [ 22:04:57 DEBUG "value": [ 22:04:57 DEBUG "pod": "ds-cts-2" 22:04:57 DEBUG }, 22:04:57 DEBUG "value": [ 22:04:57 DEBUG "value": [ 22:04:57 DEBUG "pod": "am-767f6b67b5-vbltz" 22:04:57 DEBUG "value": [ 22:04:57 DEBUG "0" 22:04:57 DEBUG }, 22:04:57 DEBUG ] 22:04:57 DEBUG "instance": "10.142.0.43:9100" 22:04:57 DEBUG "value": [ 22:04:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:04:57 DEBUG "pod": "ds-idrepo-2" 22:04:57 DEBUG }, 22:04:57 DEBUG 1713647097, 22:04:57 DEBUG 1713647097, 22:04:57 DEBUG "0" 22:04:57 DEBUG "value": [ 22:04:57 DEBUG "value": [ 22:04:57 DEBUG }, 22:04:57 DEBUG 1713647097, 22:04:57 DEBUG 1713647097, 22:04:57 DEBUG }, 22:04:57 DEBUG "value": [ 22:04:57 DEBUG 1713647097, 22:04:57 DEBUG 1713647097, 22:04:57 DEBUG }, 22:04:57 DEBUG 1713647097, 22:04:57 DEBUG ] 22:04:57 DEBUG "value": [ 22:04:57 DEBUG } 22:04:57 DEBUG }, 22:04:57 DEBUG 1713647097, 22:04:57 DEBUG }, 22:04:57 DEBUG }, 22:04:57 DEBUG "value": [ 22:04:57 DEBUG 1713647097, 22:04:57 DEBUG "4054919132160" 22:04:57 DEBUG ] 22:04:57 DEBUG 1713647097, 22:04:57 DEBUG 1713647097, 22:04:57 DEBUG "value": [ 22:04:57 DEBUG "3336.9666666666667" 22:04:57 DEBUG "50.7995" 22:04:57 DEBUG "value": [ 22:04:57 DEBUG 1713647097, 22:04:57 DEBUG "4403.604743083004" 22:04:57 DEBUG "4014.833333333333" 22:04:57 DEBUG "value": [ 22:04:57 DEBUG "1052725248" 22:04:57 DEBUG } 22:04:57 DEBUG ] 22:04:57 DEBUG 1713647097, 22:04:57 DEBUG "value": [ 22:04:57 DEBUG "0" 22:04:57 DEBUG "value": [ 22:04:57 DEBUG "value": [ 22:04:57 DEBUG "0.5352666666666664" 22:04:57 DEBUG ] 22:04:57 DEBUG } 22:04:57 DEBUG "0.07466666666665368" 22:04:57 DEBUG ] 22:04:57 DEBUG "0" 22:04:57 DEBUG 1713647097, 22:04:57 DEBUG ] 22:04:57 DEBUG ] 22:04:57 DEBUG 1713647097, 22:04:57 DEBUG "0" 22:04:57 DEBUG ] 22:04:57 DEBUG ] 22:04:57 DEBUG 1713647097, 22:04:57 DEBUG ] 22:04:57 DEBUG ] 22:04:57 DEBUG } 22:04:57 DEBUG "0" 22:04:57 DEBUG 1713647097, 22:04:57 DEBUG ] 22:04:57 DEBUG 1713647097, 22:04:57 DEBUG 1713647097, 22:04:57 DEBUG ] 22:04:57 DEBUG } 22:04:57 DEBUG ] 22:04:57 DEBUG } 22:04:57 DEBUG "0.3840000000000411" 22:04:57 DEBUG ] 22:04:57 DEBUG "100440705.13333333" 22:04:57 DEBUG } 22:04:57 DEBUG } 22:04:57 DEBUG "28.833333333333332" 22:04:57 DEBUG ] 22:04:57 DEBUG } 22:04:57 DEBUG } 22:04:57 DEBUG "599702.5" 22:04:57 DEBUG } 22:04:57 DEBUG } 22:04:57 DEBUG } 22:04:57 DEBUG ] 22:04:57 DEBUG "100436475.4" 22:04:57 DEBUG } 22:04:57 DEBUG "11.910743502982902" 22:04:57 DEBUG "9.628231778656216" 22:04:57 DEBUG } 22:04:57 DEBUG ] 22:04:57 DEBUG } 22:04:57 DEBUG ] 22:04:57 DEBUG ] 22:04:57 DEBUG } 22:04:57 DEBUG ] 22:04:57 DEBUG ] 22:04:57 DEBUG ] 22:04:57 DEBUG ] 22:04:57 DEBUG } 22:04:57 DEBUG ] 22:04:57 DEBUG ] 22:04:57 DEBUG ] 22:04:57 DEBUG ] 22:04:57 DEBUG } 22:04:57 DEBUG } 22:04:57 DEBUG ] 22:04:57 DEBUG } 22:04:57 DEBUG ] 22:04:57 DEBUG ] 22:04:57 DEBUG ] 22:04:57 DEBUG } 22:04:57 DEBUG } 22:04:57 DEBUG } 22:04:57 DEBUG } 22:04:57 DEBUG ] 22:04:57 DEBUG } 22:04:57 DEBUG } 22:04:57 DEBUG } 22:04:57 DEBUG } 22:04:57 DEBUG ] 22:04:57 DEBUG } 22:04:57 DEBUG } 22:04:57 DEBUG } 22:04:57 DEBUG } 22:04:57 DEBUG ] 22:04:57 DEBUG ] 22:04:57 DEBUG ] 22:04:57 DEBUG } 22:04:57 DEBUG } 22:04:57 DEBUG } 22:04:57 DEBUG } 22:04:57 DEBUG } 22:04:57 DEBUG } 22:04:57 DEBUG ] 22:04:57 DEBUG } 22:04:57 DEBUG } 22:04:57 DEBUG } 22:04:57 DEBUG ] 22:04:57 DEBUG } 22:04:57 DEBUG ] 22:04:57 DEBUG } 22:04:57 DEBUG } 22:04:57 DEBUG } 22:04:57 DEBUG } 22:04:57 DEBUG ] 22:04:57 DEBUG ] 22:04:57 DEBUG } 22:04:57 DEBUG ] 22:04:57 DEBUG } 22:04:57 DEBUG } 22:04:57 DEBUG } 22:04:57 DEBUG } 22:04:57 DEBUG } 22:04:57 DEBUG } 22:04:57 DEBUG } 22:04:57 DEBUG } 22:04:57 DEBUG } 22:04:57 DEBUG } 22:04:57 DEBUG } 22:04:57 DEBUG } 22:04:57 DEBUG } 22:04:57 DEBUG } 22:04:57 DEBUG } 22:04:57 DEBUG } 22:04:57 DEBUG } 22:04:57 DEBUG } 22:05:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713647154" 22:05:54 ERROR [http_cmd]: ERROR 22:05:54 ERROR --- status code --- 22:05:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:05:54 ERROR --- http response --- 22:05:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:05:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:05:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647154" 22:05:54 INFO [http_cmd]: http status code OK 22:05:54 DEBUG --- status code --- 22:05:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:05:54 DEBUG --- http response --- 22:05:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:05:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713647154" 22:05:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647154" 22:05:54 INFO [http_cmd]: http status code OK 22:05:54 INFO [http_cmd]: http status code OK 22:05:54 DEBUG --- status code --- 22:05:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:05:54 DEBUG --- http response --- 22:05:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:05:54 DEBUG ----- output ----- 22:05:54 DEBUG { 22:05:54 DEBUG --- status code --- 22:05:54 DEBUG "status": "success", 22:05:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:05:54 DEBUG "data": { 22:05:54 DEBUG --- http response --- 22:05:54 DEBUG "resultType": "vector", 22:05:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:05:54 DEBUG "result": [ 22:05:54 DEBUG ----- output ----- 22:05:54 DEBUG { 22:05:54 DEBUG { 22:05:54 DEBUG "metric": { 22:05:54 DEBUG "status": "success", 22:05:54 DEBUG "pod": "ds-idrepo-1" 22:05:54 DEBUG "data": { 22:05:54 DEBUG }, 22:05:54 DEBUG "resultType": "vector", 22:05:54 DEBUG "value": [ 22:05:54 DEBUG "result": [ 22:05:54 DEBUG 22:05:54 DEBUG { 22:05:54 DEBUG 22:05:54 DEBUG "metric": { 22:05:54 DEBUG 22:05:54 DEBUG "pod": "ds-idrepo-0" 22:05:54 DEBUG [...] 22:05:54 DEBUG }, 22:05:54 DEBUG 22:05:54 DEBUG "value": [ 22:05:54 DEBUG 22:05:54 DEBUG 22:05:54 DEBUG 22:05:54 DEBUG 22:05:54 DEBUG "pod": "overseer-0-967d957f7-zk66v" 22:05:54 DEBUG 22:05:54 DEBUG }, 22:05:54 DEBUG [...] 22:05:54 DEBUG "value": [ 22:05:54 DEBUG 22:05:54 DEBUG 1713647154, 22:05:54 DEBUG 22:05:54 DEBUG "28473197.80312812" 22:05:54 DEBUG 22:05:54 DEBUG ] 22:05:54 DEBUG "pod": "ds-cts-2" 22:05:54 DEBUG } 22:05:54 DEBUG }, 22:05:54 DEBUG ] 22:05:54 DEBUG "value": [ 22:05:54 DEBUG } 22:05:54 DEBUG 1713647154, 22:05:54 DEBUG } 22:05:54 DEBUG "0" 22:05:54 DEBUG ] 22:05:54 DEBUG } 22:05:54 DEBUG ] 22:05:54 DEBUG } 22:05:54 DEBUG } 22:05:56 INFO 22:05:56 INFO [loop_until]: kubectl --namespace=xlou top pods 22:05:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:05:56 INFO [loop_until]: OK (rc = 0) 22:05:56 DEBUG --- stdout --- 22:05:56 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 22290m 8793Mi am-767f6b67b5-qcqz2 21699m 8926Mi am-767f6b67b5-vbltz 24027m 8944Mi ds-cts-0 20101m 12309Mi ds-cts-1 19520m 12277Mi ds-cts-2 20456m 12479Mi ds-idrepo-0 9490m 3912Mi ds-idrepo-1 9548m 3966Mi ds-idrepo-2 9623m 3878Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 4m 1238Mi lodemon-7dd48676f6-tdxl9 14m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-967d957f7-zk66v 12074m 916Mi 22:05:56 DEBUG --- stderr --- 22:05:56 DEBUG 22:05:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647156" 22:05:56 INFO [http_cmd]: http status code OK 22:05:56 DEBUG --- status code --- 22:05:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:05:56 DEBUG --- http response --- 22:05:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:05:56 DEBUG ----- output ----- 22:05:56 DEBUG { 22:05:56 DEBUG "status": "success", 22:05:56 DEBUG "data": { 22:05:56 DEBUG "resultType": "vector", 22:05:56 DEBUG "result": [ 22:05:56 DEBUG { 22:05:56 DEBUG "metric": { 22:05:56 DEBUG "pod": "am-767f6b67b5-9zw22" 22:05:56 DEBUG }, 22:05:56 DEBUG "value": [ 22:05:56 DEBUG 22:05:56 DEBUG 22:05:56 DEBUG 22:05:56 DEBUG [...] 22:05:56 DEBUG 22:05:56 DEBUG 22:05:56 DEBUG 22:05:56 DEBUG "pod": "am-767f6b67b5-vbltz" 22:05:56 DEBUG }, 22:05:56 DEBUG "value": [ 22:05:56 DEBUG 1713647156, 22:05:56 DEBUG "3333.5666666666666" 22:05:56 DEBUG ] 22:05:56 DEBUG } 22:05:56 DEBUG ] 22:05:56 DEBUG } 22:05:56 DEBUG } 22:05:57 INFO 22:05:57 INFO [loop_until]: kubectl --namespace=xlou top node 22:05:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:05:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647157" 22:05:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647157" 22:05:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647157" 22:05:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647157" 22:05:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647157" 22:05:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647157" 22:05:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647157" 22:05:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713647157" 22:05:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647157" 22:05:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647157" 22:05:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647157" 22:05:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647157" 22:05:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647157" 22:05:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647157" 22:05:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713647157" 22:05:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647157" 22:05:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647157" 22:05:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713647157" 22:05:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647157" 22:05:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647157" 22:05:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647157" 22:05:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647157" 22:05:57 INFO [http_cmd]: http status code OK 22:05:57 INFO [http_cmd]: http status code OK 22:05:57 INFO [http_cmd]: http status code OK 22:05:57 INFO [http_cmd]: http status code OK 22:05:57 INFO [http_cmd]: http status code OK 22:05:57 INFO [http_cmd]: http status code OK 22:05:57 INFO [http_cmd]: http status code OK 22:05:57 INFO [http_cmd]: http status code OK 22:05:57 INFO [http_cmd]: http status code OK 22:05:57 INFO [http_cmd]: http status code OK 22:05:57 INFO [http_cmd]: http status code OK 22:05:57 INFO [loop_until]: OK (rc = 0) 22:05:57 INFO [http_cmd]: http status code OK 22:05:57 INFO [http_cmd]: http status code OK 22:05:57 INFO [http_cmd]: http status code OK 22:05:57 INFO [http_cmd]: http status code OK 22:05:57 INFO [http_cmd]: http status code OK 22:05:57 INFO [http_cmd]: http status code OK 22:05:57 INFO [http_cmd]: http status code OK 22:05:57 DEBUG --- status code --- 22:05:57 INFO [http_cmd]: http status code OK 22:05:57 DEBUG --- status code --- 22:05:57 DEBUG --- status code --- 22:05:57 DEBUG --- status code --- 22:05:57 DEBUG --- status code --- 22:05:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:05:57 DEBUG --- status code --- 22:05:57 DEBUG --- status code --- 22:05:57 DEBUG --- status code --- 22:05:57 DEBUG --- status code --- 22:05:57 DEBUG --- status code --- 22:05:57 INFO [http_cmd]: http status code OK 22:05:57 INFO [http_cmd]: http status code OK 22:05:57 DEBUG --- stdout --- 22:05:57 DEBUG --- status code --- 22:05:57 DEBUG --- status code --- 22:05:57 DEBUG --- status code --- 22:05:57 DEBUG --- status code --- 22:05:57 DEBUG --- status code --- 22:05:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:05:57 DEBUG --- status code --- 22:05:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:05:57 DEBUG --- status code --- 22:05:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:05:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:05:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:05:57 DEBUG --- status code --- 22:05:57 DEBUG --- http response --- 22:05:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:05:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:05:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:05:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:05:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:05:57 DEBUG --- http response --- 22:05:57 DEBUG --- status code --- 22:05:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:05:57 INFO [http_cmd]: http status code OK 22:05:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:05:57 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6207Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 9906m 33% 7673Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 22050m 73% 11522Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 82m 0% 3364Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 19820m 66% 15541Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 162m 0% 5162Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 22801m 76% 11916Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19809m 66% 3086Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 25103m 84% 11184Mi 10% gke-xlou-cdm-ds-0f601751-j488 10333m 34% 7282Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 18853m 63% 17365Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 10100m 33% 7937Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20580m 68% 15189Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 12133m 76% 9014Mi 15% 22:05:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:05:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:05:57 DEBUG --- status code --- 22:05:57 DEBUG --- http response --- 22:05:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:05:57 DEBUG --- http response --- 22:05:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:05:57 DEBUG --- http response --- 22:05:57 DEBUG --- http response --- 22:05:57 DEBUG --- http response --- 22:05:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:05:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:05:57 DEBUG --- http response --- 22:05:57 DEBUG --- http response --- 22:05:57 DEBUG --- http response --- 22:05:57 DEBUG --- http response --- 22:05:57 DEBUG --- status code --- 22:05:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:05:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:05:57 DEBUG ----- output ----- 22:05:57 DEBUG --- status code --- 22:05:57 DEBUG --- http response --- 22:05:57 DEBUG --- stderr --- 22:05:57 DEBUG --- http response --- 22:05:57 DEBUG --- http response --- 22:05:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:05:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:05:57 DEBUG --- http response --- 22:05:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:05:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:05:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:05:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:05:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:05:57 DEBUG --- http response --- 22:05:57 DEBUG ----- output ----- 22:05:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:05:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:05:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:05:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:05:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:05:57 DEBUG --- http response --- 22:05:57 DEBUG { 22:05:57 DEBUG --- http response --- 22:05:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:05:57 DEBUG 22:05:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:05:57 DEBUG ----- output ----- 22:05:57 DEBUG ----- output ----- 22:05:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:05:57 DEBUG ----- output ----- 22:05:57 DEBUG ----- output ----- 22:05:57 DEBUG --- http response --- 22:05:57 DEBUG ----- output ----- 22:05:57 DEBUG ----- output ----- 22:05:57 DEBUG ----- output ----- 22:05:57 DEBUG ----- output ----- 22:05:57 DEBUG { 22:05:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:05:57 DEBUG ----- output ----- 22:05:57 DEBUG ----- output ----- 22:05:57 DEBUG { 22:05:57 DEBUG ----- output ----- 22:05:57 DEBUG --- http response --- 22:05:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:05:57 DEBUG "status": "success", 22:05:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:05:57 DEBUG --- http response --- 22:05:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:05:57 DEBUG { 22:05:57 DEBUG { 22:05:57 DEBUG --- http response --- 22:05:57 DEBUG { 22:05:57 DEBUG { 22:05:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:05:57 DEBUG { 22:05:57 DEBUG { 22:05:57 DEBUG { 22:05:57 DEBUG { 22:05:57 DEBUG "status": "success", 22:05:57 DEBUG ----- output ----- 22:05:57 DEBUG { 22:05:57 DEBUG ----- output ----- 22:05:57 DEBUG "status": "success", 22:05:57 DEBUG { 22:05:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 22:05:57 DEBUG ----- output ----- 22:05:57 DEBUG "data": { 22:05:57 DEBUG ----- output ----- 22:05:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:05:57 DEBUG ----- output ----- 22:05:57 DEBUG "status": "success", 22:05:57 DEBUG "status": "success", 22:05:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:05:57 DEBUG "status": "success", 22:05:57 DEBUG "status": "success", 22:05:57 DEBUG ----- output ----- 22:05:57 DEBUG "status": "success", 22:05:57 DEBUG "status": "success", 22:05:57 DEBUG "status": "success", 22:05:57 DEBUG "status": "success", 22:05:57 DEBUG "data": { 22:05:57 DEBUG { 22:05:57 DEBUG "status": "success", 22:05:57 DEBUG "data": { 22:05:57 DEBUG "data": { 22:05:57 DEBUG "status": "success", 22:05:57 DEBUG ----- output ----- 22:05:57 DEBUG { 22:05:57 DEBUG "resultType": "vector", 22:05:57 DEBUG { 22:05:57 DEBUG ----- output ----- 22:05:57 DEBUG { 22:05:57 DEBUG "data": { 22:05:57 DEBUG "data": { 22:05:57 DEBUG ----- output ----- 22:05:57 DEBUG "data": { 22:05:57 DEBUG "data": { 22:05:57 DEBUG { 22:05:57 DEBUG "data": { 22:05:57 DEBUG "data": { 22:05:57 DEBUG "data": { 22:05:57 DEBUG "data": { 22:05:57 DEBUG "resultType": "vector", 22:05:57 DEBUG "result": [ 22:05:57 DEBUG "resultType": "vector", 22:05:57 DEBUG { 22:05:57 DEBUG "resultType": "vector", 22:05:57 DEBUG "data": { 22:05:57 DEBUG { 22:05:57 DEBUG "status": "success", 22:05:57 DEBUG "data": { 22:05:57 DEBUG "result": [ 22:05:57 DEBUG { 22:05:57 DEBUG "status": "success", 22:05:57 DEBUG "resultType": "vector", 22:05:57 DEBUG "resultType": "vector", 22:05:57 DEBUG { 22:05:57 DEBUG "resultType": "vector", 22:05:57 DEBUG "resultType": "vector", 22:05:57 DEBUG "status": "success", 22:05:57 DEBUG "data": { 22:05:57 DEBUG "resultType": "vector", 22:05:57 DEBUG "resultType": "vector", 22:05:57 DEBUG "resultType": "vector", 22:05:57 DEBUG { 22:05:57 DEBUG "status": "success", 22:05:57 DEBUG "result": [ 22:05:57 DEBUG "status": "success", 22:05:57 DEBUG "result": [ 22:05:57 DEBUG "resultType": "vector", 22:05:57 DEBUG "status": "success", 22:05:57 DEBUG "status": "success", 22:05:57 DEBUG "resultType": "vector", 22:05:57 DEBUG "result": [ 22:05:57 DEBUG "status": "success", 22:05:57 DEBUG "data": { 22:05:57 DEBUG "result": [ 22:05:57 DEBUG "result": [ 22:05:57 DEBUG "status": "success", 22:05:57 DEBUG "result": [ 22:05:57 DEBUG "result": [ 22:05:57 DEBUG "resultType": "vector", 22:05:57 DEBUG "resultType": "vector", 22:05:57 DEBUG "result": [ 22:05:57 DEBUG "result": [ 22:05:57 DEBUG "result": [ 22:05:57 DEBUG "metric": { 22:05:57 DEBUG "data": { 22:05:57 DEBUG { 22:05:57 DEBUG "data": { 22:05:57 DEBUG { 22:05:57 DEBUG "result": [ 22:05:57 DEBUG "data": { 22:05:57 DEBUG "data": { 22:05:57 DEBUG { 22:05:57 DEBUG { 22:05:57 DEBUG "data": { 22:05:57 DEBUG "resultType": "vector", 22:05:57 DEBUG { 22:05:57 DEBUG { 22:05:57 DEBUG "data": { 22:05:57 DEBUG { 22:05:57 DEBUG { 22:05:57 DEBUG "result": [ 22:05:57 DEBUG "result": [ 22:05:57 DEBUG { 22:05:57 DEBUG { 22:05:57 DEBUG { 22:05:57 DEBUG "pod": "am-767f6b67b5-9zw22" 22:05:57 DEBUG "resultType": "vector", 22:05:57 DEBUG "result": [ 22:05:57 DEBUG "resultType": "vector", 22:05:57 DEBUG "result": [ 22:05:57 DEBUG { 22:05:57 DEBUG "resultType": "vector", 22:05:57 DEBUG "resultType": "vector", 22:05:57 DEBUG "result": [ 22:05:57 DEBUG "metric": { 22:05:57 DEBUG "resultType": "vector", 22:05:57 DEBUG "result": [ 22:05:57 DEBUG "metric": { 22:05:57 DEBUG "metric": { 22:05:57 DEBUG "resultType": "vector", 22:05:57 DEBUG "result": [ 22:05:57 DEBUG "metric": { 22:05:57 DEBUG { 22:05:57 DEBUG { 22:05:57 DEBUG "metric": { 22:05:57 DEBUG "metric": { 22:05:57 DEBUG "metric": { 22:05:57 DEBUG }, 22:05:57 DEBUG { 22:05:57 DEBUG "metric": { 22:05:57 DEBUG "metric": { 22:05:57 DEBUG { 22:05:57 DEBUG "metric": { 22:05:57 DEBUG "result": [ 22:05:57 DEBUG { 22:05:57 DEBUG "metric": { 22:05:57 DEBUG "pod": "am-767f6b67b5-9zw22" 22:05:57 DEBUG "result": [ 22:05:57 DEBUG { 22:05:57 DEBUG "metric": { 22:05:57 DEBUG "pod": "ds-idrepo-1" 22:05:57 DEBUG "metric": { 22:05:57 DEBUG { 22:05:57 DEBUG "pod": "ds-cts-0" 22:05:57 DEBUG }, 22:05:57 DEBUG "metric": { 22:05:57 DEBUG "pod": "am-767f6b67b5-9zw22" 22:05:57 DEBUG "pod": "ds-idrepo-0" 22:05:57 DEBUG "instance": "10.142.0.107:9100" 22:05:57 DEBUG "value": [ 22:05:57 DEBUG "metric": { 22:05:57 DEBUG "instance": "10.142.0.107:9100" 22:05:57 DEBUG }, 22:05:57 DEBUG "metric": { 22:05:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:05:57 DEBUG { 22:05:57 DEBUG "metric": { 22:05:57 DEBUG "instance": "10.142.0.107:9100" 22:05:57 DEBUG }, 22:05:57 DEBUG { 22:05:57 DEBUG "metric": { 22:05:57 DEBUG "pod": "ds-cts-0" 22:05:57 DEBUG }, 22:05:57 DEBUG "instance": "10.142.0.107:9100" 22:05:57 DEBUG "metric": { 22:05:57 DEBUG "metric": { 22:05:57 DEBUG "value": [ 22:05:57 DEBUG "pod": "ds-idrepo-0" 22:05:57 DEBUG }, 22:05:57 DEBUG }, 22:05:57 DEBUG }, 22:05:57 DEBUG 22:05:57 DEBUG "instance": "10.142.0.107:9100" 22:05:57 DEBUG "pod": "am-767f6b67b5-9zw22" 22:05:57 DEBUG "value": [ 22:05:57 DEBUG 22:05:57 DEBUG }, 22:05:57 DEBUG "metric": { 22:05:57 DEBUG "pod": "am-767f6b67b5-9zw22" 22:05:57 DEBUG "value": [ 22:05:57 DEBUG }, 22:05:57 DEBUG "value": [ 22:05:57 DEBUG "pod": "ds-idrepo-0" 22:05:57 DEBUG }, 22:05:57 DEBUG "value": [ 22:05:57 DEBUG }, 22:05:57 DEBUG "value": [ 22:05:57 DEBUG "pod": "ds-idrepo-1" 22:05:57 DEBUG 22:05:57 DEBUG "value": [ 22:05:57 DEBUG "value": [ 22:05:57 DEBUG "value": [ 22:05:57 DEBUG }, 22:05:57 DEBUG 22:05:57 DEBUG }, 22:05:57 DEBUG }, 22:05:57 DEBUG 22:05:57 DEBUG "pod": "ds-cts-0" 22:05:57 DEBUG "value": [ 22:05:57 DEBUG "pod": "ds-idrepo-1" 22:05:57 DEBUG }, 22:05:57 DEBUG 22:05:57 DEBUG "pod": "am-767f6b67b5-9zw22" 22:05:57 DEBUG 22:05:57 DEBUG }, 22:05:57 DEBUG "value": [ 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:05:57 DEBUG }, 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG "value": [ 22:05:57 DEBUG 22:05:57 DEBUG "value": [ 22:05:57 DEBUG "value": [ 22:05:57 DEBUG 22:05:57 DEBUG }, 22:05:57 DEBUG 22:05:57 DEBUG }, 22:05:57 DEBUG "value": [ 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG }, 22:05:57 DEBUG "value": [ 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG }, 22:05:57 DEBUG "value": [ 22:05:57 DEBUG 22:05:57 DEBUG [...] 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG [...] 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG "value": [ 22:05:57 DEBUG 22:05:57 DEBUG "value": [ 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG [...] 22:05:57 DEBUG "value": [ 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG "value": [ 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG [...] 22:05:57 DEBUG 22:05:57 DEBUG [...] 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG [...] 22:05:57 DEBUG [...] 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG [...] 22:05:57 DEBUG [...] 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG [...] 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG [...] 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG [...] 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG [...] 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG [...] 22:05:57 DEBUG 22:05:57 DEBUG [...] 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG [...] 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG [...] 22:05:57 DEBUG 22:05:57 DEBUG [...] 22:05:57 DEBUG "pod": "ds-cts-2" 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG "pod": "am-767f6b67b5-vbltz" 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG "instance": "10.142.0.43:9100" 22:05:57 DEBUG }, 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG [...] 22:05:57 DEBUG 22:05:57 DEBUG }, 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG "pod": "am-767f6b67b5-vbltz" 22:05:57 DEBUG 22:05:57 DEBUG "pod": "am-767f6b67b5-vbltz" 22:05:58 DEBUG }, 22:05:57 DEBUG }, 22:05:58 DEBUG "value": [ 22:05:57 DEBUG [...] 22:05:57 DEBUG "pod": "ds-cts-2" 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG "value": [ 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG "pod": "overseer-0-967d957f7-zk66v" 22:05:57 DEBUG "instance": "10.142.0.43:9100" 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG 22:05:57 DEBUG "value": [ 22:05:57 DEBUG 22:05:57 DEBUG "pod": "ds-idrepo-2" 22:05:58 DEBUG "instance": "10.142.0.43:9100" 22:05:58 DEBUG }, 22:05:58 DEBUG 22:05:58 DEBUG "value": [ 22:05:57 DEBUG "instance": "10.142.0.43:9100" 22:05:58 DEBUG 1713647157, 22:05:57 DEBUG [...] 22:05:58 DEBUG 22:05:58 DEBUG }, 22:05:58 DEBUG 22:05:58 DEBUG [...] 22:05:58 DEBUG 1713647157, 22:05:58 DEBUG "pod": "am-767f6b67b5-vbltz" 22:05:58 DEBUG "pod": "ds-idrepo-2" 22:05:58 DEBUG }, 22:05:58 DEBUG }, 22:05:58 DEBUG 22:05:58 DEBUG 22:05:58 DEBUG 22:05:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:05:58 DEBUG "pod": "ds-idrepo-2" 22:05:58 DEBUG }, 22:05:58 DEBUG }, 22:05:58 DEBUG "value": [ 22:05:58 DEBUG 22:05:58 DEBUG 1713647157, 22:05:58 DEBUG "3322.3333333333335" 22:05:58 DEBUG "4137" 22:05:58 DEBUG 22:05:58 DEBUG 22:05:58 DEBUG "value": [ 22:05:58 DEBUG 1713647157, 22:05:58 DEBUG 22:05:58 DEBUG "100059326.83333334" 22:05:58 DEBUG }, 22:05:58 DEBUG }, 22:05:58 DEBUG "value": [ 22:05:58 DEBUG "value": [ 22:05:58 DEBUG "pod": "ds-cts-2" 22:05:58 DEBUG }, 22:05:58 DEBUG 1713647157, 22:05:58 DEBUG }, 22:05:58 DEBUG 22:05:58 DEBUG "value": [ 22:05:58 DEBUG 1713647157, 22:05:58 DEBUG "value": [ 22:05:58 DEBUG "instance": "10.142.0.43:9100" 22:05:58 DEBUG ] 22:05:58 DEBUG }, 22:05:58 DEBUG ] 22:05:58 DEBUG 22:05:58 DEBUG 22:05:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:05:58 DEBUG "0" 22:05:58 DEBUG 22:05:58 DEBUG ] 22:05:58 DEBUG } 22:05:58 DEBUG "value": [ 22:05:58 DEBUG 1713647157, 22:05:58 DEBUG 1713647157, 22:05:58 DEBUG }, 22:05:58 DEBUG "value": [ 22:05:58 DEBUG 1713647157, 22:05:58 DEBUG "11.858554985557964" 22:05:58 DEBUG "pod": "overseer-0-967d957f7-zk66v" 22:05:58 DEBUG "value": [ 22:05:58 DEBUG "0.41870000000002955" 22:05:58 DEBUG 1713647157, 22:05:58 DEBUG }, 22:05:58 DEBUG } 22:05:58 DEBUG "value": [ 22:05:58 DEBUG } 22:05:58 DEBUG 22:05:58 DEBUG "pod": "overseer-0-967d957f7-zk66v" 22:05:58 DEBUG }, 22:05:58 DEBUG ] 22:05:58 DEBUG 22:05:58 DEBUG "value": [ 22:05:58 DEBUG ] 22:05:58 DEBUG 1713647157, 22:05:58 DEBUG "0" 22:05:58 DEBUG "100055286.03333333" 22:05:58 DEBUG ] 22:05:58 DEBUG "29.7" 22:05:58 DEBUG ] 22:05:58 DEBUG 1713647157, 22:05:58 DEBUG }, 22:05:58 DEBUG 1713647157, 22:05:58 DEBUG "0" 22:05:58 DEBUG ] 22:05:58 DEBUG } 22:05:58 DEBUG ] 22:05:58 DEBUG 1713647157, 22:05:58 DEBUG ] 22:05:58 DEBUG "pod": "am-767f6b67b5-vbltz" 22:05:58 DEBUG }, 22:05:58 DEBUG "value": [ 22:05:58 DEBUG } 22:05:58 DEBUG "pod": "am-767f6b67b5-vbltz" 22:05:58 DEBUG 1713647157, 22:05:58 DEBUG "0" 22:05:58 DEBUG } 22:05:58 DEBUG ] 22:05:58 DEBUG } 22:05:58 DEBUG "value": [ 22:05:58 DEBUG ] 22:05:58 DEBUG } 22:05:58 DEBUG "0" 22:05:58 DEBUG "value": [ 22:05:58 DEBUG "3333.866666666667" 22:05:58 DEBUG ] 22:05:58 DEBUG ] 22:05:58 DEBUG "value": [ 22:05:58 DEBUG } 22:05:58 DEBUG "0.09133333333332605" 22:05:58 DEBUG } 22:05:58 DEBUG }, 22:05:58 DEBUG "value": [ 22:05:58 DEBUG 1713647157, 22:05:58 DEBUG ] 22:05:58 DEBUG }, 22:05:58 DEBUG ] 22:05:58 DEBUG } 22:05:58 DEBUG } 22:05:58 DEBUG ] 22:05:58 DEBUG "0" 22:05:58 DEBUG 1713647157, 22:05:58 DEBUG } 22:05:58 DEBUG ] 22:05:58 DEBUG ] 22:05:58 DEBUG } 22:05:58 DEBUG ] 22:05:58 DEBUG } 22:05:58 DEBUG } 22:05:58 DEBUG 1713647157, 22:05:58 DEBUG } 22:05:58 DEBUG ] 22:05:58 DEBUG } 22:05:58 DEBUG "value": [ 22:05:58 DEBUG "7.821210935309821" 22:05:58 DEBUG 1713647157, 22:05:58 DEBUG } 22:05:58 DEBUG "value": [ 22:05:58 DEBUG } 22:05:58 DEBUG ] 22:05:58 DEBUG } 22:05:58 DEBUG ] 22:05:58 DEBUG "0.6338666666666655" 22:05:58 DEBUG ] 22:05:58 DEBUG } 22:05:58 DEBUG 1713647157, 22:05:58 DEBUG ] 22:05:58 DEBUG } 22:05:58 DEBUG } 22:05:58 DEBUG ] 22:05:58 DEBUG "4060163267584" 22:05:58 DEBUG } 22:05:58 DEBUG 1713647157, 22:05:58 DEBUG ] 22:05:58 DEBUG "1056608256" 22:05:58 DEBUG } 22:05:58 DEBUG 1713647157, 22:05:58 DEBUG ] 22:05:58 DEBUG } 22:05:58 DEBUG } 22:05:58 DEBUG } 22:05:58 DEBUG ] 22:05:58 DEBUG } 22:05:58 DEBUG } 22:05:58 DEBUG "0" 22:05:58 DEBUG } 22:05:58 DEBUG ] 22:05:58 DEBUG } 22:05:58 DEBUG ] 22:05:58 DEBUG ] 22:05:58 DEBUG "55.631" 22:05:58 DEBUG ] 22:05:58 DEBUG ] 22:05:58 DEBUG "599811.5" 22:05:58 DEBUG } 22:05:58 DEBUG } 22:05:58 DEBUG ] 22:05:58 DEBUG } 22:05:58 DEBUG } 22:05:58 DEBUG ] 22:05:58 DEBUG } 22:05:58 DEBUG } 22:05:58 DEBUG } 22:05:58 DEBUG } 22:05:58 DEBUG } 22:05:58 DEBUG } 22:05:58 DEBUG } 22:05:58 DEBUG } 22:05:58 DEBUG ] 22:05:58 DEBUG } 22:05:58 DEBUG } 22:05:58 DEBUG ] 22:05:58 DEBUG } 22:05:58 DEBUG } 22:05:58 DEBUG ] 22:05:58 DEBUG } 22:05:58 DEBUG ] 22:05:58 DEBUG ] 22:05:58 DEBUG ] 22:05:58 DEBUG } 22:05:58 DEBUG } 22:05:58 DEBUG } 22:05:58 DEBUG ] 22:05:58 DEBUG } 22:05:58 DEBUG } 22:05:58 DEBUG } 22:05:58 DEBUG } 22:05:58 DEBUG ] 22:05:58 DEBUG } 22:05:58 DEBUG } 22:05:58 DEBUG } 22:05:58 DEBUG } 22:05:58 DEBUG } 22:05:58 DEBUG } 22:05:58 DEBUG } 22:05:58 DEBUG } 22:05:58 DEBUG } 22:06:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713647214" 22:06:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647214" 22:06:54 ERROR [http_cmd]: ERROR 22:06:54 ERROR --- status code --- 22:06:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:06:54 ERROR --- http response --- 22:06:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:06:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:06:54 INFO [http_cmd]: http status code OK 22:06:54 DEBUG --- status code --- 22:06:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:06:54 DEBUG --- http response --- 22:06:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:06:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647214" 22:06:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713647214" 22:06:54 INFO [http_cmd]: http status code OK 22:06:54 INFO [http_cmd]: http status code OK 22:06:54 DEBUG --- status code --- 22:06:54 DEBUG --- status code --- 22:06:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:06:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:06:54 DEBUG --- http response --- 22:06:54 DEBUG --- http response --- 22:06:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:06:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 22:06:54 DEBUG ----- output ----- 22:06:54 DEBUG ----- output ----- 22:06:54 DEBUG { 22:06:54 DEBUG { 22:06:54 DEBUG "status": "success", 22:06:54 DEBUG "status": "success", 22:06:54 DEBUG "data": { 22:06:54 DEBUG "data": { 22:06:54 DEBUG "resultType": "vector", 22:06:54 DEBUG "resultType": "vector", 22:06:54 DEBUG "result": [ 22:06:54 DEBUG "result": [ 22:06:54 DEBUG { 22:06:54 DEBUG { 22:06:54 DEBUG "metric": { 22:06:54 DEBUG "metric": { 22:06:54 DEBUG "pod": "ds-idrepo-0" 22:06:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:06:54 DEBUG }, 22:06:54 DEBUG }, 22:06:54 DEBUG "value": [ 22:06:54 DEBUG "value": [ 22:06:54 DEBUG 22:06:54 DEBUG 22:06:54 DEBUG 22:06:54 DEBUG 22:06:54 DEBUG [...] 22:06:54 DEBUG 22:06:54 DEBUG 22:06:54 DEBUG 22:06:54 DEBUG 22:06:54 DEBUG [...] 22:06:54 DEBUG 22:06:54 DEBUG 22:06:54 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 22:06:54 DEBUG 22:06:54 DEBUG }, 22:06:54 DEBUG 22:06:54 DEBUG "value": [ 22:06:54 DEBUG "pod": "ds-cts-2" 22:06:54 DEBUG 1713647214, 22:06:54 DEBUG }, 22:06:54 DEBUG "0" 22:06:54 DEBUG "value": [ 22:06:54 DEBUG 1713647214, 22:06:54 DEBUG "0" 22:06:54 DEBUG ] 22:06:54 DEBUG ] 22:06:54 DEBUG } 22:06:54 DEBUG } 22:06:54 DEBUG ] 22:06:54 DEBUG ] 22:06:54 DEBUG } 22:06:54 DEBUG } 22:06:54 DEBUG } 22:06:54 DEBUG } 22:06:56 INFO 22:06:56 INFO [loop_until]: kubectl --namespace=xlou top pods 22:06:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:06:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647216" 22:06:56 INFO [http_cmd]: http status code OK 22:06:56 DEBUG --- status code --- 22:06:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:06:56 DEBUG --- http response --- 22:06:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:06:56 DEBUG ----- output ----- 22:06:56 DEBUG { 22:06:56 DEBUG "status": "success", 22:06:56 DEBUG "data": { 22:06:56 DEBUG "resultType": "vector", 22:06:56 DEBUG "result": [ 22:06:56 DEBUG { 22:06:56 DEBUG "metric": { 22:06:56 DEBUG "pod": "am-767f6b67b5-9zw22" 22:06:56 DEBUG }, 22:06:56 DEBUG "value": [ 22:06:56 DEBUG 22:06:56 DEBUG 22:06:56 DEBUG 22:06:56 DEBUG [...] 22:06:56 DEBUG 22:06:56 DEBUG 22:06:56 DEBUG 22:06:56 DEBUG "pod": "am-767f6b67b5-vbltz" 22:06:56 DEBUG }, 22:06:56 DEBUG "value": [ 22:06:56 DEBUG 1713647216, 22:06:56 DEBUG "3356.9333333333334" 22:06:56 DEBUG ] 22:06:56 DEBUG } 22:06:56 DEBUG ] 22:06:56 DEBUG } 22:06:56 DEBUG } 22:06:56 INFO [loop_until]: OK (rc = 0) 22:06:56 DEBUG --- stdout --- 22:06:56 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 22621m 8803Mi am-767f6b67b5-qcqz2 22658m 9002Mi am-767f6b67b5-vbltz 24778m 8957Mi ds-cts-0 20419m 12310Mi ds-cts-1 19617m 12281Mi ds-cts-2 20224m 12474Mi ds-idrepo-0 9410m 3912Mi ds-idrepo-1 9442m 3966Mi ds-idrepo-2 9491m 3878Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-7dd48676f6-tdxl9 17m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-967d957f7-zk66v 11903m 1116Mi 22:06:56 DEBUG --- stderr --- 22:06:56 DEBUG 22:06:58 INFO 22:06:58 INFO [loop_until]: kubectl --namespace=xlou top node 22:06:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647218" 22:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647218" 22:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647218" 22:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647218" 22:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647218" 22:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647218" 22:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713647218" 22:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647218" 22:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647218" 22:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647218" 22:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647218" 22:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647218" 22:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647218" 22:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647218" 22:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647218" 22:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647218" 22:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647218" 22:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647218" 22:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713647218" 22:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647218" 22:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647218" 22:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713647218" 22:06:58 INFO [http_cmd]: http status code OK 22:06:58 INFO [http_cmd]: http status code OK 22:06:58 INFO [http_cmd]: http status code OK 22:06:58 INFO [http_cmd]: http status code OK 22:06:58 INFO [loop_until]: OK (rc = 0) 22:06:58 INFO [http_cmd]: http status code OK 22:06:58 INFO [http_cmd]: http status code OK 22:06:58 INFO [http_cmd]: http status code OK 22:06:58 INFO [http_cmd]: http status code OK 22:06:58 INFO [http_cmd]: http status code OK 22:06:58 INFO [http_cmd]: http status code OK 22:06:58 INFO [http_cmd]: http status code OK 22:06:58 INFO [http_cmd]: http status code OK 22:06:58 INFO [http_cmd]: http status code OK 22:06:58 INFO [http_cmd]: http status code OK 22:06:58 DEBUG --- status code --- 22:06:58 INFO [http_cmd]: http status code OK 22:06:58 INFO [http_cmd]: http status code OK 22:06:58 INFO [http_cmd]: http status code OK 22:06:58 DEBUG --- status code --- 22:06:58 DEBUG --- status code --- 22:06:58 DEBUG --- status code --- 22:06:58 DEBUG --- status code --- 22:06:58 INFO [http_cmd]: http status code OK 22:06:58 DEBUG --- stdout --- 22:06:58 DEBUG --- status code --- 22:06:58 DEBUG --- status code --- 22:06:58 DEBUG --- status code --- 22:06:58 DEBUG --- status code --- 22:06:58 DEBUG --- status code --- 22:06:58 DEBUG --- status code --- 22:06:58 DEBUG --- status code --- 22:06:58 DEBUG --- status code --- 22:06:58 DEBUG --- status code --- 22:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:06:58 INFO [http_cmd]: http status code OK 22:06:58 DEBUG --- status code --- 22:06:58 DEBUG --- status code --- 22:06:58 INFO [http_cmd]: http status code OK 22:06:58 DEBUG --- status code --- 22:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:06:58 DEBUG --- status code --- 22:06:58 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 61m 0% 6194Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 9959m 33% 7678Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 22408m 75% 11530Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 78m 0% 3360Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 20260m 67% 15563Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 138m 0% 5177Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 22758m 76% 11862Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19200m 64% 3066Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 25053m 83% 11206Mi 10% gke-xlou-cdm-ds-0f601751-j488 10338m 34% 7297Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19742m 66% 17361Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 10178m 34% 7943Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20290m 67% 15181Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 12111m 76% 9015Mi 15% 22:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:06:58 INFO [http_cmd]: http status code OK 22:06:58 INFO [http_cmd]: http status code OK 22:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:06:58 DEBUG --- http response --- 22:06:58 DEBUG --- status code --- 22:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:06:58 DEBUG --- status code --- 22:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:06:58 DEBUG --- http response --- 22:06:58 DEBUG --- http response --- 22:06:58 DEBUG --- http response --- 22:06:58 DEBUG --- http response --- 22:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:06:58 DEBUG --- stderr --- 22:06:58 DEBUG --- http response --- 22:06:58 DEBUG --- http response --- 22:06:58 DEBUG --- http response --- 22:06:58 DEBUG --- http response --- 22:06:58 DEBUG --- http response --- 22:06:58 DEBUG --- http response --- 22:06:58 DEBUG --- http response --- 22:06:58 DEBUG --- status code --- 22:06:58 DEBUG --- status code --- 22:06:58 DEBUG --- http response --- 22:06:58 DEBUG --- http response --- 22:06:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:06:58 DEBUG --- http response --- 22:06:58 DEBUG --- http response --- 22:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:06:58 DEBUG --- http response --- 22:06:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:06:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:06:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:06:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:06:58 DEBUG --- http response --- 22:06:58 DEBUG 22:06:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:06:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:06:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:06:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:06:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:06:58 DEBUG ----- output ----- 22:06:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:06:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:06:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:06:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:06:58 DEBUG ----- output ----- 22:06:58 DEBUG --- http response --- 22:06:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:06:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:06:58 DEBUG --- http response --- 22:06:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:06:58 DEBUG ----- output ----- 22:06:58 DEBUG ----- output ----- 22:06:58 DEBUG ----- output ----- 22:06:58 DEBUG ----- output ----- 22:06:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:06:58 DEBUG ----- output ----- 22:06:58 DEBUG ----- output ----- 22:06:58 DEBUG ----- output ----- 22:06:58 DEBUG { 22:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:06:58 DEBUG { 22:06:58 DEBUG ----- output ----- 22:06:58 DEBUG ----- output ----- 22:06:58 DEBUG --- http response --- 22:06:58 DEBUG ----- output ----- 22:06:58 DEBUG ----- output ----- 22:06:58 DEBUG { 22:06:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:06:58 DEBUG ----- output ----- 22:06:58 DEBUG ----- output ----- 22:06:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:06:58 DEBUG ----- output ----- 22:06:58 DEBUG { 22:06:58 DEBUG { 22:06:58 DEBUG { 22:06:58 DEBUG { 22:06:58 DEBUG ----- output ----- 22:06:58 DEBUG { 22:06:58 DEBUG { 22:06:58 DEBUG "status": "success", 22:06:58 DEBUG --- http response --- 22:06:58 DEBUG ----- output ----- 22:06:58 DEBUG "status": "success", 22:06:58 DEBUG { 22:06:58 DEBUG { 22:06:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:06:58 DEBUG { 22:06:58 DEBUG { 22:06:58 DEBUG "status": "success", 22:06:58 DEBUG ----- output ----- 22:06:58 DEBUG { 22:06:58 DEBUG { 22:06:58 DEBUG ----- output ----- 22:06:58 DEBUG { 22:06:58 DEBUG "status": "success", 22:06:58 DEBUG "status": "success", 22:06:58 DEBUG "status": "success", 22:06:58 DEBUG "status": "success", 22:06:58 DEBUG { 22:06:58 DEBUG "status": "success", 22:06:58 DEBUG "status": "success", 22:06:58 DEBUG "data": { 22:06:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:06:58 DEBUG { 22:06:58 DEBUG "data": { 22:06:58 DEBUG "status": "success", 22:06:58 DEBUG "status": "success", 22:06:58 DEBUG ----- output ----- 22:06:58 DEBUG "status": "success", 22:06:58 DEBUG "status": "success", 22:06:58 DEBUG "data": { 22:06:58 DEBUG { 22:06:58 DEBUG "status": "success", 22:06:58 DEBUG "status": "success", 22:06:58 DEBUG { 22:06:58 DEBUG "status": "success", 22:06:58 DEBUG "data": { 22:06:58 DEBUG "data": { 22:06:58 DEBUG "data": { 22:06:58 DEBUG "data": { 22:06:58 DEBUG "status": "success", 22:06:58 DEBUG "data": { 22:06:58 DEBUG "resultType": "vector", 22:06:58 DEBUG "resultType": "vector", 22:06:58 DEBUG ----- output ----- 22:06:58 DEBUG "status": "success", 22:06:58 DEBUG "resultType": "vector", 22:06:58 DEBUG "data": { 22:06:58 DEBUG "data": { 22:06:58 DEBUG { 22:06:58 DEBUG "data": { 22:06:58 DEBUG "data": { 22:06:58 DEBUG "resultType": "vector", 22:06:58 DEBUG "status": "success", 22:06:58 DEBUG "data": { 22:06:58 DEBUG "data": { 22:06:58 DEBUG "status": "success", 22:06:58 DEBUG "data": { 22:06:58 DEBUG "resultType": "vector", 22:06:58 DEBUG "resultType": "vector", 22:06:58 DEBUG "resultType": "vector", 22:06:58 DEBUG "resultType": "vector", 22:06:58 DEBUG "data": { 22:06:58 DEBUG "data": { 22:06:58 DEBUG "result": [ 22:06:58 DEBUG "result": [ 22:06:58 DEBUG { 22:06:58 DEBUG "status": "success", 22:06:58 DEBUG "result": [ 22:06:58 DEBUG "resultType": "vector", 22:06:58 DEBUG "resultType": "vector", 22:06:58 DEBUG "status": "success", 22:06:58 DEBUG "resultType": "vector", 22:06:58 DEBUG "resultType": "vector", 22:06:58 DEBUG "result": [ 22:06:58 DEBUG "data": { 22:06:58 DEBUG "resultType": "vector", 22:06:58 DEBUG "resultType": "vector", 22:06:58 DEBUG "resultType": "vector", 22:06:58 DEBUG "data": { 22:06:58 DEBUG "result": [ 22:06:58 DEBUG "result": [ 22:06:58 DEBUG "result": [ 22:06:58 DEBUG "result": [ 22:06:58 DEBUG "resultType": "vector", 22:06:58 DEBUG "resultType": "vector", 22:06:58 DEBUG { 22:06:58 DEBUG { 22:06:58 DEBUG "data": { 22:06:58 DEBUG "data": { 22:06:58 DEBUG { 22:06:58 DEBUG "result": [ 22:06:58 DEBUG "result": [ 22:06:58 DEBUG "data": { 22:06:58 DEBUG "result": [ 22:06:58 DEBUG { 22:06:58 DEBUG "result": [ 22:06:58 DEBUG "resultType": "vector", 22:06:58 DEBUG "result": [ 22:06:58 DEBUG "result": [ 22:06:58 DEBUG "result": [ 22:06:58 DEBUG "resultType": "vector", 22:06:58 DEBUG { 22:06:58 DEBUG { 22:06:58 DEBUG { 22:06:58 DEBUG { 22:06:58 DEBUG "result": [ 22:06:58 DEBUG "result": [ 22:06:58 DEBUG "metric": { 22:06:58 DEBUG "metric": { 22:06:58 DEBUG "resultType": "vector", 22:06:58 DEBUG "resultType": "vector", 22:06:58 DEBUG "metric": { 22:06:58 DEBUG { 22:06:58 DEBUG { 22:06:58 DEBUG "resultType": "vector", 22:06:58 DEBUG { 22:06:58 DEBUG "metric": { 22:06:58 DEBUG { 22:06:58 DEBUG "metric": { 22:06:58 DEBUG "pod": "ds-cts-0" 22:06:58 DEBUG { 22:06:58 DEBUG { 22:06:58 DEBUG "result": [ 22:06:58 DEBUG "metric": { 22:06:58 DEBUG "metric": { 22:06:58 DEBUG "metric": { 22:06:58 DEBUG "metric": { 22:06:58 DEBUG { 22:06:58 DEBUG { 22:06:58 DEBUG "pod": "ds-cts-0" 22:06:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:06:58 DEBUG "result": [ 22:06:58 DEBUG "result": [ 22:06:58 DEBUG "pod": "ds-cts-0" 22:06:58 DEBUG "metric": { 22:06:58 DEBUG "metric": { 22:06:58 DEBUG "result": [ 22:06:58 DEBUG "metric": { 22:06:58 DEBUG "pod": "ds-idrepo-0" 22:06:58 DEBUG { 22:06:58 DEBUG }, 22:06:58 DEBUG "result": [ 22:06:58 DEBUG "metric": { 22:06:58 DEBUG "metric": { 22:06:58 DEBUG { 22:06:58 DEBUG "pod": "am-767f6b67b5-9zw22" 22:06:58 DEBUG "pod": "am-767f6b67b5-9zw22" 22:06:58 DEBUG "instance": "10.142.0.107:9100" 22:06:58 DEBUG "pod": "ds-idrepo-0" 22:06:58 DEBUG "metric": { 22:06:58 DEBUG "metric": { 22:06:58 DEBUG }, 22:06:58 DEBUG }, 22:06:58 DEBUG { 22:06:58 DEBUG "metric": { 22:06:58 DEBUG }, 22:06:58 DEBUG "value": [ 22:06:58 DEBUG "pod": "ds-idrepo-0" 22:06:58 DEBUG { 22:06:58 DEBUG "instance": "10.142.0.107:9100" 22:06:58 DEBUG }, 22:06:58 DEBUG "metric": { 22:06:58 DEBUG "value": [ 22:06:58 DEBUG { 22:06:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:06:58 DEBUG "instance": "10.142.0.107:9100" 22:06:58 DEBUG "metric": { 22:06:58 DEBUG }, 22:06:58 DEBUG }, 22:06:58 DEBUG }, 22:06:58 DEBUG }, 22:06:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:06:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:06:58 DEBUG "value": [ 22:06:58 DEBUG "value": [ 22:06:58 DEBUG "pod": "am-767f6b67b5-9zw22" 22:06:58 DEBUG { 22:06:58 DEBUG 22:06:58 DEBUG "instance": "10.142.0.107:9100" 22:06:58 DEBUG }, 22:06:58 DEBUG "metric": { 22:06:58 DEBUG }, 22:06:58 DEBUG "value": [ 22:06:58 DEBUG "pod": "am-767f6b67b5-9zw22" 22:06:58 DEBUG 22:06:58 DEBUG "metric": { 22:06:58 DEBUG }, 22:06:58 DEBUG }, 22:06:58 DEBUG "pod": "am-767f6b67b5-9zw22" 22:06:58 DEBUG "value": [ 22:06:58 DEBUG "value": [ 22:06:58 DEBUG "value": [ 22:06:58 DEBUG "value": [ 22:06:58 DEBUG }, 22:06:58 DEBUG "value": [ 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG }, 22:06:58 DEBUG "metric": { 22:06:58 DEBUG "instance": "10.142.0.107:9100" 22:06:58 DEBUG }, 22:06:58 DEBUG "value": [ 22:06:58 DEBUG "pod": "am-767f6b67b5-9zw22" 22:06:58 DEBUG }, 22:06:58 DEBUG 22:06:58 DEBUG }, 22:06:58 DEBUG 22:06:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:06:58 DEBUG "value": [ 22:06:58 DEBUG "value": [ 22:06:58 DEBUG }, 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG }, 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG "value": [ 22:06:58 DEBUG }, 22:06:58 DEBUG 22:06:58 DEBUG "value": [ 22:06:58 DEBUG 22:06:58 DEBUG "value": [ 22:06:58 DEBUG "value": [ 22:06:58 DEBUG 22:06:58 DEBUG "value": [ 22:06:58 DEBUG 22:06:58 DEBUG }, 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG "value": [ 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG [...] 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG "value": [ 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG [...] 22:06:58 DEBUG "value": [ 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG [...] 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG "value": [ 22:06:58 DEBUG 22:06:58 DEBUG [...] 22:06:58 DEBUG [...] 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG [...] 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG [...] 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG [...] 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG [...] 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG [...] 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG [...] 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG [...] 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG [...] 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG [...] 22:06:58 DEBUG 22:06:58 DEBUG [...] 22:06:58 DEBUG [...] 22:06:58 DEBUG 22:06:58 DEBUG [...] 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG "pod": "am-767f6b67b5-vbltz" 22:06:58 DEBUG [...] 22:06:58 DEBUG 22:06:58 DEBUG "pod": "ds-idrepo-2" 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG [...] 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG "pod": "ds-cts-2" 22:06:58 DEBUG [...] 22:06:58 DEBUG [...] 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG }, 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG }, 22:06:58 DEBUG [...] 22:06:58 DEBUG 22:06:58 DEBUG "pod": "ds-cts-2" 22:06:58 DEBUG "pod": "overseer-0-967d957f7-zk66v" 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG "pod": "ds-cts-2" 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG "pod": "ds-idrepo-2" 22:06:58 DEBUG }, 22:06:58 DEBUG }, 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG "pod": "overseer-0-967d957f7-zk66v" 22:06:58 DEBUG }, 22:06:58 DEBUG "value": [ 22:06:58 DEBUG 22:06:58 DEBUG "instance": "10.142.0.43:9100" 22:06:58 DEBUG "value": [ 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG }, 22:06:58 DEBUG }, 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG }, 22:06:58 DEBUG "instance": "10.142.0.43:9100" 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG "value": [ 22:06:58 DEBUG 22:06:58 DEBUG "value": [ 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG 1713647218, 22:06:58 DEBUG "value": [ 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG }, 22:06:58 DEBUG 1713647218, 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG "value": [ 22:06:58 DEBUG "value": [ 22:06:58 DEBUG "pod": "am-767f6b67b5-vbltz" 22:06:58 DEBUG }, 22:06:58 DEBUG "value": [ 22:06:58 DEBUG }, 22:06:58 DEBUG "instance": "10.142.0.43:9100" 22:06:58 DEBUG "pod": "ds-idrepo-2" 22:06:58 DEBUG 22:06:58 DEBUG 1713647218, 22:06:58 DEBUG 22:06:58 DEBUG 1713647218, 22:06:58 DEBUG 22:06:58 DEBUG 22:06:58 DEBUG "4.498175270729881" 22:06:58 DEBUG 1713647218, 22:06:58 DEBUG "instance": "10.142.0.43:9100" 22:06:58 DEBUG "pod": "am-767f6b67b5-vbltz" 22:06:58 DEBUG "value": [ 22:06:58 DEBUG "0" 22:06:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:06:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:06:58 DEBUG 1713647218, 22:06:58 DEBUG 1713647218, 22:06:58 DEBUG "value": [ 22:06:58 DEBUG 1713647218, 22:06:58 DEBUG "value": [ 22:06:58 DEBUG }, 22:06:58 DEBUG "pod": "am-767f6b67b5-vbltz" 22:06:58 DEBUG }, 22:06:58 DEBUG "instance": "10.142.0.43:9100" 22:06:58 DEBUG "0" 22:06:58 DEBUG "pod": "am-767f6b67b5-vbltz" 22:06:58 DEBUG "0.5770333333333345" 22:06:58 DEBUG "pod": "overseer-0-967d957f7-zk66v" 22:06:58 DEBUG "pod": "am-767f6b67b5-vbltz" 22:06:58 DEBUG ] 22:06:58 DEBUG "3852.9333333333334" 22:06:58 DEBUG }, 22:06:58 DEBUG }, 22:06:58 DEBUG 1713647218, 22:06:58 DEBUG ] 22:06:58 DEBUG }, 22:06:58 DEBUG }, 22:06:58 DEBUG "0" 22:06:58 DEBUG ] 22:06:58 DEBUG } 22:06:58 DEBUG "60.521" 22:06:58 DEBUG 1713647218, 22:06:58 DEBUG "value": [ 22:06:58 DEBUG }, 22:06:58 DEBUG "value": [ 22:06:58 DEBUG }, 22:06:58 DEBUG ] 22:06:58 DEBUG }, 22:06:58 DEBUG "value": [ 22:06:58 DEBUG }, 22:06:58 DEBUG }, 22:06:58 DEBUG } 22:06:58 DEBUG ] 22:06:58 DEBUG "value": [ 22:06:58 DEBUG "value": [ 22:06:58 DEBUG "100726706.76666667" 22:06:58 DEBUG } 22:06:58 DEBUG ] 22:06:58 DEBUG "value": [ 22:06:58 DEBUG ] 22:06:58 DEBUG "0" 22:06:58 DEBUG ] 22:06:58 DEBUG "0.3664333333334336" 22:06:58 DEBUG 1713647218, 22:06:58 DEBUG "value": [ 22:06:58 DEBUG 1713647218, 22:06:58 DEBUG "value": [ 22:06:58 DEBUG 1713647218, 22:06:58 DEBUG "100723325.8" 22:06:58 DEBUG ] 22:06:58 DEBUG 1713647218, 22:06:58 DEBUG "value": [ 22:06:58 DEBUG "value": [ 22:06:58 DEBUG ] 22:06:58 DEBUG } 22:06:58 DEBUG ] 22:06:58 DEBUG } 22:06:58 DEBUG } 22:06:58 DEBUG "value": [ 22:06:58 DEBUG } 22:06:58 DEBUG 1713647218, 22:06:58 DEBUG } 22:06:58 DEBUG ] 22:06:58 DEBUG } 22:06:58 DEBUG ] 22:06:58 DEBUG "0.06266666666668357" 22:06:58 DEBUG 1713647218, 22:06:58 DEBUG "0" 22:06:58 DEBUG ] 22:06:58 DEBUG } 22:06:58 DEBUG } 22:06:58 DEBUG "3357.5333333333333" 22:06:58 DEBUG 1713647218, 22:06:58 DEBUG 1713647218, 22:06:58 DEBUG 1713647218, 22:06:58 DEBUG } 22:06:58 DEBUG 1713647218, 22:06:58 DEBUG 1713647218, 22:06:58 DEBUG ] 22:06:58 DEBUG 1713647218, 22:06:58 DEBUG } 22:06:58 DEBUG "1266262016" 22:06:58 DEBUG } 22:06:58 DEBUG } 22:06:58 DEBUG ] 22:06:58 DEBUG } 22:06:58 DEBUG ] 22:06:58 DEBUG "0" 22:06:58 DEBUG ] 22:06:58 DEBUG } 22:06:58 DEBUG ] 22:06:58 DEBUG ] 22:06:58 DEBUG ] 22:06:58 DEBUG "0" 22:06:58 DEBUG "28.4" 22:06:58 DEBUG "599656" 22:06:58 DEBUG } 22:06:58 DEBUG "3328.733333333333" 22:06:58 DEBUG "4065258474496" 22:06:58 DEBUG } 22:06:58 DEBUG "11.873557471339526" 22:06:58 DEBUG ] 22:06:58 DEBUG ] 22:06:58 DEBUG } 22:06:58 DEBUG ] 22:06:58 DEBUG } 22:06:58 DEBUG ] 22:06:58 DEBUG } 22:06:58 DEBUG ] 22:06:58 DEBUG } 22:06:58 DEBUG } 22:06:58 DEBUG } 22:06:58 DEBUG ] 22:06:58 DEBUG ] 22:06:58 DEBUG ] 22:06:58 DEBUG ] 22:06:58 DEBUG ] 22:06:58 DEBUG ] 22:06:58 DEBUG ] 22:06:58 DEBUG } 22:06:58 DEBUG } 22:06:58 DEBUG } 22:06:58 DEBUG } 22:06:58 DEBUG ] 22:06:58 DEBUG } 22:06:58 DEBUG ] 22:06:58 DEBUG } 22:06:58 DEBUG } 22:06:58 DEBUG } 22:06:58 DEBUG ] 22:06:58 DEBUG } 22:06:58 DEBUG } 22:06:58 DEBUG } 22:06:58 DEBUG } 22:06:58 DEBUG } 22:06:58 DEBUG } 22:06:58 DEBUG } 22:06:58 DEBUG ] 22:06:58 DEBUG } 22:06:58 DEBUG } 22:06:58 DEBUG } 22:06:58 DEBUG ] 22:06:58 DEBUG } 22:06:58 DEBUG } 22:06:58 DEBUG } 22:06:58 DEBUG } 22:06:58 DEBUG ] 22:06:58 DEBUG } 22:06:58 DEBUG ] 22:06:58 DEBUG ] 22:06:58 DEBUG } 22:06:58 DEBUG ] 22:06:58 DEBUG } 22:06:58 DEBUG } 22:06:58 DEBUG } 22:06:58 DEBUG } 22:06:58 DEBUG } 22:06:58 DEBUG ] 22:06:58 DEBUG ] 22:06:58 DEBUG } 22:06:58 DEBUG } 22:06:58 DEBUG } 22:06:58 DEBUG } 22:06:58 DEBUG } 22:06:58 DEBUG } 22:06:58 DEBUG } 22:06:58 DEBUG } 22:06:58 DEBUG } 22:06:58 DEBUG } 22:06:58 DEBUG } 22:06:58 DEBUG } 22:07:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713647274" 22:07:54 ERROR [http_cmd]: ERROR 22:07:54 ERROR --- status code --- 22:07:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:07:54 ERROR --- http response --- 22:07:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647274" 22:07:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:07:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:07:54 INFO [http_cmd]: http status code OK 22:07:54 DEBUG --- status code --- 22:07:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:07:54 DEBUG --- http response --- 22:07:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:07:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713647274" 22:07:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647274" 22:07:54 INFO [http_cmd]: http status code OK 22:07:54 DEBUG --- status code --- 22:07:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:07:54 DEBUG --- http response --- 22:07:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:07:54 DEBUG ----- output ----- 22:07:54 DEBUG { 22:07:54 DEBUG "status": "success", 22:07:54 DEBUG "data": { 22:07:54 DEBUG "resultType": "vector", 22:07:54 DEBUG "result": [ 22:07:54 DEBUG { 22:07:54 DEBUG "metric": { 22:07:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:07:54 DEBUG }, 22:07:54 DEBUG "value": [ 22:07:54 DEBUG 22:07:54 DEBUG 22:07:54 DEBUG 22:07:54 DEBUG [...] 22:07:54 DEBUG 22:07:54 DEBUG 22:07:54 DEBUG 22:07:54 INFO [http_cmd]: http status code OK 22:07:54 DEBUG --- status code --- 22:07:54 DEBUG "pod": "am-767f6b67b5-9zw22" 22:07:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:07:54 DEBUG }, 22:07:54 DEBUG --- http response --- 22:07:54 DEBUG "value": [ 22:07:54 DEBUG 1713647274, 22:07:54 DEBUG "22814.02147252281" 22:07:54 DEBUG ] 22:07:54 DEBUG } 22:07:54 DEBUG ] 22:07:54 DEBUG } 22:07:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:07:54 DEBUG } 22:07:54 DEBUG ----- output ----- 22:07:54 DEBUG { 22:07:54 DEBUG "status": "success", 22:07:54 DEBUG "data": { 22:07:54 DEBUG "resultType": "vector", 22:07:54 DEBUG "result": [ 22:07:54 DEBUG { 22:07:54 DEBUG "metric": { 22:07:54 DEBUG "pod": "ds-idrepo-0" 22:07:54 DEBUG }, 22:07:54 DEBUG "value": [ 22:07:54 DEBUG 22:07:54 DEBUG 22:07:54 DEBUG 22:07:54 DEBUG [...] 22:07:54 DEBUG 22:07:54 DEBUG 22:07:54 DEBUG 22:07:54 DEBUG "pod": "ds-cts-2" 22:07:54 DEBUG }, 22:07:54 DEBUG "value": [ 22:07:54 DEBUG 1713647274, 22:07:54 DEBUG "0" 22:07:54 DEBUG ] 22:07:54 DEBUG } 22:07:54 DEBUG ] 22:07:54 DEBUG } 22:07:54 DEBUG } 22:07:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647276" 22:07:56 INFO [http_cmd]: http status code OK 22:07:56 DEBUG --- status code --- 22:07:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:07:56 DEBUG --- http response --- 22:07:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:07:56 DEBUG ----- output ----- 22:07:56 DEBUG { 22:07:56 DEBUG "status": "success", 22:07:56 DEBUG "data": { 22:07:56 DEBUG "resultType": "vector", 22:07:56 DEBUG "result": [ 22:07:56 DEBUG { 22:07:56 DEBUG "metric": { 22:07:56 DEBUG "pod": "am-767f6b67b5-9zw22" 22:07:56 DEBUG }, 22:07:56 DEBUG "value": [ 22:07:56 DEBUG 22:07:56 DEBUG 22:07:56 DEBUG 22:07:56 DEBUG [...] 22:07:56 DEBUG 22:07:56 DEBUG 22:07:56 DEBUG 22:07:56 DEBUG "pod": "am-767f6b67b5-vbltz" 22:07:56 DEBUG }, 22:07:56 DEBUG "value": [ 22:07:56 DEBUG 1713647276, 22:07:56 DEBUG "3337.9666666666667" 22:07:56 DEBUG ] 22:07:56 DEBUG } 22:07:56 DEBUG ] 22:07:56 DEBUG } 22:07:56 DEBUG } 22:07:56 INFO 22:07:56 INFO [loop_until]: kubectl --namespace=xlou top pods 22:07:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:07:56 INFO [loop_until]: OK (rc = 0) 22:07:56 DEBUG --- stdout --- 22:07:56 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 22515m 8804Mi am-767f6b67b5-qcqz2 23174m 8946Mi am-767f6b67b5-vbltz 24470m 8940Mi ds-cts-0 20141m 12322Mi ds-cts-1 19045m 12279Mi ds-cts-2 20377m 12476Mi ds-idrepo-0 9590m 3922Mi ds-idrepo-1 9527m 3966Mi ds-idrepo-2 9801m 3878Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-7dd48676f6-tdxl9 19m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-967d957f7-zk66v 11801m 1119Mi 22:07:56 DEBUG --- stderr --- 22:07:56 DEBUG 22:07:58 INFO 22:07:58 INFO [loop_until]: kubectl --namespace=xlou top node 22:07:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647278" 22:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647278" 22:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647278" 22:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647278" 22:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647278" 22:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713647278" 22:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647278" 22:07:58 INFO [http_cmd]: http status code OK 22:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647278" 22:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647278" 22:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647278" 22:07:58 DEBUG --- status code --- 22:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647278" 22:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647278" 22:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647278" 22:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647278" 22:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713647278" 22:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647278" 22:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647278" 22:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647278" 22:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647278" 22:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713647278" 22:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647278" 22:07:58 INFO [http_cmd]: http status code OK 22:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647278" 22:07:58 INFO [http_cmd]: http status code OK 22:07:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:07:58 INFO [http_cmd]: http status code OK 22:07:58 INFO [http_cmd]: http status code OK 22:07:58 INFO [http_cmd]: http status code OK 22:07:58 INFO [http_cmd]: http status code OK 22:07:58 INFO [loop_until]: OK (rc = 0) 22:07:58 INFO [http_cmd]: http status code OK 22:07:58 INFO [http_cmd]: http status code OK 22:07:58 INFO [http_cmd]: http status code OK 22:07:58 INFO [http_cmd]: http status code OK 22:07:58 INFO [http_cmd]: http status code OK 22:07:58 INFO [http_cmd]: http status code OK 22:07:58 INFO [http_cmd]: http status code OK 22:07:58 DEBUG --- status code --- 22:07:58 DEBUG --- status code --- 22:07:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:07:58 DEBUG --- http response --- 22:07:58 INFO [http_cmd]: http status code OK 22:07:58 INFO [http_cmd]: http status code OK 22:07:58 INFO [http_cmd]: http status code OK 22:07:58 DEBUG --- status code --- 22:07:58 DEBUG --- status code --- 22:07:58 DEBUG --- status code --- 22:07:58 DEBUG --- status code --- 22:07:58 DEBUG --- stdout --- 22:07:58 INFO [http_cmd]: http status code OK 22:07:58 INFO [http_cmd]: http status code OK 22:07:58 DEBUG --- status code --- 22:07:58 DEBUG --- status code --- 22:07:58 DEBUG --- status code --- 22:07:58 DEBUG --- status code --- 22:07:58 DEBUG --- status code --- 22:07:58 DEBUG --- status code --- 22:07:58 DEBUG --- status code --- 22:07:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:07:58 INFO [http_cmd]: http status code OK 22:07:58 DEBUG --- http response --- 22:07:58 INFO [http_cmd]: http status code OK 22:07:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:07:58 DEBUG --- status code --- 22:07:58 DEBUG --- status code --- 22:07:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:07:58 DEBUG --- status code --- 22:07:58 INFO [http_cmd]: http status code OK 22:07:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:07:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:07:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:07:58 DEBUG --- status code --- 22:07:58 DEBUG --- status code --- 22:07:58 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 61m 0% 6203Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 9943m 33% 7689Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 22353m 74% 11538Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 81m 0% 3366Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 19700m 65% 15556Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 152m 0% 5190Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 22558m 75% 11850Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19726m 66% 3064Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 24102m 80% 11190Mi 10% gke-xlou-cdm-ds-0f601751-j488 10370m 34% 7286Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19514m 65% 17354Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 10191m 34% 7935Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20512m 68% 15177Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 12057m 75% 9010Mi 15% 22:07:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:07:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:07:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:07:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:07:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:07:58 DEBUG --- http response --- 22:07:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:07:58 DEBUG --- http response --- 22:07:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:07:58 DEBUG ----- output ----- 22:07:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:07:58 DEBUG ----- output ----- 22:07:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:07:58 DEBUG --- http response --- 22:07:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:07:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:07:58 DEBUG --- status code --- 22:07:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:07:58 DEBUG --- http response --- 22:07:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:07:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:07:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:07:58 DEBUG --- http response --- 22:07:58 DEBUG --- http response --- 22:07:58 DEBUG --- http response --- 22:07:58 DEBUG --- http response --- 22:07:58 DEBUG --- http response --- 22:07:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:07:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:07:58 DEBUG --- http response --- 22:07:58 DEBUG --- status code --- 22:07:58 DEBUG { 22:07:58 DEBUG --- status code --- 22:07:58 DEBUG ----- output ----- 22:07:58 DEBUG { 22:07:58 DEBUG --- http response --- 22:07:58 DEBUG --- http response --- 22:07:58 DEBUG --- http response --- 22:07:58 DEBUG --- http response --- 22:07:58 DEBUG --- http response --- 22:07:58 DEBUG ----- output ----- 22:07:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:07:58 DEBUG --- http response --- 22:07:58 DEBUG --- http response --- 22:07:58 DEBUG --- stderr --- 22:07:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:07:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 22:07:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:07:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:07:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 22:07:58 DEBUG ----- output ----- 22:07:58 DEBUG --- http response --- 22:07:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:07:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:07:58 DEBUG "status": "success", 22:07:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:07:58 DEBUG { 22:07:58 DEBUG "status": "success", 22:07:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 22:07:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:07:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:07:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:07:58 DEBUG ----- output ----- 22:07:58 DEBUG { 22:07:58 DEBUG ----- output ----- 22:07:58 DEBUG { 22:07:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:07:58 DEBUG 22:07:58 DEBUG ----- output ----- 22:07:58 DEBUG ----- output ----- 22:07:58 DEBUG ----- output ----- 22:07:58 DEBUG ----- output ----- 22:07:58 DEBUG ----- output ----- 22:07:58 DEBUG { 22:07:58 DEBUG "status": "success", 22:07:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:07:58 DEBUG --- http response --- 22:07:58 DEBUG "data": { 22:07:58 DEBUG --- http response --- 22:07:58 DEBUG "status": "success", 22:07:58 DEBUG "data": { 22:07:58 DEBUG ----- output ----- 22:07:58 DEBUG ----- output ----- 22:07:58 DEBUG ----- output ----- 22:07:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:07:58 DEBUG { 22:07:58 DEBUG "status": "success", 22:07:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:07:58 DEBUG "status": "success", 22:07:58 DEBUG ----- output ----- 22:07:58 DEBUG { 22:07:58 DEBUG { 22:07:58 DEBUG { 22:07:58 DEBUG { 22:07:58 DEBUG { 22:07:58 DEBUG "data": { 22:07:58 DEBUG ----- output ----- 22:07:58 DEBUG ----- output ----- 22:07:58 DEBUG { 22:07:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:07:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:07:58 DEBUG "data": { 22:07:58 DEBUG "resultType": "vector", 22:07:58 DEBUG { 22:07:58 DEBUG { 22:07:58 DEBUG { 22:07:58 DEBUG ----- output ----- 22:07:58 DEBUG "status": "success", 22:07:58 DEBUG "data": { 22:07:58 DEBUG ----- output ----- 22:07:58 DEBUG "data": { 22:07:58 DEBUG { 22:07:58 DEBUG "status": "success", 22:07:58 DEBUG "status": "success", 22:07:58 DEBUG "status": "success", 22:07:58 DEBUG "data": { 22:07:58 DEBUG "status": "success", 22:07:58 DEBUG "resultType": "vector", 22:07:58 DEBUG { 22:07:58 DEBUG "status": "success", 22:07:58 DEBUG "resultType": "vector", 22:07:58 DEBUG ----- output ----- 22:07:58 DEBUG ----- output ----- 22:07:58 DEBUG "resultType": "vector", 22:07:58 DEBUG "result": [ 22:07:58 DEBUG "status": "success", 22:07:58 DEBUG "status": "success", 22:07:58 DEBUG "status": "success", 22:07:58 DEBUG { 22:07:58 DEBUG "data": { 22:07:58 DEBUG "resultType": "vector", 22:07:58 DEBUG { 22:07:58 DEBUG "resultType": "vector", 22:07:58 DEBUG "status": "success", 22:07:58 DEBUG "data": { 22:07:58 DEBUG "data": { 22:07:58 DEBUG "resultType": "vector", 22:07:58 DEBUG "status": "success", 22:07:58 DEBUG "data": { 22:07:58 DEBUG "result": [ 22:07:58 DEBUG "data": { 22:07:58 DEBUG "result": [ 22:07:58 DEBUG { 22:07:58 DEBUG "status": "success", 22:07:58 DEBUG { 22:07:58 DEBUG "status": "success", 22:07:58 DEBUG { 22:07:58 DEBUG "data": { 22:07:58 DEBUG "data": { 22:07:58 DEBUG "data": { 22:07:58 DEBUG "status": "success", 22:07:58 DEBUG "resultType": "vector", 22:07:58 DEBUG "result": [ 22:07:58 DEBUG "status": "success", 22:07:58 DEBUG "result": [ 22:07:58 DEBUG "data": { 22:07:58 DEBUG "resultType": "vector", 22:07:58 DEBUG "resultType": "vector", 22:07:58 DEBUG "result": [ 22:07:58 DEBUG "resultType": "vector", 22:07:58 DEBUG "data": { 22:07:58 DEBUG { 22:07:58 DEBUG "resultType": "vector", 22:07:58 DEBUG { 22:07:58 DEBUG "status": "success", 22:07:58 DEBUG "data": { 22:07:58 DEBUG "data": { 22:07:58 DEBUG "result": [ 22:07:58 DEBUG "metric": { 22:07:58 DEBUG "resultType": "vector", 22:07:58 DEBUG "resultType": "vector", 22:07:58 DEBUG "result": [ 22:07:58 DEBUG "data": { 22:07:58 DEBUG "result": [ 22:07:58 DEBUG { 22:07:58 DEBUG "data": { 22:07:58 DEBUG { 22:07:58 DEBUG "resultType": "vector", 22:07:58 DEBUG "result": [ 22:07:58 DEBUG "result": [ 22:07:58 DEBUG { 22:07:58 DEBUG "metric": { 22:07:58 DEBUG "resultType": "vector", 22:07:58 DEBUG "metric": { 22:07:58 DEBUG "result": [ 22:07:58 DEBUG "metric": { 22:07:58 DEBUG "data": { 22:07:58 DEBUG "resultType": "vector", 22:07:58 DEBUG "resultType": "vector", 22:07:58 DEBUG { 22:07:58 DEBUG "pod": "ds-cts-0" 22:07:58 DEBUG "result": [ 22:07:58 DEBUG { 22:07:58 DEBUG "resultType": "vector", 22:07:58 DEBUG "resultType": "vector", 22:07:58 DEBUG { 22:07:58 DEBUG "metric": { 22:07:58 DEBUG "resultType": "vector", 22:07:58 DEBUG "metric": { 22:07:58 DEBUG "pod": "ds-cts-0" 22:07:58 DEBUG { 22:07:58 DEBUG { 22:07:58 DEBUG "result": [ 22:07:58 DEBUG "instance": "10.142.0.107:9100" 22:07:58 DEBUG "result": [ 22:07:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:07:58 DEBUG { 22:07:58 DEBUG "pod": "am-767f6b67b5-9zw22" 22:07:58 DEBUG "resultType": "vector", 22:07:58 DEBUG "result": [ 22:07:58 DEBUG "result": [ 22:07:58 DEBUG "metric": { 22:07:58 DEBUG }, 22:07:58 DEBUG { 22:07:58 DEBUG "metric": { 22:07:58 DEBUG "result": [ 22:07:58 DEBUG "result": [ 22:07:58 DEBUG "metric": { 22:07:58 DEBUG "pod": "ds-idrepo-0" 22:07:58 DEBUG "result": [ 22:07:58 DEBUG }, 22:07:58 DEBUG "metric": { 22:07:58 DEBUG "result": [ 22:07:58 DEBUG "metric": { 22:07:58 DEBUG { 22:07:58 DEBUG }, 22:07:58 DEBUG { 22:07:58 DEBUG }, 22:07:58 DEBUG "metric": { 22:07:58 DEBUG }, 22:07:58 DEBUG "value": [ 22:07:58 DEBUG { 22:07:58 DEBUG { 22:07:58 DEBUG "pod": "ds-idrepo-0" 22:07:58 DEBUG "value": [ 22:07:58 DEBUG "metric": { 22:07:58 DEBUG "pod": "am-767f6b67b5-9zw22" 22:07:58 DEBUG { 22:07:58 DEBUG { 22:07:58 DEBUG "pod": "am-767f6b67b5-9zw22" 22:07:58 DEBUG }, 22:07:58 DEBUG { 22:07:58 DEBUG "value": [ 22:07:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:07:58 DEBUG { 22:07:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:07:58 DEBUG "metric": { 22:07:58 DEBUG "value": [ 22:07:58 DEBUG "metric": { 22:07:58 DEBUG "value": [ 22:07:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:07:58 DEBUG "result": [ 22:07:58 DEBUG 22:07:58 DEBUG "metric": { 22:07:58 DEBUG "metric": { 22:07:58 DEBUG }, 22:07:58 DEBUG 22:07:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:07:58 DEBUG }, 22:07:58 DEBUG "metric": { 22:07:58 DEBUG "metric": { 22:07:58 DEBUG }, 22:07:58 DEBUG "value": [ 22:07:58 DEBUG "metric": { 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG "metric": { 22:07:58 DEBUG }, 22:07:58 DEBUG "pod": "am-767f6b67b5-9zw22" 22:07:58 DEBUG 22:07:58 DEBUG "instance": "10.142.0.107:9100" 22:07:58 DEBUG 22:07:58 DEBUG }, 22:07:58 DEBUG { 22:07:58 DEBUG 22:07:58 DEBUG "pod": "ds-idrepo-0" 22:07:58 DEBUG "instance": "10.142.0.107:9100" 22:07:58 DEBUG "value": [ 22:07:58 DEBUG 22:07:58 DEBUG }, 22:07:58 DEBUG "value": [ 22:07:58 DEBUG "pod": "am-767f6b67b5-9zw22" 22:07:58 DEBUG "pod": "ds-cts-0" 22:07:58 DEBUG "value": [ 22:07:58 DEBUG 22:07:58 DEBUG "instance": "10.142.0.107:9100" 22:07:58 DEBUG 22:07:58 DEBUG }, 22:07:58 DEBUG "pod": "am-767f6b67b5-9zw22" 22:07:58 DEBUG "value": [ 22:07:58 DEBUG }, 22:07:58 DEBUG 22:07:58 DEBUG }, 22:07:58 DEBUG 22:07:58 DEBUG "value": [ 22:07:58 DEBUG "metric": { 22:07:58 DEBUG 22:07:58 DEBUG }, 22:07:58 DEBUG }, 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG "value": [ 22:07:58 DEBUG 22:07:58 DEBUG }, 22:07:58 DEBUG }, 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG }, 22:07:58 DEBUG [...] 22:07:58 DEBUG "value": [ 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG "value": [ 22:07:58 DEBUG 22:07:58 DEBUG "value": [ 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG "instance": "10.142.0.107:9100" 22:07:58 DEBUG [...] 22:07:58 DEBUG "value": [ 22:07:58 DEBUG "value": [ 22:07:58 DEBUG 22:07:58 DEBUG [...] 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG "value": [ 22:07:58 DEBUG "value": [ 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG "value": [ 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG }, 22:07:58 DEBUG 22:07:58 DEBUG [...] 22:07:58 DEBUG 22:07:58 DEBUG [...] 22:07:58 DEBUG 22:07:58 DEBUG }, 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG [...] 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG [...] 22:07:58 DEBUG 22:07:58 DEBUG "value": [ 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG "value": [ 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG [...] 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG [...] 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG [...] 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG [...] 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG [...] 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG "pod": "ds-cts-2" 22:07:58 DEBUG 22:07:58 DEBUG [...] 22:07:58 DEBUG 22:07:58 DEBUG [...] 22:07:58 DEBUG 22:07:58 DEBUG "instance": "10.142.0.43:9100" 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG [...] 22:07:58 DEBUG [...] 22:07:58 DEBUG 22:07:58 DEBUG "pod": "ds-cts-2" 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG [...] 22:07:58 DEBUG "pod": "am-767f6b67b5-vbltz" 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG [...] 22:07:58 DEBUG }, 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG [...] 22:07:58 DEBUG }, 22:07:58 DEBUG "pod": "overseer-0-967d957f7-zk66v" 22:07:58 DEBUG 22:07:58 DEBUG [...] 22:07:58 DEBUG "pod": "am-767f6b67b5-vbltz" 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG }, 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG }, 22:07:58 DEBUG 22:07:58 DEBUG "pod": "ds-idrepo-2" 22:07:58 DEBUG 22:07:58 DEBUG "value": [ 22:07:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:07:58 DEBUG 22:07:58 DEBUG [...] 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG "value": [ 22:07:58 DEBUG }, 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG }, 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG "instance": "10.142.0.43:9100" 22:07:58 DEBUG "value": [ 22:07:58 DEBUG [...] 22:07:58 DEBUG 22:07:58 DEBUG "value": [ 22:07:58 DEBUG "pod": "am-767f6b67b5-vbltz" 22:07:58 DEBUG }, 22:07:58 DEBUG 22:07:58 DEBUG 1713647278, 22:07:58 DEBUG }, 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG 1713647278, 22:07:58 DEBUG "value": [ 22:07:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:07:58 DEBUG 22:07:58 DEBUG "value": [ 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG }, 22:07:58 DEBUG 1713647278, 22:07:58 DEBUG 22:07:58 DEBUG 22:07:58 DEBUG 1713647278, 22:07:58 DEBUG }, 22:07:58 DEBUG "value": [ 22:07:58 DEBUG 22:07:58 DEBUG "instance": "10.142.0.43:9100" 22:07:58 DEBUG "value": [ 22:07:58 DEBUG "pod": "ds-idrepo-2" 22:07:58 DEBUG "pod": "overseer-0-967d957f7-zk66v" 22:07:58 DEBUG 22:07:58 DEBUG "pod": "am-767f6b67b5-vbltz" 22:07:58 DEBUG 22:07:58 DEBUG "0.4080666666667734" 22:07:58 DEBUG 1713647278, 22:07:58 DEBUG }, 22:07:58 DEBUG 22:07:58 DEBUG 1713647278, 22:07:58 DEBUG "pod": "ds-idrepo-2" 22:07:58 DEBUG "pod": "overseer-0-967d957f7-zk66v" 22:07:58 DEBUG "value": [ 22:07:58 DEBUG "0.5674333333333361" 22:07:58 DEBUG 22:07:58 DEBUG "pod": "ds-cts-2" 22:07:58 DEBUG "3328.866666666667" 22:07:58 DEBUG 1713647278, 22:07:58 DEBUG "value": [ 22:07:58 DEBUG "30.333333333333332" 22:07:58 DEBUG }, 22:07:58 DEBUG 1713647278, 22:07:58 DEBUG }, 22:07:58 DEBUG "value": [ 22:07:58 DEBUG 22:07:58 DEBUG "pod": "am-767f6b67b5-vbltz" 22:07:58 DEBUG "instance": "10.142.0.43:9100" 22:07:58 DEBUG ] 22:07:58 DEBUG "0" 22:07:58 DEBUG ] 22:07:58 DEBUG "instance": "10.142.0.43:9100" 22:07:58 DEBUG "3707.5666666666666" 22:07:58 DEBUG }, 22:07:58 DEBUG "value": [ 22:07:58 DEBUG 1713647278, 22:07:58 DEBUG "4070387977216" 22:07:58 DEBUG 22:07:58 DEBUG }, 22:07:58 DEBUG ] 22:07:58 DEBUG "64.0575" 22:07:58 DEBUG 1713647278, 22:07:58 DEBUG ] 22:07:58 DEBUG "value": [ 22:07:58 DEBUG "11.852521561239142" 22:07:58 DEBUG 1713647278, 22:07:58 DEBUG }, 22:07:58 DEBUG }, 22:07:58 DEBUG }, 22:07:58 DEBUG }, 22:07:58 DEBUG } 22:07:58 DEBUG "value": [ 22:07:58 DEBUG } 22:07:58 DEBUG }, 22:07:58 DEBUG ] 22:07:58 DEBUG }, 22:07:58 DEBUG 1713647278, 22:07:58 DEBUG ] 22:07:58 DEBUG ] 22:07:58 DEBUG "pod": "am-767f6b67b5-vbltz" 22:07:58 DEBUG "value": [ 22:07:58 DEBUG } 22:07:58 DEBUG ] 22:07:58 DEBUG "0" 22:07:58 DEBUG } 22:07:58 DEBUG 1713647278, 22:07:58 DEBUG ] 22:07:58 DEBUG "0" 22:07:58 DEBUG "value": [ 22:07:58 DEBUG "value": [ 22:07:58 DEBUG "value": [ 22:07:58 DEBUG "value": [ 22:07:58 DEBUG ] 22:07:58 DEBUG 1713647278, 22:07:58 DEBUG ] 22:07:58 DEBUG "value": [ 22:07:58 DEBUG } 22:07:58 DEBUG "value": [ 22:07:58 DEBUG "0" 22:07:58 DEBUG } 22:07:58 DEBUG } 22:07:58 DEBUG }, 22:07:58 DEBUG 1713647278, 22:07:58 DEBUG ] 22:07:58 DEBUG } 22:07:58 DEBUG ] 22:07:58 DEBUG ] 22:07:58 DEBUG "101116559.30000001" 22:07:58 DEBUG } 22:07:58 DEBUG ] 22:07:58 DEBUG 1713647278, 22:07:58 DEBUG 1713647278, 22:07:58 DEBUG 1713647278, 22:07:58 DEBUG 1713647278, 22:07:58 DEBUG } 22:07:58 DEBUG "1268973568" 22:07:58 DEBUG } 22:07:58 DEBUG 1713647278, 22:07:58 DEBUG ] 22:07:58 DEBUG 1713647278, 22:07:58 DEBUG ] 22:07:58 DEBUG } 22:07:58 DEBUG ] 22:07:58 DEBUG "value": [ 22:07:58 DEBUG "0" 22:07:58 DEBUG } 22:07:58 DEBUG ] 22:07:58 DEBUG } 22:07:58 DEBUG } 22:07:58 DEBUG ] 22:07:58 DEBUG ] 22:07:58 DEBUG } 22:07:58 DEBUG "0" 22:07:58 DEBUG "3340.6" 22:07:58 DEBUG "0.08766666666665704" 22:07:58 DEBUG "11.812836255288056" 22:07:58 DEBUG } 22:07:58 DEBUG ] 22:07:58 DEBUG } 22:07:58 DEBUG "101121061.53333333" 22:07:58 DEBUG } 22:07:58 DEBUG "0" 22:07:58 DEBUG ] 22:07:58 DEBUG ] 22:07:58 DEBUG } 22:07:58 DEBUG 1713647278, 22:07:58 DEBUG ] 22:07:58 DEBUG } 22:07:58 DEBUG } 22:07:58 DEBUG } 22:07:58 DEBUG } 22:07:58 DEBUG } 22:07:58 DEBUG ] 22:07:58 DEBUG ] 22:07:58 DEBUG ] 22:07:58 DEBUG ] 22:07:58 DEBUG ] 22:07:58 DEBUG } 22:07:58 DEBUG ] 22:07:58 DEBUG } 22:07:58 DEBUG ] 22:07:58 DEBUG } 22:07:58 DEBUG } 22:07:58 DEBUG } 22:07:58 DEBUG "600303.5" 22:07:58 DEBUG } 22:07:58 DEBUG ] 22:07:58 DEBUG } 22:07:58 DEBUG ] 22:07:58 DEBUG } 22:07:58 DEBUG } 22:07:58 DEBUG } 22:07:58 DEBUG } 22:07:58 DEBUG } 22:07:58 DEBUG } 22:07:58 DEBUG ] 22:07:58 DEBUG } 22:07:58 DEBUG } 22:07:58 DEBUG } 22:07:58 DEBUG } 22:07:58 DEBUG ] 22:07:58 DEBUG ] 22:07:58 DEBUG } 22:07:58 DEBUG } 22:07:58 DEBUG } 22:07:58 DEBUG ] 22:07:58 DEBUG ] 22:07:58 DEBUG ] 22:07:58 DEBUG } 22:07:58 DEBUG } 22:07:58 DEBUG ] 22:07:58 DEBUG ] 22:07:58 DEBUG } 22:07:58 DEBUG } 22:07:58 DEBUG } 22:07:58 DEBUG } 22:07:58 DEBUG } 22:07:58 DEBUG } 22:07:58 DEBUG ] 22:07:58 DEBUG } 22:07:58 DEBUG } 22:07:58 DEBUG } 22:07:58 DEBUG ] 22:07:58 DEBUG } 22:07:58 DEBUG } 22:07:58 DEBUG } 22:07:58 DEBUG } 22:07:58 DEBUG } 22:07:58 DEBUG } 22:07:58 DEBUG } 22:07:58 DEBUG } 22:07:58 DEBUG } 22:07:58 DEBUG } 22:08:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713647334" 22:08:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647334" 22:08:54 ERROR [http_cmd]: ERROR 22:08:54 ERROR --- status code --- 22:08:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:08:54 ERROR --- http response --- 22:08:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:08:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:08:54 INFO [http_cmd]: http status code OK 22:08:54 DEBUG --- status code --- 22:08:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:08:54 DEBUG --- http response --- 22:08:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:08:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647334" 22:08:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713647334" 22:08:54 INFO [http_cmd]: http status code OK 22:08:54 DEBUG --- status code --- 22:08:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:08:54 DEBUG --- http response --- 22:08:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 22:08:54 DEBUG ----- output ----- 22:08:54 DEBUG { 22:08:54 DEBUG "status": "success", 22:08:54 DEBUG "data": { 22:08:54 DEBUG "resultType": "vector", 22:08:54 DEBUG "result": [ 22:08:54 DEBUG { 22:08:54 DEBUG "metric": { 22:08:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:08:54 DEBUG }, 22:08:54 DEBUG "value": [ 22:08:54 DEBUG 22:08:54 DEBUG 22:08:54 DEBUG 22:08:54 DEBUG [...] 22:08:54 DEBUG 22:08:54 DEBUG 22:08:54 DEBUG 22:08:54 DEBUG "pod": "ds-cts-0" 22:08:54 DEBUG }, 22:08:54 DEBUG "value": [ 22:08:54 DEBUG 1713647334, 22:08:54 DEBUG "0" 22:08:54 DEBUG ] 22:08:54 DEBUG } 22:08:54 DEBUG ] 22:08:54 DEBUG } 22:08:54 DEBUG } 22:08:54 INFO [http_cmd]: http status code OK 22:08:54 DEBUG --- status code --- 22:08:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:08:54 DEBUG --- http response --- 22:08:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:08:54 DEBUG ----- output ----- 22:08:54 DEBUG { 22:08:54 DEBUG "status": "success", 22:08:54 DEBUG "data": { 22:08:54 DEBUG "resultType": "vector", 22:08:54 DEBUG "result": [ 22:08:54 DEBUG { 22:08:54 DEBUG "metric": { 22:08:54 DEBUG "pod": "ds-idrepo-0" 22:08:54 DEBUG }, 22:08:54 DEBUG "value": [ 22:08:54 DEBUG 22:08:54 DEBUG 22:08:54 DEBUG 22:08:54 DEBUG [...] 22:08:54 DEBUG 22:08:54 DEBUG 22:08:54 DEBUG 22:08:54 DEBUG "pod": "ds-cts-2" 22:08:54 DEBUG }, 22:08:54 DEBUG "value": [ 22:08:54 DEBUG 1713647334, 22:08:54 DEBUG "0" 22:08:54 DEBUG ] 22:08:54 DEBUG } 22:08:54 DEBUG ] 22:08:54 DEBUG } 22:08:54 DEBUG } 22:08:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647336" 22:08:56 INFO [http_cmd]: http status code OK 22:08:56 DEBUG --- status code --- 22:08:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:08:56 DEBUG --- http response --- 22:08:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:08:56 DEBUG ----- output ----- 22:08:56 DEBUG { 22:08:56 DEBUG "status": "success", 22:08:56 DEBUG "data": { 22:08:56 DEBUG "resultType": "vector", 22:08:56 DEBUG "result": [ 22:08:56 DEBUG { 22:08:56 DEBUG "metric": { 22:08:56 DEBUG "pod": "am-767f6b67b5-9zw22" 22:08:56 DEBUG }, 22:08:56 DEBUG "value": [ 22:08:56 DEBUG 22:08:56 DEBUG 22:08:56 DEBUG 22:08:56 DEBUG [...] 22:08:56 DEBUG 22:08:56 DEBUG 22:08:56 DEBUG 22:08:56 DEBUG "pod": "am-767f6b67b5-vbltz" 22:08:56 DEBUG }, 22:08:56 DEBUG "value": [ 22:08:56 DEBUG 1713647336, 22:08:56 DEBUG "3338" 22:08:56 DEBUG ] 22:08:56 DEBUG } 22:08:56 DEBUG ] 22:08:56 DEBUG } 22:08:56 DEBUG } 22:08:56 INFO 22:08:56 INFO [loop_until]: kubectl --namespace=xlou top pods 22:08:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:08:56 INFO [loop_until]: OK (rc = 0) 22:08:56 DEBUG --- stdout --- 22:08:56 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 22700m 8807Mi am-767f6b67b5-qcqz2 22356m 8943Mi am-767f6b67b5-vbltz 24495m 8944Mi ds-cts-0 20181m 12321Mi ds-cts-1 19237m 12283Mi ds-cts-2 20291m 12481Mi ds-idrepo-0 9237m 3921Mi ds-idrepo-1 9466m 3966Mi ds-idrepo-2 9472m 3878Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-7dd48676f6-tdxl9 17m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-967d957f7-zk66v 11762m 1120Mi 22:08:56 DEBUG --- stderr --- 22:08:56 DEBUG 22:08:58 INFO 22:08:58 INFO [loop_until]: kubectl --namespace=xlou top node 22:08:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:08:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713647338" 22:08:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647338" 22:08:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647338" 22:08:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647338" 22:08:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647338" 22:08:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647338" 22:08:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647338" 22:08:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713647338" 22:08:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647338" 22:08:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647338" 22:08:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647338" 22:08:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647338" 22:08:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647338" 22:08:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647338" 22:08:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647338" 22:08:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647338" 22:08:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647338" 22:08:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713647338" 22:08:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647338" 22:08:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647338" 22:08:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647338" 22:08:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647338" 22:08:58 INFO [http_cmd]: http status code OK 22:08:58 INFO [http_cmd]: http status code OK 22:08:58 INFO [http_cmd]: http status code OK 22:08:58 DEBUG --- status code --- 22:08:58 INFO [http_cmd]: http status code OK 22:08:58 INFO [http_cmd]: http status code OK 22:08:58 INFO [http_cmd]: http status code OK 22:08:58 INFO [http_cmd]: http status code OK 22:08:58 INFO [http_cmd]: http status code OK 22:08:58 INFO [http_cmd]: http status code OK 22:08:58 INFO [http_cmd]: http status code OK 22:08:58 INFO [http_cmd]: http status code OK 22:08:58 INFO [http_cmd]: http status code OK 22:08:58 DEBUG --- status code --- 22:08:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:08:58 DEBUG --- http response --- 22:08:58 INFO [http_cmd]: http status code OK 22:08:58 DEBUG --- status code --- 22:08:58 INFO [http_cmd]: http status code OK 22:08:58 DEBUG --- status code --- 22:08:58 INFO [loop_until]: OK (rc = 0) 22:08:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:08:58 INFO [http_cmd]: http status code OK 22:08:58 DEBUG --- status code --- 22:08:58 DEBUG --- status code --- 22:08:58 DEBUG --- status code --- 22:08:58 DEBUG --- status code --- 22:08:58 INFO [http_cmd]: http status code OK 22:08:58 INFO [http_cmd]: http status code OK 22:08:58 DEBUG --- status code --- 22:08:58 DEBUG --- status code --- 22:08:58 DEBUG --- status code --- 22:08:58 DEBUG --- status code --- 22:08:58 INFO [http_cmd]: http status code OK 22:08:58 INFO [http_cmd]: http status code OK 22:08:58 INFO [http_cmd]: http status code OK 22:08:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:08:58 DEBUG --- status code --- 22:08:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:08:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:08:58 DEBUG --- status code --- 22:08:58 DEBUG --- stdout --- 22:08:58 DEBUG --- http response --- 22:08:58 DEBUG --- status code --- 22:08:58 INFO [http_cmd]: http status code OK 22:08:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:08:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:08:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:08:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:08:58 DEBUG --- status code --- 22:08:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:08:58 DEBUG --- status code --- 22:08:58 INFO [http_cmd]: http status code OK 22:08:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:08:58 DEBUG --- http response --- 22:08:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:08:58 DEBUG --- status code --- 22:08:58 DEBUG --- status code --- 22:08:58 DEBUG --- status code --- 22:08:58 DEBUG ----- output ----- 22:08:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:08:58 DEBUG --- http response --- 22:08:58 DEBUG --- http response --- 22:08:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:08:58 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 59m 0% 6214Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 9945m 33% 7680Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 21666m 72% 11545Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 86m 0% 3349Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 20373m 68% 15562Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 152m 0% 5182Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 22824m 76% 11852Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19752m 66% 3070Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 25029m 83% 11197Mi 10% gke-xlou-cdm-ds-0f601751-j488 10294m 34% 7283Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19373m 64% 17363Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 10146m 33% 7941Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20497m 68% 15178Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 12105m 76% 9023Mi 15% 22:08:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:08:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:08:58 DEBUG --- status code --- 22:08:58 DEBUG --- http response --- 22:08:58 DEBUG --- http response --- 22:08:58 DEBUG --- http response --- 22:08:58 DEBUG --- http response --- 22:08:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:08:58 DEBUG --- http response --- 22:08:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:08:58 DEBUG --- status code --- 22:08:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:08:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:08:58 DEBUG --- http response --- 22:08:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:08:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:08:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:08:58 DEBUG { 22:08:58 DEBUG --- http response --- 22:08:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:08:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:08:58 DEBUG --- http response --- 22:08:58 DEBUG --- stderr --- 22:08:58 DEBUG ----- output ----- 22:08:58 DEBUG --- http response --- 22:08:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:08:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:08:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:08:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:08:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:08:58 DEBUG --- http response --- 22:08:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 22:08:58 DEBUG --- http response --- 22:08:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:08:58 DEBUG ----- output ----- 22:08:58 DEBUG --- http response --- 22:08:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:08:58 DEBUG --- http response --- 22:08:58 DEBUG --- http response --- 22:08:58 DEBUG --- http response --- 22:08:58 DEBUG "status": "success", 22:08:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:08:58 DEBUG ----- output ----- 22:08:58 DEBUG ----- output ----- 22:08:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:08:58 DEBUG 22:08:58 DEBUG { 22:08:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:08:58 DEBUG --- http response --- 22:08:58 DEBUG ----- output ----- 22:08:58 DEBUG ----- output ----- 22:08:58 DEBUG ----- output ----- 22:08:58 DEBUG ----- output ----- 22:08:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:08:58 DEBUG ----- output ----- 22:08:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:08:58 DEBUG --- http response --- 22:08:58 DEBUG { 22:08:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:08:58 DEBUG ----- output ----- 22:08:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:08:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:08:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:08:58 DEBUG "data": { 22:08:58 DEBUG ----- output ----- 22:08:58 DEBUG { 22:08:58 DEBUG { 22:08:58 DEBUG "status": "success", 22:08:58 DEBUG "status": "success", 22:08:58 DEBUG ----- output ----- 22:08:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:08:58 DEBUG { 22:08:58 DEBUG { 22:08:58 DEBUG { 22:08:58 DEBUG { 22:08:58 DEBUG "status": "success", 22:08:58 DEBUG { 22:08:58 DEBUG ----- output ----- 22:08:58 DEBUG { 22:08:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:08:58 DEBUG ----- output ----- 22:08:58 DEBUG { 22:08:58 DEBUG ----- output ----- 22:08:58 DEBUG ----- output ----- 22:08:58 DEBUG ----- output ----- 22:08:58 DEBUG "resultType": "vector", 22:08:58 DEBUG { 22:08:58 DEBUG "status": "success", 22:08:58 DEBUG "data": { 22:08:58 DEBUG ----- output ----- 22:08:58 DEBUG "data": { 22:08:58 DEBUG { 22:08:58 DEBUG ----- output ----- 22:08:58 DEBUG "status": "success", 22:08:58 DEBUG "status": "success", 22:08:58 DEBUG "status": "success", 22:08:58 DEBUG "data": { 22:08:58 DEBUG "status": "success", 22:08:58 DEBUG "status": "success", 22:08:58 DEBUG "status": "success", 22:08:58 DEBUG ----- output ----- 22:08:58 DEBUG { 22:08:58 DEBUG "status": "success", 22:08:58 DEBUG { 22:08:58 DEBUG { 22:08:58 DEBUG { 22:08:58 DEBUG "result": [ 22:08:58 DEBUG ----- output ----- 22:08:58 DEBUG "status": "success", 22:08:58 DEBUG "data": { 22:08:58 DEBUG "resultType": "vector", 22:08:58 DEBUG { 22:08:58 DEBUG "resultType": "vector", 22:08:58 DEBUG "status": "success", 22:08:58 DEBUG { 22:08:58 DEBUG "data": { 22:08:58 DEBUG "data": { 22:08:58 DEBUG "data": { 22:08:58 DEBUG "resultType": "vector", 22:08:58 DEBUG "data": { 22:08:58 DEBUG "data": { 22:08:58 DEBUG "data": { 22:08:58 DEBUG { 22:08:58 DEBUG "status": "success", 22:08:58 DEBUG "data": { 22:08:58 DEBUG "status": "success", 22:08:58 DEBUG "status": "success", 22:08:58 DEBUG "status": "success", 22:08:58 DEBUG { 22:08:58 DEBUG { 22:08:58 DEBUG "data": { 22:08:58 DEBUG "resultType": "vector", 22:08:58 DEBUG "result": [ 22:08:58 DEBUG "status": "success", 22:08:58 DEBUG "result": [ 22:08:58 DEBUG "data": { 22:08:58 DEBUG "status": "success", 22:08:58 DEBUG "resultType": "vector", 22:08:58 DEBUG "resultType": "vector", 22:08:58 DEBUG "resultType": "vector", 22:08:58 DEBUG "result": [ 22:08:58 DEBUG "resultType": "vector", 22:08:58 DEBUG "resultType": "vector", 22:08:58 DEBUG "resultType": "vector", 22:08:58 DEBUG "status": "success", 22:08:58 DEBUG "data": { 22:08:58 DEBUG "resultType": "vector", 22:08:58 DEBUG "data": { 22:08:58 DEBUG "data": { 22:08:58 DEBUG "data": { 22:08:58 DEBUG "metric": { 22:08:58 DEBUG "status": "success", 22:08:58 DEBUG "resultType": "vector", 22:08:58 DEBUG "result": [ 22:08:58 DEBUG { 22:08:58 DEBUG "data": { 22:08:58 DEBUG { 22:08:58 DEBUG "resultType": "vector", 22:08:58 DEBUG "data": { 22:08:58 DEBUG "result": [ 22:08:58 DEBUG "result": [ 22:08:58 DEBUG "result": [ 22:08:58 DEBUG { 22:08:58 DEBUG "result": [ 22:08:58 DEBUG "result": [ 22:08:58 DEBUG "result": [ 22:08:58 DEBUG "data": { 22:08:58 DEBUG "resultType": "vector", 22:08:58 DEBUG "result": [ 22:08:58 DEBUG "resultType": "vector", 22:08:58 DEBUG "resultType": "vector", 22:08:58 DEBUG "resultType": "vector", 22:08:58 DEBUG "pod": "ds-idrepo-0" 22:08:58 DEBUG "data": { 22:08:58 DEBUG "result": [ 22:08:58 DEBUG { 22:08:58 DEBUG "metric": { 22:08:58 DEBUG "resultType": "vector", 22:08:58 DEBUG "metric": { 22:08:58 DEBUG "result": [ 22:08:58 DEBUG "resultType": "vector", 22:08:58 DEBUG { 22:08:58 DEBUG { 22:08:58 DEBUG { 22:08:58 DEBUG "metric": { 22:08:58 DEBUG { 22:08:58 DEBUG { 22:08:58 DEBUG { 22:08:58 DEBUG "resultType": "vector", 22:08:58 DEBUG "result": [ 22:08:58 DEBUG { 22:08:58 DEBUG "result": [ 22:08:58 DEBUG { 22:08:58 DEBUG "result": [ 22:08:58 DEBUG }, 22:08:58 DEBUG "resultType": "vector", 22:08:58 DEBUG { 22:08:58 DEBUG "metric": { 22:08:58 DEBUG "pod": "am-767f6b67b5-9zw22" 22:08:58 DEBUG "result": [ 22:08:58 DEBUG "pod": "ds-idrepo-1" 22:08:58 DEBUG { 22:08:58 DEBUG "result": [ 22:08:58 DEBUG "metric": { 22:08:58 DEBUG "metric": { 22:08:58 DEBUG "metric": { 22:08:58 DEBUG "pod": "ds-cts-0" 22:08:58 DEBUG "metric": { 22:08:58 DEBUG "metric": { 22:08:58 DEBUG "metric": { 22:08:58 DEBUG "result": [ 22:08:58 DEBUG { 22:08:58 DEBUG "metric": { 22:08:58 DEBUG "metric": { 22:08:58 DEBUG "result": [ 22:08:58 DEBUG { 22:08:58 DEBUG "value": [ 22:08:58 DEBUG "result": [ 22:08:58 DEBUG "metric": { 22:08:58 DEBUG "pod": "am-767f6b67b5-9zw22" 22:08:58 DEBUG }, 22:08:58 DEBUG { 22:08:58 DEBUG }, 22:08:58 DEBUG "metric": { 22:08:58 DEBUG { 22:08:58 DEBUG "pod": "am-767f6b67b5-9zw22" 22:08:58 DEBUG "pod": "am-767f6b67b5-9zw22" 22:08:58 DEBUG "instance": "10.142.0.107:9100" 22:08:58 DEBUG }, 22:08:58 DEBUG "pod": "ds-idrepo-1" 22:08:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:08:58 DEBUG }, 22:08:58 DEBUG "value": [ 22:08:58 DEBUG "metric": { 22:08:58 DEBUG "pod": "ds-cts-0" 22:08:58 DEBUG "pod": "ds-idrepo-0" 22:08:58 DEBUG { 22:08:58 DEBUG "metric": { 22:08:58 DEBUG 22:08:58 DEBUG { 22:08:58 DEBUG "instance": "10.142.0.107:9100" 22:08:58 DEBUG }, 22:08:58 DEBUG "value": [ 22:08:58 DEBUG "metric": { 22:08:58 DEBUG "value": [ 22:08:58 DEBUG "instance": "10.142.0.107:9100" 22:08:58 DEBUG "metric": { 22:08:58 DEBUG }, 22:08:58 DEBUG }, 22:08:58 DEBUG }, 22:08:58 DEBUG "value": [ 22:08:58 DEBUG }, 22:08:58 DEBUG "instance": "10.142.0.107:9100" 22:08:58 DEBUG 22:08:58 DEBUG { 22:08:58 DEBUG "pod": "ds-idrepo-0" 22:08:58 DEBUG }, 22:08:58 DEBUG }, 22:08:58 DEBUG "metric": { 22:08:58 DEBUG "pod": "ds-cts-0" 22:08:58 DEBUG 22:08:58 DEBUG "metric": { 22:08:58 DEBUG }, 22:08:58 DEBUG "value": [ 22:08:58 DEBUG 22:08:58 DEBUG "instance": "10.142.0.107:9100" 22:08:58 DEBUG 22:08:58 DEBUG }, 22:08:58 DEBUG "pod": "ds-idrepo-1" 22:08:58 DEBUG "value": [ 22:08:58 DEBUG "value": [ 22:08:58 DEBUG "value": [ 22:08:58 DEBUG 22:08:58 DEBUG "value": [ 22:08:58 DEBUG }, 22:08:58 DEBUG 22:08:58 DEBUG "metric": { 22:08:58 DEBUG }, 22:08:58 DEBUG "value": [ 22:08:58 DEBUG "value": [ 22:08:58 DEBUG "pod": "am-767f6b67b5-9zw22" 22:08:58 DEBUG }, 22:08:58 DEBUG 22:08:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:08:58 DEBUG "value": [ 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG }, 22:08:58 DEBUG 22:08:58 DEBUG "value": [ 22:08:58 DEBUG }, 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG "value": [ 22:08:58 DEBUG 22:08:58 DEBUG "pod": "am-767f6b67b5-9zw22" 22:08:58 DEBUG "value": [ 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG }, 22:08:58 DEBUG "value": [ 22:08:58 DEBUG 22:08:58 DEBUG [...] 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG "value": [ 22:08:58 DEBUG 22:08:58 DEBUG [...] 22:08:58 DEBUG "value": [ 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG [...] 22:08:58 DEBUG }, 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG }, 22:08:58 DEBUG "value": [ 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG [...] 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG [...] 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG "value": [ 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG "value": [ 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG [...] 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG [...] 22:08:58 DEBUG [...] 22:08:58 DEBUG [...] 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG [...] 22:08:58 DEBUG [...] 22:08:58 DEBUG [...] 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG [...] 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG "pod": "ds-cts-2" 22:08:58 DEBUG [...] 22:08:58 DEBUG 22:08:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:08:58 DEBUG [...] 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG "pod": "ds-idrepo-2" 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG }, 22:08:58 DEBUG 22:08:58 DEBUG [...] 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG }, 22:08:58 DEBUG [...] 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG }, 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG [...] 22:08:58 DEBUG }, 22:08:58 DEBUG 22:08:58 DEBUG "pod": "am-767f6b67b5-vbltz" 22:08:58 DEBUG "pod": "am-767f6b67b5-vbltz" 22:08:58 DEBUG [...] 22:08:58 DEBUG "value": [ 22:08:58 DEBUG [...] 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG "value": [ 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG "value": [ 22:08:58 DEBUG 1713647338, 22:08:58 DEBUG 22:08:58 DEBUG "pod": "ds-idrepo-2" 22:08:58 DEBUG 22:08:58 DEBUG [...] 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG }, 22:08:58 DEBUG }, 22:08:58 DEBUG 22:08:58 DEBUG 1713647338, 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG "pod": "am-767f6b67b5-vbltz" 22:08:58 DEBUG "pod": "am-767f6b67b5-vbltz" 22:08:58 DEBUG "instance": "10.142.0.43:9100" 22:08:58 DEBUG 1713647338, 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG [...] 22:08:58 DEBUG "11.884198201348491" 22:08:58 DEBUG 22:08:58 DEBUG "pod": "ds-cts-2" 22:08:58 DEBUG }, 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG "value": [ 22:08:58 DEBUG "instance": "10.142.0.43:9100" 22:08:58 DEBUG "value": [ 22:08:58 DEBUG "value": [ 22:08:58 DEBUG 22:08:58 DEBUG "0" 22:08:58 DEBUG 22:08:58 DEBUG 22:08:58 DEBUG }, 22:08:58 DEBUG }, 22:08:58 DEBUG }, 22:08:58 DEBUG "0.5681666666666653" 22:08:58 DEBUG 22:08:58 DEBUG "instance": "10.142.0.43:9100" 22:08:58 DEBUG 22:08:58 DEBUG ] 22:08:58 DEBUG 22:08:58 DEBUG }, 22:08:58 DEBUG "value": [ 22:08:58 DEBUG "pod": "am-767f6b67b5-vbltz" 22:08:58 DEBUG }, 22:08:58 DEBUG "value": [ 22:08:58 DEBUG 1713647338, 22:08:58 DEBUG }, 22:08:58 DEBUG 1713647338, 22:08:58 DEBUG 1713647338, 22:08:58 DEBUG 22:08:58 DEBUG "instance": "10.142.0.43:9100" 22:08:58 DEBUG }, 22:08:58 DEBUG "value": [ 22:08:58 DEBUG "value": [ 22:08:58 DEBUG "value": [ 22:08:58 DEBUG "value": [ 22:08:58 DEBUG ] 22:08:58 DEBUG "pod": "ds-cts-0" 22:08:58 DEBUG }, 22:08:58 DEBUG 22:08:58 DEBUG } 22:08:58 DEBUG "pod": "ds-idrepo-2" 22:08:58 DEBUG "value": [ 22:08:58 DEBUG 1713647338, 22:08:58 DEBUG "0" 22:08:58 DEBUG "pod": "ds-cts-2" 22:08:58 DEBUG 1713647338, 22:08:58 DEBUG "0" 22:08:58 DEBUG "value": [ 22:08:58 DEBUG "69.31649999999999" 22:08:58 DEBUG "3335.1" 22:08:58 DEBUG 1713647338, 22:08:58 DEBUG ] 22:08:58 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 22:08:58 DEBUG 22:08:58 DEBUG 1713647338, 22:08:58 DEBUG 1713647338, 22:08:58 DEBUG 1713647338, 22:08:58 DEBUG } 22:08:58 DEBUG }, 22:08:58 DEBUG "value": [ 22:08:58 DEBUG 22:08:58 DEBUG ] 22:08:58 DEBUG } 22:08:58 DEBUG 1713647338, 22:08:58 DEBUG }, 22:08:58 DEBUG ] 22:08:58 DEBUG "0" 22:08:58 DEBUG ] 22:08:58 DEBUG 1713647338, 22:08:58 DEBUG 22:08:58 DEBUG ] 22:08:58 DEBUG ] 22:08:58 DEBUG "0.38373333333329923" 22:08:58 DEBUG } 22:08:58 DEBUG }, 22:08:58 DEBUG "instance": "10.142.0.43:9100" 22:08:58 DEBUG "3869.7000000000003" 22:08:58 DEBUG "3344.3333333333335" 22:08:58 DEBUG "4075328769024" 22:08:58 DEBUG ] 22:08:58 DEBUG "value": [ 22:08:58 DEBUG 1713647338, 22:08:58 DEBUG "pod": "am-767f6b67b5-vbltz" 22:08:58 DEBUG } 22:08:58 DEBUG "0" 22:08:58 DEBUG ] 22:08:58 DEBUG }, 22:08:58 DEBUG "value": [ 22:08:58 DEBUG ] 22:08:58 DEBUG } 22:08:58 DEBUG "0.07599999999999721" 22:08:58 DEBUG 22:08:58 DEBUG } 22:08:58 DEBUG } 22:08:58 DEBUG ] 22:08:58 DEBUG ] 22:08:58 DEBUG "value": [ 22:08:58 DEBUG }, 22:08:58 DEBUG ] 22:08:58 DEBUG ] 22:08:58 DEBUG ] 22:08:58 DEBUG } 22:08:58 DEBUG 1713647338, 22:08:58 DEBUG "101104641.2" 22:08:58 DEBUG }, 22:08:58 DEBUG } 22:08:58 DEBUG "value": [ 22:08:58 DEBUG 1713647338, 22:08:58 DEBUG } 22:08:58 DEBUG } 22:08:58 DEBUG ] 22:08:58 DEBUG ] 22:08:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:08:58 DEBUG ] 22:08:58 DEBUG ] 22:08:58 DEBUG } 22:08:58 DEBUG } 22:08:58 DEBUG 1713647338, 22:08:58 DEBUG "value": [ 22:08:58 DEBUG } 22:08:58 DEBUG } 22:08:58 DEBUG } 22:08:58 DEBUG } 22:08:58 DEBUG "0" 22:08:58 DEBUG ] 22:08:58 DEBUG "value": [ 22:08:58 DEBUG ] 22:08:58 DEBUG 1713647338, 22:08:58 DEBUG "30.566666666666666" 22:08:58 DEBUG ] 22:08:58 DEBUG ] 22:08:58 DEBUG } 22:08:58 DEBUG } 22:08:58 DEBUG }, 22:08:58 DEBUG } 22:08:58 DEBUG } 22:08:58 DEBUG ] 22:08:58 DEBUG } 22:08:58 DEBUG "0" 22:08:58 DEBUG 1713647338, 22:08:58 DEBUG ] 22:08:58 DEBUG ] 22:08:58 DEBUG ] 22:08:58 DEBUG ] 22:08:58 DEBUG } 22:08:58 DEBUG 1713647338, 22:08:58 DEBUG } 22:08:58 DEBUG "0" 22:08:58 DEBUG ] 22:08:58 DEBUG } 22:08:58 DEBUG } 22:08:58 DEBUG } 22:08:58 DEBUG ] 22:08:58 DEBUG "value": [ 22:08:58 DEBUG } 22:08:58 DEBUG } 22:08:58 DEBUG } 22:08:58 DEBUG } 22:08:58 DEBUG ] 22:08:58 DEBUG "101109069.53333333" 22:08:58 DEBUG } 22:08:58 DEBUG } 22:08:58 DEBUG } 22:08:58 DEBUG } 22:08:58 DEBUG ] 22:08:58 DEBUG "600558.5" 22:08:58 DEBUG } 22:08:58 DEBUG ] 22:08:58 DEBUG } 22:08:58 DEBUG } 22:08:58 DEBUG } 22:08:58 DEBUG } 22:08:58 DEBUG 1713647338, 22:08:58 DEBUG } 22:08:58 DEBUG ] 22:08:58 DEBUG } 22:08:58 DEBUG } 22:08:58 DEBUG } 22:08:58 DEBUG ] 22:08:58 DEBUG } 22:08:58 DEBUG ] 22:08:58 DEBUG } 22:08:58 DEBUG ] 22:08:58 DEBUG } 22:08:58 DEBUG "1268383744" 22:08:58 DEBUG ] 22:08:58 DEBUG } 22:08:58 DEBUG } 22:08:58 DEBUG } 22:08:58 DEBUG } 22:08:58 DEBUG ] 22:08:58 DEBUG } 22:08:58 DEBUG ] 22:08:58 DEBUG } 22:08:58 DEBUG } 22:08:58 DEBUG } 22:08:58 DEBUG ] 22:08:58 DEBUG } 22:08:58 DEBUG } 22:08:58 DEBUG } 22:08:58 DEBUG ] 22:08:58 DEBUG } 22:08:58 DEBUG } 22:08:58 DEBUG ] 22:08:58 DEBUG } 22:08:58 DEBUG } 22:08:58 DEBUG } 22:08:58 DEBUG } 22:08:58 DEBUG } 22:09:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713647394" 22:09:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647394" 22:09:54 ERROR [http_cmd]: ERROR 22:09:54 ERROR --- status code --- 22:09:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:09:54 ERROR --- http response --- 22:09:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:09:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:09:54 INFO [http_cmd]: http status code OK 22:09:54 DEBUG --- status code --- 22:09:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:09:54 DEBUG --- http response --- 22:09:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:09:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647394" 22:09:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713647394" 22:09:54 INFO [http_cmd]: http status code OK 22:09:54 INFO [http_cmd]: http status code OK 22:09:54 DEBUG --- status code --- 22:09:54 DEBUG --- status code --- 22:09:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:09:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:09:54 DEBUG --- http response --- 22:09:54 DEBUG --- http response --- 22:09:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:09:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 22:09:54 DEBUG ----- output ----- 22:09:54 DEBUG ----- output ----- 22:09:54 DEBUG { 22:09:54 DEBUG { 22:09:54 DEBUG "status": "success", 22:09:54 DEBUG "status": "success", 22:09:54 DEBUG "data": { 22:09:54 DEBUG "data": { 22:09:54 DEBUG "resultType": "vector", 22:09:54 DEBUG "resultType": "vector", 22:09:54 DEBUG "result": [ 22:09:54 DEBUG "result": [ 22:09:54 DEBUG { 22:09:54 DEBUG { 22:09:54 DEBUG "metric": { 22:09:54 DEBUG "metric": { 22:09:54 DEBUG "pod": "ds-idrepo-0" 22:09:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:09:54 DEBUG }, 22:09:54 DEBUG }, 22:09:54 DEBUG "value": [ 22:09:54 DEBUG "value": [ 22:09:54 DEBUG 22:09:54 DEBUG 22:09:54 DEBUG 22:09:54 DEBUG 22:09:54 DEBUG 22:09:54 DEBUG 22:09:54 DEBUG [...] 22:09:54 DEBUG [...] 22:09:54 DEBUG 22:09:54 DEBUG 22:09:54 DEBUG 22:09:54 DEBUG 22:09:54 DEBUG 22:09:54 DEBUG 22:09:54 DEBUG "pod": "ds-cts-2" 22:09:54 DEBUG "pod": "ds-idrepo-1" 22:09:54 DEBUG }, 22:09:54 DEBUG }, 22:09:54 DEBUG "value": [ 22:09:54 DEBUG "value": [ 22:09:54 DEBUG 1713647394, 22:09:54 DEBUG 1713647394, 22:09:54 DEBUG "0" 22:09:54 DEBUG "10989.335300825005" 22:09:54 DEBUG ] 22:09:54 DEBUG ] 22:09:54 DEBUG } 22:09:54 DEBUG } 22:09:54 DEBUG ] 22:09:54 DEBUG ] 22:09:54 DEBUG } 22:09:54 DEBUG } 22:09:54 DEBUG } 22:09:54 DEBUG } 22:09:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647396" 22:09:56 INFO [http_cmd]: http status code OK 22:09:56 DEBUG --- status code --- 22:09:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:09:56 DEBUG --- http response --- 22:09:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:09:56 DEBUG ----- output ----- 22:09:56 DEBUG { 22:09:56 DEBUG "status": "success", 22:09:56 DEBUG "data": { 22:09:56 DEBUG "resultType": "vector", 22:09:56 DEBUG "result": [ 22:09:56 DEBUG { 22:09:56 DEBUG "metric": { 22:09:56 DEBUG "pod": "am-767f6b67b5-9zw22" 22:09:56 DEBUG }, 22:09:56 DEBUG "value": [ 22:09:56 DEBUG 22:09:56 DEBUG 22:09:56 DEBUG 22:09:56 DEBUG [...] 22:09:56 DEBUG 22:09:56 DEBUG 22:09:56 DEBUG 22:09:56 DEBUG "pod": "am-767f6b67b5-vbltz" 22:09:56 DEBUG }, 22:09:56 DEBUG "value": [ 22:09:56 DEBUG 1713647396, 22:09:56 DEBUG "3323.2999999999997" 22:09:56 DEBUG ] 22:09:56 DEBUG } 22:09:56 DEBUG ] 22:09:56 DEBUG } 22:09:56 DEBUG } 22:09:56 INFO 22:09:56 INFO [loop_until]: kubectl --namespace=xlou top pods 22:09:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:09:56 INFO [loop_until]: OK (rc = 0) 22:09:56 DEBUG --- stdout --- 22:09:56 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 22814m 8809Mi am-767f6b67b5-qcqz2 22773m 8948Mi am-767f6b67b5-vbltz 24789m 8943Mi ds-cts-0 20383m 12326Mi ds-cts-1 19628m 12281Mi ds-cts-2 20114m 12475Mi ds-idrepo-0 9885m 3921Mi ds-idrepo-1 9790m 3966Mi ds-idrepo-2 9506m 3878Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-7dd48676f6-tdxl9 15m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-967d957f7-zk66v 12170m 1120Mi 22:09:56 DEBUG --- stderr --- 22:09:56 DEBUG 22:09:58 INFO 22:09:58 INFO [loop_until]: kubectl --namespace=xlou top node 22:09:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:09:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647398" 22:09:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713647398" 22:09:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647398" 22:09:58 INFO [http_cmd]: http status code OK 22:09:58 DEBUG --- status code --- 22:09:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:09:58 DEBUG --- http response --- 22:09:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:09:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713647398" 22:09:58 DEBUG ----- output ----- 22:09:58 DEBUG { 22:09:58 DEBUG "status": "success", 22:09:58 DEBUG "data": { 22:09:58 DEBUG "resultType": "vector", 22:09:58 DEBUG "result": [ 22:09:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647398" 22:09:58 DEBUG { 22:09:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647398" 22:09:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647398" 22:09:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647398" 22:09:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647398" 22:09:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647398" 22:09:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647398" 22:09:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647398" 22:09:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647398" 22:09:58 DEBUG "metric": { 22:09:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647398" 22:09:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647398" 22:09:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647398" 22:09:58 INFO [http_cmd]: http status code OK 22:09:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647398" 22:09:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647398" 22:09:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647398" 22:09:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647398" 22:09:58 INFO [http_cmd]: http status code OK 22:09:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713647398" 22:09:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647398" 22:09:58 INFO [http_cmd]: http status code OK 22:09:58 INFO [http_cmd]: http status code OK 22:09:58 INFO [http_cmd]: http status code OK 22:09:58 DEBUG "pod": "ds-idrepo-0" 22:09:58 INFO [http_cmd]: http status code OK 22:09:58 INFO [http_cmd]: http status code OK 22:09:58 INFO [http_cmd]: http status code OK 22:09:58 DEBUG --- status code --- 22:09:58 INFO [http_cmd]: http status code OK 22:09:58 INFO [http_cmd]: http status code OK 22:09:58 INFO [http_cmd]: http status code OK 22:09:58 INFO [http_cmd]: http status code OK 22:09:58 INFO [http_cmd]: http status code OK 22:09:58 DEBUG --- status code --- 22:09:58 INFO [http_cmd]: http status code OK 22:09:58 INFO [http_cmd]: http status code OK 22:09:58 DEBUG --- status code --- 22:09:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:09:58 DEBUG --- status code --- 22:09:58 INFO [http_cmd]: http status code OK 22:09:58 DEBUG }, 22:09:58 INFO [http_cmd]: http status code OK 22:09:58 INFO [http_cmd]: http status code OK 22:09:58 DEBUG --- status code --- 22:09:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:09:58 DEBUG --- status code --- 22:09:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:09:58 DEBUG --- status code --- 22:09:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:09:58 DEBUG --- http response --- 22:09:58 DEBUG --- status code --- 22:09:58 DEBUG --- status code --- 22:09:58 INFO [http_cmd]: http status code OK 22:09:58 DEBUG --- status code --- 22:09:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:09:58 DEBUG --- status code --- 22:09:58 DEBUG --- status code --- 22:09:58 DEBUG --- status code --- 22:09:58 DEBUG --- http response --- 22:09:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:09:58 DEBUG --- status code --- 22:09:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:09:58 DEBUG --- status code --- 22:09:58 DEBUG --- status code --- 22:09:58 DEBUG --- http response --- 22:09:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:09:58 DEBUG --- status code --- 22:09:58 DEBUG --- http response --- 22:09:58 INFO [http_cmd]: http status code OK 22:09:58 DEBUG --- status code --- 22:09:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:09:58 INFO [http_cmd]: http status code OK 22:09:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:09:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:09:58 DEBUG --- status code --- 22:09:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:09:58 DEBUG --- http response --- 22:09:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:09:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:09:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:09:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:09:58 DEBUG --- http response --- 22:09:58 DEBUG --- http response --- 22:09:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:09:58 DEBUG "value": [ 22:09:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:09:58 DEBUG --- http response --- 22:09:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:09:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:09:58 DEBUG --- status code --- 22:09:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:09:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:09:58 DEBUG ----- output ----- 22:09:58 DEBUG --- status code --- 22:09:58 DEBUG --- http response --- 22:09:58 DEBUG --- http response --- 22:09:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:09:58 DEBUG --- http response --- 22:09:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:09:58 DEBUG --- http response --- 22:09:58 DEBUG --- http response --- 22:09:58 DEBUG --- http response --- 22:09:58 DEBUG ----- output ----- 22:09:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:09:58 DEBUG --- http response --- 22:09:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:09:58 DEBUG 22:09:58 DEBUG --- http response --- 22:09:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:09:58 DEBUG ----- output ----- 22:09:58 DEBUG --- http response --- 22:09:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:09:58 DEBUG ----- output ----- 22:09:58 DEBUG --- http response --- 22:09:58 DEBUG { 22:09:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:09:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:09:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 22:09:58 DEBUG --- http response --- 22:09:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:09:58 DEBUG ----- output ----- 22:09:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:09:58 DEBUG ----- output ----- 22:09:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:09:58 DEBUG { 22:09:58 DEBUG ----- output ----- 22:09:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:09:58 DEBUG ----- output ----- 22:09:58 DEBUG 22:09:58 DEBUG ----- output ----- 22:09:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:09:58 DEBUG { 22:09:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:09:58 DEBUG --- http response --- 22:09:58 DEBUG { 22:09:58 DEBUG "status": "success", 22:09:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:09:58 DEBUG --- http response --- 22:09:58 DEBUG ----- output ----- 22:09:58 DEBUG ----- output ----- 22:09:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:09:58 DEBUG ----- output ----- 22:09:58 DEBUG { 22:09:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:09:58 DEBUG { 22:09:58 DEBUG ----- output ----- 22:09:58 DEBUG "status": "success", 22:09:58 DEBUG { 22:09:58 DEBUG ----- output ----- 22:09:58 DEBUG { 22:09:58 DEBUG 22:09:58 DEBUG { 22:09:58 DEBUG ----- output ----- 22:09:58 DEBUG "status": "success", 22:09:58 DEBUG ----- output ----- 22:09:58 DEBUG "status": "success", 22:09:58 DEBUG "data": { 22:09:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:09:58 DEBUG ----- output ----- 22:09:58 DEBUG { 22:09:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:09:58 DEBUG { 22:09:58 DEBUG ----- output ----- 22:09:58 DEBUG { 22:09:58 DEBUG "status": "success", 22:09:58 DEBUG ----- output ----- 22:09:58 DEBUG "status": "success", 22:09:58 DEBUG { 22:09:58 DEBUG "data": { 22:09:58 DEBUG "status": "success", 22:09:58 DEBUG { 22:09:58 DEBUG "status": "success", 22:09:58 DEBUG [...] 22:09:58 DEBUG "status": "success", 22:09:58 DEBUG { 22:09:58 DEBUG "data": { 22:09:58 DEBUG { 22:09:58 DEBUG "data": { 22:09:58 DEBUG "resultType": "vector", 22:09:58 DEBUG ----- output ----- 22:09:58 DEBUG { 22:09:58 DEBUG "status": "success", 22:09:58 DEBUG ----- output ----- 22:09:58 DEBUG "status": "success", 22:09:58 DEBUG { 22:09:58 DEBUG "status": "success", 22:09:58 DEBUG "data": { 22:09:58 DEBUG { 22:09:58 DEBUG "data": { 22:09:58 DEBUG "status": "success", 22:09:58 DEBUG "resultType": "vector", 22:09:58 DEBUG "data": { 22:09:58 DEBUG "status": "success", 22:09:58 DEBUG "data": { 22:09:58 DEBUG 22:09:58 DEBUG "data": { 22:09:58 DEBUG "status": "success", 22:09:58 DEBUG "resultType": "vector", 22:09:58 DEBUG "status": "success", 22:09:58 DEBUG "resultType": "vector", 22:09:58 DEBUG "result": [ 22:09:58 DEBUG "status": "success", 22:09:58 DEBUG "data": { 22:09:58 DEBUG { 22:09:58 DEBUG "data": { 22:09:58 DEBUG { 22:09:58 DEBUG "data": { 22:09:58 DEBUG "status": "success", 22:09:58 DEBUG "resultType": "vector", 22:09:58 DEBUG "status": "success", 22:09:58 DEBUG "resultType": "vector", 22:09:58 DEBUG "result": [ 22:09:58 DEBUG "result": [ 22:09:58 DEBUG "resultType": "vector", 22:09:58 DEBUG "data": { 22:09:58 DEBUG "resultType": "vector", 22:09:58 DEBUG "result": [ 22:09:58 DEBUG "resultType": "vector", 22:09:58 DEBUG "data": { 22:09:58 DEBUG "result": [ 22:09:58 DEBUG "data": { 22:09:58 DEBUG "result": [ 22:09:58 DEBUG { 22:09:58 DEBUG "data": { 22:09:58 DEBUG "resultType": "vector", 22:09:58 DEBUG "result": [ 22:09:58 DEBUG "resultType": "vector", 22:09:58 DEBUG "result": [ 22:09:58 DEBUG "resultType": "vector", 22:09:58 DEBUG "data": { 22:09:58 DEBUG "result": [ 22:09:58 DEBUG "data": { 22:09:58 DEBUG { 22:09:58 DEBUG "data": { 22:09:58 DEBUG { 22:09:58 DEBUG "result": [ 22:09:58 DEBUG { 22:09:58 DEBUG { 22:09:58 DEBUG 22:09:58 DEBUG "result": [ 22:09:58 DEBUG "resultType": "vector", 22:09:58 DEBUG { 22:09:58 DEBUG "resultType": "vector", 22:09:58 DEBUG { 22:09:58 DEBUG "metric": { 22:09:58 DEBUG "resultType": "vector", 22:09:58 DEBUG { 22:09:58 DEBUG "status": "success", 22:09:58 DEBUG { 22:09:58 DEBUG "status": "success", 22:09:58 DEBUG "data": { 22:09:58 DEBUG "resultType": "vector", 22:09:58 DEBUG "result": [ 22:09:58 DEBUG "resultType": "vector", 22:09:58 DEBUG "metric": { 22:09:58 DEBUG "resultType": "vector", 22:09:58 DEBUG "metric": { 22:09:58 DEBUG "metric": { 22:09:58 DEBUG "resultType": "vector", 22:09:58 DEBUG "metric": { 22:09:58 DEBUG 22:09:58 DEBUG { 22:09:58 DEBUG "result": [ 22:09:58 DEBUG "metric": { 22:09:58 DEBUG "result": [ 22:09:58 DEBUG "metric": { 22:09:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:09:58 DEBUG "result": [ 22:09:58 DEBUG "metric": { 22:09:58 DEBUG "data": { 22:09:58 DEBUG "metric": { 22:09:58 DEBUG "result": [ 22:09:58 DEBUG "resultType": "vector", 22:09:58 DEBUG { 22:09:58 DEBUG "metric": { 22:09:58 DEBUG "result": [ 22:09:58 DEBUG "instance": "10.142.0.107:9100" 22:09:58 DEBUG "result": [ 22:09:58 DEBUG { 22:09:58 DEBUG "metric": { 22:09:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:09:58 DEBUG "pod": "ds-idrepo-0" 22:09:58 DEBUG "pod": "ds-idrepo-2" 22:09:58 DEBUG "metric": { 22:09:58 DEBUG { 22:09:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:09:58 DEBUG { 22:09:58 DEBUG "pod": "ds-cts-0" 22:09:58 DEBUG }, 22:09:58 DEBUG { 22:09:58 DEBUG "pod": "am-767f6b67b5-9zw22" 22:09:58 DEBUG }, 22:09:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:09:58 DEBUG { 22:09:58 DEBUG "result": [ 22:09:58 DEBUG { 22:09:58 DEBUG "instance": "10.142.0.107:9100" 22:09:58 DEBUG { 22:09:58 DEBUG }, 22:09:58 DEBUG "pod": "am-767f6b67b5-9zw22" 22:09:58 DEBUG "pod": "ds-idrepo-0" 22:09:58 DEBUG "result": [ 22:09:58 DEBUG }, 22:09:58 DEBUG }, 22:09:58 DEBUG "value": [ 22:09:58 DEBUG "pod": "am-767f6b67b5-9zw22" 22:09:58 DEBUG "metric": { 22:09:58 DEBUG }, 22:09:58 DEBUG "metric": { 22:09:58 DEBUG "value": [ 22:09:58 DEBUG "metric": { 22:09:58 DEBUG "resultType": "vector", 22:09:58 DEBUG "value": [ 22:09:58 DEBUG }, 22:09:58 DEBUG }, 22:09:58 DEBUG "metric": { 22:09:58 DEBUG { 22:09:58 DEBUG "metric": { 22:09:58 DEBUG }, 22:09:58 DEBUG "metric": { 22:09:58 DEBUG "value": [ 22:09:58 DEBUG }, 22:09:58 DEBUG }, 22:09:58 DEBUG { 22:09:58 DEBUG "value": [ 22:09:58 DEBUG 22:09:58 DEBUG }, 22:09:58 DEBUG }, 22:09:58 DEBUG "instance": "10.142.0.107:9100" 22:09:58 DEBUG "value": [ 22:09:58 DEBUG "instance": "10.142.0.107:9100" 22:09:58 DEBUG 22:09:58 DEBUG "pod": "am-767f6b67b5-9zw22" 22:09:58 DEBUG "result": [ 22:09:58 DEBUG 22:09:58 DEBUG "value": [ 22:09:58 DEBUG "value": [ 22:09:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:09:58 DEBUG "metric": { 22:09:58 DEBUG "pod": "ds-cts-0" 22:09:58 DEBUG "value": [ 22:09:58 DEBUG "pod": "am-767f6b67b5-9zw22" 22:09:58 DEBUG 22:09:58 DEBUG "value": [ 22:09:58 DEBUG 22:09:58 DEBUG "metric": { 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG "value": [ 22:09:58 DEBUG "value": [ 22:09:58 DEBUG }, 22:09:58 DEBUG 22:09:58 DEBUG }, 22:09:58 DEBUG 22:09:58 DEBUG }, 22:09:58 DEBUG { 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG }, 22:09:58 DEBUG "pod": "am-767f6b67b5-9zw22" 22:09:58 DEBUG }, 22:09:58 DEBUG 22:09:58 DEBUG }, 22:09:58 DEBUG 22:09:58 DEBUG "value": [ 22:09:58 DEBUG 22:09:58 DEBUG "pod": "ds-cts-0" 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG 1713647398, 22:09:58 DEBUG 22:09:58 DEBUG "value": [ 22:09:58 DEBUG 22:09:58 DEBUG "value": [ 22:09:58 DEBUG 22:09:58 DEBUG "value": [ 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG "value": [ 22:09:58 DEBUG }, 22:09:58 DEBUG "value": [ 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG }, 22:09:58 DEBUG 22:09:58 DEBUG [...] 22:09:58 DEBUG "0" 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG [...] 22:09:58 DEBUG 22:09:58 DEBUG [...] 22:09:58 DEBUG "metric": { 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG "value": [ 22:09:58 DEBUG 22:09:58 DEBUG "value": [ 22:09:58 DEBUG [...] 22:09:58 DEBUG [...] 22:09:58 DEBUG 22:09:58 DEBUG [...] 22:09:58 DEBUG "value": [ 22:09:58 DEBUG [...] 22:09:58 DEBUG 22:09:58 DEBUG ] 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG [...] 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG "instance": "10.142.0.107:9100" 22:09:58 DEBUG [...] 22:09:58 DEBUG [...] 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG } 22:09:58 DEBUG [...] 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG [...] 22:09:58 DEBUG 22:09:58 DEBUG }, 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG [...] 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG "pod": "ds-idrepo-2" 22:09:58 DEBUG 22:09:58 DEBUG [...] 22:09:58 DEBUG 22:09:58 DEBUG [...] 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG "value": [ 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG [...] 22:09:58 DEBUG 22:09:58 DEBUG [...] 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG "instance": "10.142.0.43:9100" 22:09:58 DEBUG }, 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG }, 22:09:58 DEBUG ] 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG "pod": "ds-cts-2" 22:09:58 DEBUG }, 22:09:58 DEBUG "pod": "am-767f6b67b5-vbltz" 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG [...] 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG "value": [ 22:09:58 DEBUG 22:09:58 DEBUG "pod": "am-767f6b67b5-vbltz" 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 22:09:58 DEBUG [...] 22:09:58 DEBUG } 22:09:58 DEBUG 22:09:58 DEBUG "pod": "am-767f6b67b5-vbltz" 22:09:58 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 22:09:58 DEBUG 22:09:58 DEBUG "value": [ 22:09:58 DEBUG 22:09:58 DEBUG }, 22:09:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG "pod": "ds-cts-2" 22:09:58 DEBUG 22:09:58 DEBUG 1713647398, 22:09:58 DEBUG "instance": "10.142.0.43:9100" 22:09:58 DEBUG }, 22:09:58 DEBUG 22:09:58 DEBUG "value": [ 22:09:58 DEBUG }, 22:09:58 DEBUG 22:09:58 DEBUG } 22:09:58 DEBUG }, 22:09:58 DEBUG 22:09:58 DEBUG }, 22:09:58 DEBUG 22:09:58 DEBUG 1713647398, 22:09:58 DEBUG 22:09:58 DEBUG "value": [ 22:09:58 DEBUG }, 22:09:58 DEBUG 22:09:58 DEBUG [...] 22:09:58 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 22:09:58 DEBUG 22:09:58 DEBUG }, 22:09:58 DEBUG [...] 22:09:58 DEBUG 22:09:58 DEBUG }, 22:09:58 DEBUG "value": [ 22:09:58 DEBUG "pod": "ds-idrepo-2" 22:09:58 DEBUG 1713647398, 22:09:58 DEBUG "value": [ 22:09:58 DEBUG 22:09:58 DEBUG "value": [ 22:09:58 DEBUG 22:09:58 DEBUG "value": [ 22:09:58 DEBUG "instance": "10.142.0.43:9100" 22:09:58 DEBUG "0" 22:09:58 DEBUG ] 22:09:58 DEBUG 1713647398, 22:09:58 DEBUG "value": [ 22:09:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:09:58 DEBUG 22:09:58 DEBUG }, 22:09:58 DEBUG 22:09:58 DEBUG "pod": "am-767f6b67b5-vbltz" 22:09:58 DEBUG 22:09:58 DEBUG 22:09:58 DEBUG "pod": "am-767f6b67b5-vbltz" 22:09:58 DEBUG 1713647398, 22:09:58 DEBUG "73.13550000000001" 22:09:58 DEBUG "0" 22:09:58 DEBUG 1713647398, 22:09:58 DEBUG 22:09:58 DEBUG 1713647398, 22:09:58 DEBUG "instance": "10.142.0.43:9100" 22:09:58 DEBUG 1713647398, 22:09:58 DEBUG "0" 22:09:58 DEBUG } 22:09:58 DEBUG }, 22:09:58 DEBUG "3337.266666666667" 22:09:58 DEBUG 1713647398, 22:09:58 DEBUG "11.837759104092267" 22:09:58 DEBUG ] 22:09:58 DEBUG "value": [ 22:09:58 DEBUG "value": [ 22:09:58 DEBUG }, 22:09:58 INFO [loop_until]: OK (rc = 0) 22:09:58 DEBUG }, 22:09:58 DEBUG "4080576523264" 22:09:58 DEBUG "value": [ 22:09:58 DEBUG ] 22:09:58 DEBUG }, 22:09:58 DEBUG ] 22:09:58 DEBUG "0" 22:09:58 DEBUG "pod": "ds-cts-2" 22:09:58 DEBUG "0" 22:09:58 DEBUG }, 22:09:58 DEBUG ] 22:09:58 DEBUG "pod": "am-767f6b67b5-vbltz" 22:09:58 DEBUG ] 22:09:58 DEBUG "value": [ 22:09:58 DEBUG ] 22:09:58 DEBUG }, 22:09:58 DEBUG } 22:09:58 DEBUG 22:09:58 DEBUG 1713647398, 22:09:58 DEBUG 1713647398, 22:09:58 DEBUG "value": [ 22:09:58 DEBUG --- stdout --- 22:09:58 DEBUG "value": [ 22:09:58 DEBUG ] 22:09:58 DEBUG 1713647398, 22:09:58 DEBUG } 22:09:58 DEBUG "value": [ 22:09:58 DEBUG } 22:09:58 DEBUG ] 22:09:58 DEBUG }, 22:09:58 DEBUG ] 22:09:58 DEBUG "value": [ 22:09:58 DEBUG } 22:09:58 DEBUG ] 22:09:58 DEBUG } 22:09:58 DEBUG 1713647398, 22:09:58 DEBUG } 22:09:58 DEBUG "value": [ 22:09:58 DEBUG ] 22:09:58 DEBUG 22:09:58 DEBUG "0.00003432998169420541" 22:09:58 DEBUG "0.6178666666666648" 22:09:58 DEBUG 1713647398, 22:09:58 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 6211Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 9907m 33% 7683Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 22307m 74% 11553Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 86m 0% 3366Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 20037m 67% 15559Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 149m 0% 5207Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 22777m 76% 11848Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19715m 66% 3051Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 24851m 83% 11187Mi 10% gke-xlou-cdm-ds-0f601751-j488 10246m 34% 7300Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19485m 65% 17369Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 10058m 33% 7946Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20510m 68% 15180Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 12111m 76% 9026Mi 15% 22:09:58 DEBUG 1713647398, 22:09:58 DEBUG } 22:09:58 DEBUG "0.08400000000000886" 22:09:58 DEBUG ] 22:09:58 DEBUG 1713647398, 22:09:58 DEBUG ] 22:09:58 DEBUG } 22:09:58 DEBUG "value": [ 22:09:58 DEBUG } 22:09:58 DEBUG 1713647398, 22:09:58 DEBUG } 22:09:58 DEBUG }, 22:09:58 DEBUG } 22:09:58 DEBUG "0.40139999999989207" 22:09:58 DEBUG ] 22:09:58 DEBUG 1713647398, 22:09:58 DEBUG } 22:09:58 DEBUG "instance": "10.142.0.43:9100" 22:09:58 DEBUG ] 22:09:58 DEBUG ] 22:09:58 DEBUG "3874.5" 22:09:58 DEBUG --- stderr --- 22:09:58 DEBUG "3326.7999999999997" 22:09:58 DEBUG ] 22:09:58 DEBUG ] 22:09:58 DEBUG } 22:09:58 DEBUG "0" 22:09:58 DEBUG } 22:09:58 DEBUG ] 22:09:58 DEBUG 1713647398, 22:09:58 DEBUG ] 22:09:58 DEBUG "100980940.6" 22:09:58 DEBUG ] 22:09:58 DEBUG "value": [ 22:09:58 DEBUG ] 22:09:58 DEBUG } 22:09:58 DEBUG "1271103488" 22:09:58 DEBUG } 22:09:58 DEBUG }, 22:09:58 DEBUG } 22:09:58 DEBUG } 22:09:58 DEBUG ] 22:09:58 DEBUG ] 22:09:58 DEBUG 22:09:58 DEBUG } 22:09:58 DEBUG } 22:09:58 DEBUG } 22:09:58 DEBUG ] 22:09:58 DEBUG } 22:09:58 DEBUG } 22:09:58 DEBUG "28.966666666666665" 22:09:58 DEBUG } 22:09:58 DEBUG } 22:09:58 DEBUG } 22:09:58 DEBUG 1713647398, 22:09:58 DEBUG } 22:09:58 DEBUG } 22:09:58 DEBUG ] 22:09:58 DEBUG "value": [ 22:09:58 DEBUG 1713647398, 22:09:58 DEBUG ] 22:09:58 DEBUG } 22:09:58 DEBUG } 22:09:58 DEBUG } 22:09:58 DEBUG ] 22:09:58 DEBUG } 22:09:58 DEBUG } 22:09:58 DEBUG ] 22:09:58 DEBUG } 22:09:58 DEBUG ] 22:09:58 DEBUG "600549" 22:09:58 DEBUG ] 22:09:58 DEBUG } 22:09:58 DEBUG ] 22:09:58 DEBUG "100985723.66666666" 22:09:58 DEBUG } 22:09:58 DEBUG ] 22:09:58 DEBUG ] 22:09:58 DEBUG } 22:09:58 DEBUG ] 22:09:58 DEBUG } 22:09:58 DEBUG } 22:09:58 DEBUG ] 22:09:58 DEBUG } 22:09:58 DEBUG ] 22:09:58 DEBUG } 22:09:58 DEBUG ] 22:09:58 DEBUG } 22:09:58 DEBUG } 22:09:58 DEBUG } 22:09:58 DEBUG } 22:09:58 DEBUG } 22:09:58 DEBUG ] 22:09:58 DEBUG } 22:09:58 DEBUG } 22:09:58 DEBUG } 22:09:58 DEBUG } 22:09:58 DEBUG } 22:09:58 DEBUG } 22:09:58 DEBUG } 22:09:58 DEBUG } 22:09:58 DEBUG } 22:09:58 DEBUG } 22:09:58 DEBUG ] 22:09:58 DEBUG } 22:09:58 DEBUG ] 22:09:58 DEBUG } 22:09:58 DEBUG } 22:09:58 DEBUG } 22:09:58 DEBUG } 22:09:58 DEBUG } 22:10:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713647454" 22:10:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647454" 22:10:54 ERROR [http_cmd]: ERROR 22:10:54 ERROR --- status code --- 22:10:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:10:54 ERROR --- http response --- 22:10:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:10:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:10:54 INFO [http_cmd]: http status code OK 22:10:54 DEBUG --- status code --- 22:10:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:54 DEBUG --- http response --- 22:10:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:10:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647454" 22:10:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713647454" 22:10:54 INFO [http_cmd]: http status code OK 22:10:54 DEBUG --- status code --- 22:10:54 INFO [http_cmd]: http status code OK 22:10:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:54 DEBUG --- status code --- 22:10:54 DEBUG --- http response --- 22:10:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:10:54 DEBUG --- http response --- 22:10:54 DEBUG ----- output ----- 22:10:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:10:54 DEBUG { 22:10:54 DEBUG ----- output ----- 22:10:54 DEBUG "status": "success", 22:10:54 DEBUG { 22:10:54 DEBUG "data": { 22:10:54 DEBUG "status": "success", 22:10:54 DEBUG "resultType": "vector", 22:10:54 DEBUG "data": { 22:10:54 DEBUG "result": [ 22:10:54 DEBUG "resultType": "vector", 22:10:54 DEBUG { 22:10:54 DEBUG "result": [ 22:10:54 DEBUG "metric": { 22:10:54 DEBUG { 22:10:54 DEBUG "pod": "ds-idrepo-0" 22:10:54 DEBUG "metric": { 22:10:54 DEBUG }, 22:10:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:10:54 DEBUG }, 22:10:54 DEBUG "value": [ 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG [...] 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 22:10:54 DEBUG }, 22:10:54 DEBUG "value": [ 22:10:54 DEBUG 1713647454, 22:10:54 DEBUG "0" 22:10:54 DEBUG ] 22:10:54 DEBUG } 22:10:54 DEBUG ] 22:10:54 DEBUG } 22:10:54 DEBUG } 22:10:54 DEBUG "value": [ 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG [...] 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG "pod": "ds-cts-2" 22:10:54 DEBUG }, 22:10:54 DEBUG "value": [ 22:10:54 DEBUG 1713647454, 22:10:54 DEBUG "0" 22:10:54 DEBUG ] 22:10:54 DEBUG } 22:10:54 DEBUG ] 22:10:54 DEBUG } 22:10:54 DEBUG } 22:10:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647456" 22:10:56 INFO [http_cmd]: http status code OK 22:10:56 DEBUG --- status code --- 22:10:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:56 DEBUG --- http response --- 22:10:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:10:56 DEBUG ----- output ----- 22:10:56 DEBUG { 22:10:56 DEBUG "status": "success", 22:10:56 DEBUG "data": { 22:10:56 DEBUG "resultType": "vector", 22:10:56 DEBUG "result": [ 22:10:56 DEBUG { 22:10:56 DEBUG "metric": { 22:10:56 DEBUG "pod": "am-767f6b67b5-9zw22" 22:10:56 DEBUG }, 22:10:56 DEBUG "value": [ 22:10:56 DEBUG 22:10:56 DEBUG 22:10:56 DEBUG 22:10:56 DEBUG [...] 22:10:56 DEBUG 22:10:56 DEBUG 22:10:56 DEBUG 22:10:56 DEBUG "pod": "am-767f6b67b5-vbltz" 22:10:56 DEBUG }, 22:10:56 DEBUG "value": [ 22:10:56 DEBUG 1713647456, 22:10:56 DEBUG "3342.4666666666667" 22:10:56 DEBUG ] 22:10:56 DEBUG } 22:10:56 DEBUG ] 22:10:56 DEBUG } 22:10:56 DEBUG } 22:10:56 INFO 22:10:56 INFO [loop_until]: kubectl --namespace=xlou top pods 22:10:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:10:56 INFO [loop_until]: OK (rc = 0) 22:10:56 DEBUG --- stdout --- 22:10:56 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 22254m 8812Mi am-767f6b67b5-qcqz2 22370m 8947Mi am-767f6b67b5-vbltz 24108m 8940Mi ds-cts-0 19555m 12321Mi ds-cts-1 19175m 12290Mi ds-cts-2 20426m 12476Mi ds-idrepo-0 9468m 3921Mi ds-idrepo-1 9411m 3966Mi ds-idrepo-2 9644m 3878Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-7dd48676f6-tdxl9 17m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-967d957f7-zk66v 11849m 1124Mi 22:10:56 DEBUG --- stderr --- 22:10:56 DEBUG 22:10:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647458" 22:10:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647458" 22:10:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713647458" 22:10:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713647458" 22:10:58 INFO 22:10:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647458" 22:10:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647458" 22:10:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647458" 22:10:58 INFO [loop_until]: kubectl --namespace=xlou top node 22:10:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647458" 22:10:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647458" 22:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647458" 22:10:59 INFO [http_cmd]: http status code OK 22:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647459" 22:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647459" 22:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647459" 22:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647459" 22:10:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647459" 22:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647459" 22:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647459" 22:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647459" 22:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647459" 22:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647459" 22:10:59 INFO [http_cmd]: http status code OK 22:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713647459" 22:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647459" 22:10:59 DEBUG --- status code --- 22:10:59 INFO [http_cmd]: http status code OK 22:10:59 INFO [http_cmd]: http status code OK 22:10:59 INFO [http_cmd]: http status code OK 22:10:59 INFO [http_cmd]: http status code OK 22:10:59 INFO [http_cmd]: http status code OK 22:10:59 INFO [http_cmd]: http status code OK 22:10:59 INFO [http_cmd]: http status code OK 22:10:59 INFO [http_cmd]: http status code OK 22:10:59 INFO [http_cmd]: http status code OK 22:10:59 INFO [http_cmd]: http status code OK 22:10:59 INFO [http_cmd]: http status code OK 22:10:59 INFO [http_cmd]: http status code OK 22:10:59 INFO [http_cmd]: http status code OK 22:10:59 DEBUG --- status code --- 22:10:59 INFO [http_cmd]: http status code OK 22:10:59 INFO [http_cmd]: http status code OK 22:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:59 DEBUG --- status code --- 22:10:59 DEBUG --- status code --- 22:10:59 DEBUG --- status code --- 22:10:59 DEBUG --- status code --- 22:10:59 INFO [http_cmd]: http status code OK 22:10:59 DEBUG --- status code --- 22:10:59 DEBUG --- status code --- 22:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:59 INFO [http_cmd]: http status code OK 22:10:59 DEBUG --- status code --- 22:10:59 DEBUG --- status code --- 22:10:59 DEBUG --- status code --- 22:10:59 DEBUG --- status code --- 22:10:59 DEBUG --- status code --- 22:10:59 DEBUG --- status code --- 22:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:59 INFO [http_cmd]: http status code OK 22:10:59 DEBUG --- status code --- 22:10:59 DEBUG --- status code --- 22:10:59 DEBUG --- http response --- 22:10:59 INFO [http_cmd]: http status code OK 22:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:59 DEBUG --- http response --- 22:10:59 INFO [http_cmd]: http status code OK 22:10:59 DEBUG --- status code --- 22:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:59 DEBUG --- status code --- 22:10:59 DEBUG --- http response --- 22:10:59 DEBUG --- status code --- 22:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:59 DEBUG --- http response --- 22:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:59 DEBUG --- http response --- 22:10:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:10:59 DEBUG ----- output ----- 22:10:59 DEBUG --- http response --- 22:10:59 DEBUG --- http response --- 22:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:10:59 DEBUG --- status code --- 22:10:59 DEBUG --- http response --- 22:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:59 DEBUG --- http response --- 22:10:59 DEBUG --- http response --- 22:10:59 DEBUG --- http response --- 22:10:59 DEBUG --- http response --- 22:10:59 DEBUG --- http response --- 22:10:59 DEBUG --- http response --- 22:10:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:10:59 DEBUG --- http response --- 22:10:59 DEBUG { 22:10:59 DEBUG --- status code --- 22:10:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:10:59 DEBUG --- status code --- 22:10:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:10:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:10:59 DEBUG --- http response --- 22:10:59 DEBUG ----- output ----- 22:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:59 DEBUG --- http response --- 22:10:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:10:59 DEBUG --- http response --- 22:10:59 DEBUG ----- output ----- 22:10:59 DEBUG --- http response --- 22:10:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:10:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:10:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:10:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:10:59 DEBUG ----- output ----- 22:10:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:10:59 DEBUG ----- output ----- 22:10:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:10:59 DEBUG "status": "success", 22:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:59 DEBUG ----- output ----- 22:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:59 DEBUG ----- output ----- 22:10:59 DEBUG ----- output ----- 22:10:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:10:59 DEBUG { 22:10:59 DEBUG --- http response --- 22:10:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:10:59 DEBUG ----- output ----- 22:10:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:10:59 DEBUG { 22:10:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:10:59 DEBUG ----- output ----- 22:10:59 DEBUG ----- output ----- 22:10:59 DEBUG ----- output ----- 22:10:59 DEBUG { 22:10:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:10:59 DEBUG ----- output ----- 22:10:59 DEBUG { 22:10:59 DEBUG ----- output ----- 22:10:59 DEBUG "data": { 22:10:59 DEBUG --- http response --- 22:10:59 DEBUG { 22:10:59 DEBUG --- http response --- 22:10:59 DEBUG { 22:10:59 DEBUG { 22:10:59 DEBUG "status": "success", 22:10:59 DEBUG "status": "success", 22:10:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:10:59 DEBUG ----- output ----- 22:10:59 DEBUG { 22:10:59 DEBUG ----- output ----- 22:10:59 DEBUG { 22:10:59 DEBUG { 22:10:59 DEBUG "status": "success", 22:10:59 DEBUG "data": { 22:10:59 DEBUG "status": "success", 22:10:59 DEBUG "data": { 22:10:59 DEBUG ----- output ----- 22:10:59 DEBUG { 22:10:59 DEBUG "status": "success", 22:10:59 DEBUG { 22:10:59 DEBUG "resultType": "vector", 22:10:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:10:59 DEBUG "status": "success", 22:10:59 DEBUG "data": { 22:10:59 DEBUG "resultType": "vector", 22:10:59 DEBUG "data": { 22:10:59 DEBUG "resultType": "vector", 22:10:59 DEBUG "data": { 22:10:59 DEBUG "resultType": "vector", 22:10:59 DEBUG { 22:10:59 DEBUG "status": "success", 22:10:59 DEBUG "status": "success", 22:10:59 DEBUG "data": { 22:10:59 DEBUG "resultType": "vector", 22:10:59 DEBUG "result": [ 22:10:59 DEBUG { 22:10:59 DEBUG "resultType": "vector", 22:10:59 DEBUG "status": "success", 22:10:59 DEBUG "data": { 22:10:59 DEBUG "status": "success", 22:10:59 DEBUG "data": { 22:10:59 DEBUG "status": "success", 22:10:59 DEBUG "result": [ 22:10:59 DEBUG ----- output ----- 22:10:59 DEBUG "result": [ 22:10:59 DEBUG { 22:10:59 DEBUG "metric": { 22:10:59 DEBUG "pod": "ds-idrepo-0" 22:10:59 DEBUG }, 22:10:59 DEBUG "result": [ 22:10:59 DEBUG { 22:10:59 DEBUG "metric": { 22:10:59 DEBUG "data": { 22:10:59 DEBUG { 22:10:59 DEBUG { 22:10:59 DEBUG "status": "success", 22:10:59 DEBUG "resultType": "vector", 22:10:59 DEBUG ----- output ----- 22:10:59 DEBUG "result": [ 22:10:59 DEBUG "resultType": "vector", 22:10:59 DEBUG { 22:10:59 DEBUG "status": "success", 22:10:59 DEBUG "resultType": "vector", 22:10:59 DEBUG "data": { 22:10:59 DEBUG { 22:10:59 DEBUG { 22:10:59 DEBUG "status": "success", 22:10:59 DEBUG "value": [ 22:10:59 DEBUG ----- output ----- 22:10:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:10:59 DEBUG "result": [ 22:10:59 DEBUG "status": "success", 22:10:59 DEBUG "data": { 22:10:59 DEBUG ----- output ----- 22:10:59 DEBUG "resultType": "vector", 22:10:59 DEBUG "status": "success", 22:10:59 DEBUG "data": { 22:10:59 DEBUG "metric": { 22:10:59 DEBUG "pod": "ds-idrepo-1" 22:10:59 DEBUG { 22:10:59 DEBUG { 22:10:59 DEBUG "result": [ 22:10:59 DEBUG "data": { 22:10:59 DEBUG "data": { 22:10:59 DEBUG "result": [ 22:10:59 DEBUG "resultType": "vector", 22:10:59 DEBUG "metric": { 22:10:59 DEBUG "status": "success", 22:10:59 DEBUG "data": { 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG { 22:10:59 DEBUG "resultType": "vector", 22:10:59 DEBUG "result": [ 22:10:59 DEBUG { 22:10:59 DEBUG "result": [ 22:10:59 DEBUG "data": { 22:10:59 DEBUG "resultType": "vector", 22:10:59 DEBUG "result": [ 22:10:59 DEBUG "result": [ 22:10:59 DEBUG "status": "success", 22:10:59 DEBUG "metric": { 22:10:59 DEBUG "pod": "ds-cts-0" 22:10:59 DEBUG "resultType": "vector", 22:10:59 DEBUG "resultType": "vector", 22:10:59 DEBUG { 22:10:59 DEBUG "result": [ 22:10:59 DEBUG "pod": "ds-idrepo-0" 22:10:59 DEBUG "data": { 22:10:59 DEBUG "resultType": "vector", 22:10:59 DEBUG { 22:10:59 DEBUG [...] 22:10:59 DEBUG ----- output ----- 22:10:59 DEBUG { 22:10:59 DEBUG { 22:10:59 DEBUG "metric": { 22:10:59 DEBUG "status": "success", 22:10:59 DEBUG { 22:10:59 DEBUG "resultType": "vector", 22:10:59 DEBUG { 22:10:59 DEBUG }, 22:10:59 DEBUG { 22:10:59 DEBUG "data": { 22:10:59 DEBUG }, 22:10:59 DEBUG "value": [ 22:10:59 DEBUG "result": [ 22:10:59 DEBUG "result": [ 22:10:59 DEBUG "metric": { 22:10:59 DEBUG "pod": "ds-cts-0" 22:10:59 DEBUG }, 22:10:59 DEBUG "value": [ 22:10:59 DEBUG 22:10:59 DEBUG "result": [ 22:10:59 DEBUG { 22:10:59 DEBUG "metric": { 22:10:59 DEBUG "pod": "am-767f6b67b5-9zw22" 22:10:59 DEBUG "instance": "10.142.0.107:9100" 22:10:59 DEBUG }, 22:10:59 DEBUG "data": { 22:10:59 DEBUG "metric": { 22:10:59 DEBUG "result": [ 22:10:59 DEBUG { 22:10:59 DEBUG "metric": { 22:10:59 DEBUG "pod": "am-767f6b67b5-9zw22" 22:10:59 DEBUG "resultType": "vector", 22:10:59 DEBUG "result": [ 22:10:59 DEBUG { 22:10:59 DEBUG { 22:10:59 DEBUG "metric": { 22:10:59 DEBUG "instance": "10.142.0.107:9100" 22:10:59 DEBUG 22:10:59 DEBUG { 22:10:59 DEBUG "metric": { 22:10:59 DEBUG "pod": "am-767f6b67b5-9zw22" 22:10:59 DEBUG 22:10:59 DEBUG "metric": { 22:10:59 DEBUG "pod": "ds-idrepo-0" 22:10:59 DEBUG "value": [ 22:10:59 DEBUG 22:10:59 DEBUG "resultType": "vector", 22:10:59 DEBUG "resultType": "vector", 22:10:59 DEBUG "pod": "am-767f6b67b5-9zw22" 22:10:59 DEBUG }, 22:10:59 DEBUG "value": [ 22:10:59 DEBUG "metric": { 22:10:59 DEBUG "value": [ 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG [...] 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG }, 22:10:59 DEBUG "value": [ 22:10:59 DEBUG "status": "success", 22:10:59 DEBUG 22:10:59 DEBUG }, 22:10:59 DEBUG "value": [ 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG [...] 22:10:59 DEBUG 22:10:59 DEBUG }, 22:10:59 DEBUG 22:10:59 DEBUG "metric": { 22:10:59 DEBUG "pod": "ds-idrepo-1" 22:10:59 DEBUG "metric": { 22:10:59 DEBUG "instance": "10.142.0.107:9100" 22:10:59 DEBUG }, 22:10:59 DEBUG "value": [ 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG [...] 22:10:59 DEBUG 22:10:59 DEBUG }, 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG "instance": "10.142.0.43:9100" 22:10:59 DEBUG }, 22:10:59 DEBUG "value": [ 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG }, 22:10:59 DEBUG [...] 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG "instance": "10.142.0.43:9100" 22:10:59 DEBUG }, 22:10:59 DEBUG "value": [ 22:10:59 DEBUG 1713647459, 22:10:59 DEBUG "data": { 22:10:59 DEBUG "value": [ 22:10:59 DEBUG 22:10:59 DEBUG [...] 22:10:59 DEBUG 22:10:59 DEBUG [...] 22:10:59 DEBUG 1713647458, 22:10:59 DEBUG "4085517372416" 22:10:59 DEBUG ] 22:10:59 DEBUG "result": [ 22:10:59 DEBUG "value": [ 22:10:59 DEBUG [...] 22:10:59 DEBUG "pod": "am-767f6b67b5-9zw22" 22:10:59 DEBUG "value": [ 22:10:59 DEBUG "101041615.93333334" 22:10:59 DEBUG ] 22:10:59 DEBUG } 22:10:59 DEBUG ] 22:10:59 DEBUG } 22:10:59 DEBUG } 22:10:59 DEBUG { 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG "resultType": "vector", 22:10:59 DEBUG "result": [ 22:10:59 DEBUG { 22:10:59 DEBUG "result": [ 22:10:59 DEBUG { 22:10:59 DEBUG "metric": { 22:10:59 DEBUG { 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG }, 22:10:59 DEBUG 22:10:59 DEBUG "metric": { 22:10:59 DEBUG "pod": "am-767f6b67b5-9zw22" 22:10:59 DEBUG }, 22:10:59 DEBUG "value": [ 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG "pod": "ds-cts-2" 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG [...] 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG "instance": "10.142.0.107:9100" 22:10:59 DEBUG }, 22:10:59 DEBUG "value": [ 22:10:59 DEBUG [...] 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG "pod": "am-767f6b67b5-vbltz" 22:10:59 DEBUG }, 22:10:59 DEBUG 22:10:59 DEBUG [...] 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG "pod": "am-767f6b67b5-vbltz" 22:10:59 DEBUG }, 22:10:59 DEBUG "value": [ 22:10:59 DEBUG 1713647458, 22:10:59 DEBUG 22:10:59 DEBUG } 22:10:59 DEBUG ] 22:10:59 DEBUG } 22:10:59 DEBUG 22:10:59 DEBUG "status": "success", 22:10:59 DEBUG 22:10:59 DEBUG "value": [ 22:10:59 DEBUG "pod": "ds-cts-0" 22:10:59 DEBUG 22:10:59 DEBUG "metric": { 22:10:59 DEBUG "value": [ 22:10:59 DEBUG }, 22:10:59 DEBUG 22:10:59 DEBUG }, 22:10:59 DEBUG 22:10:59 DEBUG { 22:10:59 DEBUG }, 22:10:59 DEBUG "value": [ 22:10:59 DEBUG "0" 22:10:59 DEBUG "metric": { 22:10:59 DEBUG 22:10:59 DEBUG } 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG "data": { 22:10:59 DEBUG 22:10:59 DEBUG 1713647459, 22:10:59 DEBUG }, 22:10:59 DEBUG 22:10:59 DEBUG "instance": "10.142.0.107:9100" 22:10:59 DEBUG 22:10:59 DEBUG "value": [ 22:10:59 DEBUG 22:10:59 DEBUG "value": [ 22:10:59 DEBUG 22:10:59 DEBUG "metric": { 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG ] 22:10:59 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:10:59 DEBUG 22:10:59 DEBUG "pod": "ds-idrepo-2" 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG "resultType": "vector", 22:10:59 DEBUG [...] 22:10:59 DEBUG "3673.1666666666665" 22:10:59 DEBUG "value": [ 22:10:59 DEBUG 22:10:59 DEBUG }, 22:10:59 DEBUG 22:10:59 DEBUG 1713647458, 22:10:59 DEBUG "pod": "ds-cts-2" 22:10:59 DEBUG 22:10:59 DEBUG [...] 22:10:59 DEBUG "pod": "ds-idrepo-1" 22:10:59 DEBUG 22:10:59 DEBUG "pod": "am-767f6b67b5-vbltz" 22:10:59 DEBUG } 22:10:59 DEBUG }, 22:10:59 DEBUG "pod": "am-767f6b67b5-vbltz" 22:10:59 DEBUG }, 22:10:59 DEBUG 22:10:59 DEBUG [...] 22:10:59 DEBUG "result": [ 22:10:59 DEBUG 22:10:59 DEBUG ] 22:10:59 DEBUG 22:10:59 DEBUG "pod": "am-767f6b67b5-vbltz" 22:10:59 DEBUG "value": [ 22:10:59 DEBUG 22:10:59 DEBUG "0" 22:10:59 DEBUG }, 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG }, 22:10:59 DEBUG 22:10:59 DEBUG }, 22:10:59 DEBUG ] 22:10:59 DEBUG } 22:10:59 DEBUG }, 22:10:59 DEBUG "value": [ 22:10:59 DEBUG "pod": "ds-idrepo-2" 22:10:59 DEBUG 22:10:59 DEBUG { 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG }, 22:10:59 DEBUG 22:10:59 DEBUG [...] 22:10:59 DEBUG ] 22:10:59 DEBUG "value": [ 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG "value": [ 22:10:59 DEBUG [...] 22:10:59 DEBUG "value": [ 22:10:59 INFO [loop_until]: OK (rc = 0) 22:10:59 DEBUG "value": [ 22:10:59 DEBUG } 22:10:59 DEBUG "value": [ 22:10:59 DEBUG 1713647458, 22:10:59 DEBUG }, 22:10:59 DEBUG 22:10:59 DEBUG "metric": { 22:10:59 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 22:10:59 DEBUG } 22:10:59 DEBUG 22:10:59 DEBUG "value": [ 22:10:59 DEBUG 1713647459, 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG 1713647459, 22:10:59 DEBUG "0.534499999999999" 22:10:59 DEBUG ] 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG 1713647459, 22:10:59 DEBUG "600038.5" 22:10:59 DEBUG 22:10:59 DEBUG 1713647458, 22:10:59 DEBUG "76.1715" 22:10:59 DEBUG "value": [ 22:10:59 DEBUG 1713647458, 22:10:59 DEBUG "0" 22:10:59 DEBUG }, 22:10:59 DEBUG "value": [ 22:10:59 DEBUG 1713647458, 22:10:59 DEBUG "0" 22:10:59 DEBUG ] 22:10:59 DEBUG } 22:10:59 DEBUG ] 22:10:59 DEBUG } 22:10:59 DEBUG ] 22:10:59 DEBUG } 22:10:59 DEBUG ] 22:10:59 DEBUG } 22:10:59 DEBUG 22:10:59 DEBUG } 22:10:59 DEBUG 22:10:59 DEBUG } 22:10:59 DEBUG 22:10:59 DEBUG ] 22:10:59 DEBUG "3340.5" 22:10:59 DEBUG } 22:10:59 DEBUG 22:10:59 DEBUG [...] 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG --- stdout --- 22:10:59 DEBUG 22:10:59 DEBUG [...] 22:10:59 DEBUG ] 22:10:59 DEBUG } 22:10:59 DEBUG ] 22:10:59 DEBUG 22:10:59 DEBUG "0" 22:10:59 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:10:59 DEBUG 22:10:59 DEBUG "instance": "10.142.0.43:9100" 22:10:59 DEBUG } 22:10:59 DEBUG ] 22:10:59 DEBUG ] 22:10:59 DEBUG "instance": "10.142.0.43:9100" 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 6203Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 9922m 33% 7678Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 22293m 74% 11552Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 3373Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 20021m 67% 15555Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 150m 0% 5179Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 22010m 73% 11842Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19765m 66% 3055Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 24841m 83% 11195Mi 10% gke-xlou-cdm-ds-0f601751-j488 10282m 34% 7295Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19770m 66% 17370Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 10150m 33% 7945Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20443m 68% 15178Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 12110m 76% 9014Mi 15% 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG } 22:10:59 DEBUG ] 22:10:59 DEBUG } 22:10:59 DEBUG "pod": "am-767f6b67b5-vbltz" 22:10:59 DEBUG ] 22:10:59 DEBUG }, 22:10:59 DEBUG [...] 22:10:59 DEBUG }, 22:10:59 DEBUG ] 22:10:59 DEBUG } 22:10:59 DEBUG } 22:10:59 DEBUG }, 22:10:59 DEBUG 22:10:59 DEBUG [...] 22:10:59 DEBUG --- stderr --- 22:10:59 DEBUG [...] 22:10:59 DEBUG 22:10:59 DEBUG ] 22:10:59 DEBUG } 22:10:59 DEBUG } 22:10:59 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 22:10:59 DEBUG }, 22:10:59 DEBUG } 22:10:59 DEBUG "value": [ 22:10:59 DEBUG 22:10:59 DEBUG "value": [ 22:10:59 DEBUG } 22:10:59 DEBUG ] 22:10:59 DEBUG } 22:10:59 DEBUG "value": [ 22:10:59 DEBUG "pod": "ds-idrepo-2" 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG } 22:10:59 DEBUG } 22:10:59 DEBUG }, 22:10:59 DEBUG "value": [ 22:10:59 DEBUG ] 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG 1713647459, 22:10:59 DEBUG } 22:10:59 DEBUG } 22:10:59 DEBUG 1713647459, 22:10:59 DEBUG }, 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG "pod": "ds-cts-2" 22:10:59 DEBUG } 22:10:59 DEBUG "value": [ 22:10:59 DEBUG 1713647458, 22:10:59 DEBUG } 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG "0.07500000000002086" 22:10:59 DEBUG } 22:10:59 DEBUG "0.37833333333331703" 22:10:59 DEBUG "value": [ 22:10:59 DEBUG 22:10:59 DEBUG 22:10:59 DEBUG }, 22:10:59 DEBUG 1713647458, 22:10:59 DEBUG "3342" 22:10:59 DEBUG } 22:10:59 DEBUG 22:10:59 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:10:59 DEBUG ] 22:10:59 DEBUG ] 22:10:59 DEBUG 1713647459, 22:10:59 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 22:10:59 DEBUG "instance": "10.142.0.43:9100" 22:10:59 DEBUG "value": [ 22:10:59 DEBUG "0" 22:10:59 DEBUG ] 22:10:59 DEBUG [...] 22:10:59 DEBUG }, 22:10:59 DEBUG } 22:10:59 DEBUG } 22:10:59 DEBUG "0" 22:10:59 DEBUG }, 22:10:59 DEBUG }, 22:10:59 DEBUG 1713647459, 22:10:59 DEBUG ] 22:10:59 DEBUG } 22:10:59 DEBUG 22:10:59 DEBUG "value": [ 22:10:59 DEBUG ] 22:10:59 DEBUG ] 22:10:59 DEBUG ] 22:10:59 DEBUG "value": [ 22:10:59 DEBUG "value": [ 22:10:59 DEBUG "34.06666666666666" 22:10:59 DEBUG } 22:10:59 DEBUG ] 22:10:59 DEBUG 22:10:59 DEBUG 1713647458, 22:10:59 DEBUG } 22:10:59 DEBUG } 22:10:59 DEBUG } 22:10:59 DEBUG 1713647459, 22:10:59 DEBUG 1713647459, 22:10:59 DEBUG ] 22:10:59 DEBUG ] 22:10:59 DEBUG } 22:10:59 DEBUG 22:10:59 DEBUG "11.869149536427127" 22:10:59 DEBUG } 22:10:59 DEBUG } 22:10:59 DEBUG ] 22:10:59 DEBUG "0.00003120477904342791" 22:10:59 DEBUG "101046352.6" 22:10:59 DEBUG } 22:10:59 DEBUG } 22:10:59 DEBUG } 22:10:59 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:10:59 DEBUG ] 22:10:59 DEBUG } 22:10:59 DEBUG ] 22:10:59 DEBUG ] 22:10:59 DEBUG ] 22:10:59 DEBUG } 22:10:59 DEBUG }, 22:10:59 DEBUG } 22:10:59 DEBUG } 22:10:59 DEBUG } 22:10:59 DEBUG } 22:10:59 DEBUG } 22:10:59 DEBUG "value": [ 22:10:59 DEBUG ] 22:10:59 DEBUG ] 22:10:59 DEBUG ] 22:10:59 DEBUG } 22:10:59 DEBUG 1713647459, 22:10:59 DEBUG } 22:10:59 DEBUG } 22:10:59 DEBUG } 22:10:59 DEBUG "1274859520" 22:10:59 DEBUG } 22:10:59 DEBUG } 22:10:59 DEBUG } 22:10:59 DEBUG ] 22:10:59 DEBUG } 22:10:59 DEBUG ] 22:10:59 DEBUG } 22:10:59 DEBUG } 22:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713647514" 22:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647514" 22:11:54 ERROR [http_cmd]: ERROR 22:11:54 ERROR --- status code --- 22:11:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:11:54 ERROR --- http response --- 22:11:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:11:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:11:54 INFO [http_cmd]: http status code OK 22:11:54 DEBUG --- status code --- 22:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:54 DEBUG --- http response --- 22:11:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:11:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647515" 22:11:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713647515" 22:11:55 INFO [http_cmd]: http status code OK 22:11:55 DEBUG --- status code --- 22:11:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:55 DEBUG --- http response --- 22:11:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 22:11:55 DEBUG ----- output ----- 22:11:55 DEBUG { 22:11:55 DEBUG "status": "success", 22:11:55 DEBUG "data": { 22:11:55 DEBUG "resultType": "vector", 22:11:55 DEBUG "result": [ 22:11:55 DEBUG { 22:11:55 DEBUG "metric": { 22:11:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:11:55 DEBUG }, 22:11:55 DEBUG "value": [ 22:11:55 DEBUG 22:11:55 DEBUG 22:11:55 DEBUG 22:11:55 DEBUG [...] 22:11:55 DEBUG 22:11:55 DEBUG 22:11:55 DEBUG 22:11:55 DEBUG "pod": "overseer-0-967d957f7-zk66v" 22:11:55 INFO [http_cmd]: http status code OK 22:11:55 DEBUG }, 22:11:55 DEBUG --- status code --- 22:11:55 DEBUG "value": [ 22:11:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:55 DEBUG 1713647515, 22:11:55 DEBUG --- http response --- 22:11:55 DEBUG "95360.9425418211" 22:11:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:11:55 DEBUG ] 22:11:55 DEBUG ----- output ----- 22:11:55 DEBUG } 22:11:55 DEBUG ] 22:11:55 DEBUG } 22:11:55 DEBUG } 22:11:55 DEBUG { 22:11:55 DEBUG "status": "success", 22:11:55 DEBUG "data": { 22:11:55 DEBUG "resultType": "vector", 22:11:55 DEBUG "result": [ 22:11:55 DEBUG { 22:11:55 DEBUG "metric": { 22:11:55 DEBUG "pod": "ds-idrepo-0" 22:11:55 DEBUG }, 22:11:55 DEBUG "value": [ 22:11:55 DEBUG 22:11:55 DEBUG 22:11:55 DEBUG 22:11:55 DEBUG [...] 22:11:55 DEBUG 22:11:55 DEBUG 22:11:55 DEBUG 22:11:55 DEBUG "pod": "ds-cts-2" 22:11:55 DEBUG }, 22:11:55 DEBUG "value": [ 22:11:55 DEBUG 1713647515, 22:11:55 DEBUG "0" 22:11:55 DEBUG ] 22:11:55 DEBUG } 22:11:55 DEBUG ] 22:11:55 DEBUG } 22:11:55 DEBUG } 22:11:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647516" 22:11:56 INFO [http_cmd]: http status code OK 22:11:56 DEBUG --- status code --- 22:11:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:56 DEBUG --- http response --- 22:11:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:11:56 DEBUG ----- output ----- 22:11:56 DEBUG { 22:11:56 DEBUG "status": "success", 22:11:56 DEBUG "data": { 22:11:56 DEBUG "resultType": "vector", 22:11:56 DEBUG "result": [ 22:11:56 DEBUG { 22:11:56 DEBUG "metric": { 22:11:56 DEBUG "pod": "am-767f6b67b5-9zw22" 22:11:56 DEBUG }, 22:11:56 DEBUG "value": [ 22:11:56 DEBUG 22:11:56 DEBUG 22:11:56 DEBUG 22:11:56 DEBUG [...] 22:11:56 DEBUG 22:11:56 DEBUG 22:11:56 DEBUG 22:11:56 DEBUG "pod": "am-767f6b67b5-vbltz" 22:11:56 DEBUG }, 22:11:56 DEBUG "value": [ 22:11:56 DEBUG 1713647516, 22:11:56 DEBUG "2564.1666666666665" 22:11:56 DEBUG ] 22:11:56 DEBUG } 22:11:56 DEBUG ] 22:11:56 DEBUG } 22:11:56 DEBUG } 22:11:56 INFO 22:11:56 INFO [loop_until]: kubectl --namespace=xlou top pods 22:11:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:11:57 INFO [loop_until]: OK (rc = 0) 22:11:57 DEBUG --- stdout --- 22:11:57 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 22306m 8811Mi am-767f6b67b5-qcqz2 22856m 8945Mi am-767f6b67b5-vbltz 24931m 8947Mi ds-cts-0 20458m 12317Mi ds-cts-1 19831m 12289Mi ds-cts-2 19809m 12476Mi ds-idrepo-0 9678m 3921Mi ds-idrepo-1 9379m 3966Mi ds-idrepo-2 9849m 3878Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 4m 1238Mi lodemon-7dd48676f6-tdxl9 27m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-967d957f7-zk66v 11797m 1127Mi 22:11:57 DEBUG --- stderr --- 22:11:57 DEBUG 22:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647519" 22:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647519" 22:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647519" 22:11:59 INFO [http_cmd]: http status code OK 22:11:59 DEBUG --- status code --- 22:11:59 INFO [http_cmd]: http status code OK 22:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647519" 22:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647519" 22:11:59 DEBUG --- status code --- 22:11:59 DEBUG --- http response --- 22:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:59 DEBUG --- http response --- 22:11:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:11:59 DEBUG ----- output ----- 22:11:59 DEBUG { 22:11:59 DEBUG "status": "success", 22:11:59 DEBUG "data": { 22:11:59 DEBUG "resultType": "vector", 22:11:59 DEBUG "result": [ 22:11:59 DEBUG { 22:11:59 DEBUG "metric": { 22:11:59 DEBUG "instance": "10.142.0.107:9100" 22:11:59 DEBUG }, 22:11:59 DEBUG "value": [ 22:11:59 DEBUG 22:11:59 DEBUG 22:11:59 DEBUG 22:11:59 DEBUG [...] 22:11:59 DEBUG 22:11:59 DEBUG 22:11:59 DEBUG 22:11:59 DEBUG "instance": "10.142.0.43:9100" 22:11:59 DEBUG }, 22:11:59 DEBUG "value": [ 22:11:59 DEBUG 1713647519, 22:11:59 DEBUG "101596366.13333333" 22:11:59 DEBUG ] 22:11:59 DEBUG } 22:11:59 DEBUG ] 22:11:59 DEBUG } 22:11:59 DEBUG } 22:11:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:11:59 DEBUG ----- output ----- 22:11:59 DEBUG { 22:11:59 DEBUG "status": "success", 22:11:59 INFO 22:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647519" 22:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647519" 22:11:59 DEBUG "data": { 22:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647519" 22:11:59 INFO [loop_until]: kubectl --namespace=xlou top node 22:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647519" 22:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713647519" 22:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647519" 22:11:59 INFO [http_cmd]: http status code OK 22:11:59 DEBUG "resultType": "vector", 22:11:59 INFO [http_cmd]: http status code OK 22:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647519" 22:11:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647519" 22:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647519" 22:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647519" 22:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647519" 22:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647519" 22:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713647519" 22:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647519" 22:11:59 INFO [http_cmd]: http status code OK 22:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647519" 22:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647519" 22:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713647519" 22:11:59 DEBUG --- status code --- 22:11:59 INFO [http_cmd]: http status code OK 22:11:59 DEBUG "result": [ 22:11:59 INFO [http_cmd]: http status code OK 22:11:59 INFO [http_cmd]: http status code OK 22:11:59 DEBUG --- status code --- 22:11:59 INFO [http_cmd]: http status code OK 22:11:59 INFO [http_cmd]: http status code OK 22:11:59 INFO [http_cmd]: http status code OK 22:11:59 INFO [http_cmd]: http status code OK 22:11:59 DEBUG --- status code --- 22:11:59 INFO [http_cmd]: http status code OK 22:11:59 INFO [http_cmd]: http status code OK 22:11:59 INFO [http_cmd]: http status code OK 22:11:59 INFO [http_cmd]: http status code OK 22:11:59 INFO [http_cmd]: http status code OK 22:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:59 INFO [http_cmd]: http status code OK 22:11:59 INFO [http_cmd]: http status code OK 22:11:59 DEBUG --- status code --- 22:11:59 DEBUG { 22:11:59 DEBUG --- status code --- 22:11:59 DEBUG --- status code --- 22:11:59 INFO [http_cmd]: http status code OK 22:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:59 DEBUG --- status code --- 22:11:59 INFO [http_cmd]: http status code OK 22:11:59 DEBUG --- status code --- 22:11:59 DEBUG --- status code --- 22:11:59 DEBUG --- status code --- 22:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:59 DEBUG --- status code --- 22:11:59 DEBUG --- status code --- 22:11:59 DEBUG --- status code --- 22:11:59 DEBUG --- status code --- 22:11:59 DEBUG --- status code --- 22:11:59 DEBUG --- http response --- 22:11:59 DEBUG --- status code --- 22:11:59 DEBUG --- status code --- 22:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:59 DEBUG "metric": { 22:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:59 DEBUG --- status code --- 22:11:59 DEBUG --- http response --- 22:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:59 DEBUG --- status code --- 22:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:59 DEBUG --- http response --- 22:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:59 DEBUG --- http response --- 22:11:59 DEBUG "instance": "10.142.0.107:9100" 22:11:59 INFO [http_cmd]: http status code OK 22:11:59 DEBUG --- http response --- 22:11:59 DEBUG --- http response --- 22:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:59 DEBUG --- http response --- 22:11:59 DEBUG --- http response --- 22:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:11:59 DEBUG --- http response --- 22:11:59 DEBUG --- http response --- 22:11:59 DEBUG --- http response --- 22:11:59 DEBUG --- http response --- 22:11:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:11:59 DEBUG --- http response --- 22:11:59 DEBUG --- http response --- 22:11:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:11:59 DEBUG ----- output ----- 22:11:59 DEBUG --- http response --- 22:11:59 DEBUG --- http response --- 22:11:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:11:59 DEBUG ----- output ----- 22:11:59 DEBUG --- status code --- 22:11:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:11:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:11:59 DEBUG --- http response --- 22:11:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:11:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:11:59 DEBUG ----- output ----- 22:11:59 DEBUG ----- output ----- 22:11:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:11:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:11:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:11:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:11:59 DEBUG ----- output ----- 22:11:59 DEBUG { 22:11:59 DEBUG ----- output ----- 22:11:59 DEBUG { 22:11:59 DEBUG { 22:11:59 DEBUG "status": "success", 22:11:59 DEBUG "data": { 22:11:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:11:59 DEBUG ----- output ----- 22:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:59 DEBUG ----- output ----- 22:11:59 DEBUG ----- output ----- 22:11:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 22:11:59 DEBUG ----- output ----- 22:11:59 DEBUG { 22:11:59 DEBUG "status": "success", 22:11:59 DEBUG { 22:11:59 DEBUG ----- output ----- 22:11:59 DEBUG ----- output ----- 22:11:59 DEBUG ----- output ----- 22:11:59 DEBUG ----- output ----- 22:11:59 DEBUG "status": "success", 22:11:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:11:59 DEBUG "status": "success", 22:11:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:11:59 DEBUG "resultType": "vector", 22:11:59 DEBUG { 22:11:59 DEBUG --- http response --- 22:11:59 DEBUG { 22:11:59 DEBUG "status": "success", 22:11:59 DEBUG --- http response --- 22:11:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:11:59 DEBUG ----- output ----- 22:11:59 DEBUG { 22:11:59 DEBUG "status": "success", 22:11:59 DEBUG "data": { 22:11:59 DEBUG "resultType": "vector", 22:11:59 DEBUG "data": { 22:11:59 DEBUG { 22:11:59 DEBUG { 22:11:59 DEBUG { 22:11:59 DEBUG { 22:11:59 DEBUG "data": { 22:11:59 DEBUG ----- output ----- 22:11:59 DEBUG "data": { 22:11:59 DEBUG ----- output ----- 22:11:59 DEBUG "result": [ 22:11:59 DEBUG "status": "success", 22:11:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:11:59 DEBUG ----- output ----- 22:11:59 DEBUG { 22:11:59 DEBUG { 22:11:59 DEBUG "status": "success", 22:11:59 DEBUG }, 22:11:59 DEBUG "result": [ 22:11:59 DEBUG { 22:11:59 DEBUG "metric": { 22:11:59 DEBUG "pod": "ds-idrepo-1" 22:11:59 DEBUG "resultType": "vector", 22:11:59 DEBUG "status": "success", 22:11:59 DEBUG "data": { 22:11:59 DEBUG "resultType": "vector", 22:11:59 DEBUG "status": "success", 22:11:59 DEBUG "resultType": "vector", 22:11:59 DEBUG { 22:11:59 DEBUG "resultType": "vector", 22:11:59 DEBUG { 22:11:59 DEBUG { 22:11:59 DEBUG "data": { 22:11:59 DEBUG "resultType": "vector", 22:11:59 DEBUG { 22:11:59 DEBUG "status": "success", 22:11:59 DEBUG "data": { 22:11:59 DEBUG { 22:11:59 DEBUG "value": [ 22:11:59 DEBUG }, 22:11:59 DEBUG "value": [ 22:11:59 DEBUG 22:11:59 DEBUG --- http response --- 22:11:59 DEBUG "result": [ 22:11:59 DEBUG "status": "success", 22:11:59 DEBUG "result": [ 22:11:59 DEBUG "status": "success", 22:11:59 DEBUG "data": { 22:11:59 DEBUG "result": [ 22:11:59 DEBUG "status": "success", 22:11:59 DEBUG "result": [ 22:11:59 DEBUG "status": "success", 22:11:59 DEBUG "metric": { 22:11:59 DEBUG "result": [ 22:11:59 DEBUG "data": { 22:11:59 DEBUG "resultType": "vector", 22:11:59 DEBUG "status": "success", 22:11:59 DEBUG "data": { 22:11:59 DEBUG "status": "success", 22:11:59 DEBUG 22:11:59 DEBUG "status": "success", 22:11:59 DEBUG ----- output ----- 22:11:59 DEBUG 22:11:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:11:59 DEBUG ----- output ----- 22:11:59 DEBUG "data": { 22:11:59 DEBUG { 22:11:59 DEBUG "metric": { 22:11:59 DEBUG "resultType": "vector", 22:11:59 DEBUG { 22:11:59 DEBUG "data": { 22:11:59 DEBUG { 22:11:59 DEBUG "data": { 22:11:59 DEBUG "resultType": "vector", 22:11:59 DEBUG { 22:11:59 DEBUG "metric": { 22:11:59 DEBUG "pod": "ds-cts-0" 22:11:59 DEBUG "data": { 22:11:59 DEBUG "resultType": "vector", 22:11:59 DEBUG "data": { 22:11:59 DEBUG 22:11:59 DEBUG "data": { 22:11:59 DEBUG { 22:11:59 DEBUG "status": "success", 22:11:59 DEBUG "data": { 22:11:59 DEBUG { 22:11:59 DEBUG "resultType": "vector", 22:11:59 DEBUG "pod": "am-767f6b67b5-9zw22" 22:11:59 DEBUG }, 22:11:59 DEBUG "data": { 22:11:59 DEBUG "metric": { 22:11:59 DEBUG "resultType": "vector", 22:11:59 DEBUG "metric": { 22:11:59 DEBUG "pod": "am-767f6b67b5-9zw22" 22:11:59 DEBUG }, 22:11:59 DEBUG "value": [ 22:11:59 DEBUG 22:11:59 DEBUG "result": [ 22:11:59 DEBUG { 22:11:59 DEBUG "resultType": "vector", 22:11:59 DEBUG "pod": "ds-cts-0" 22:11:59 DEBUG [...] 22:11:59 DEBUG "resultType": "vector", 22:11:59 DEBUG "result": [ 22:11:59 DEBUG "resultType": "vector", 22:11:59 DEBUG { 22:11:59 DEBUG "metric": { 22:11:59 DEBUG "result": [ 22:11:59 DEBUG "result": [ 22:11:59 DEBUG "value": [ 22:11:59 DEBUG "resultType": "vector", 22:11:59 DEBUG "pod": "am-767f6b67b5-9zw22" 22:11:59 DEBUG "result": [ 22:11:59 DEBUG }, 22:11:59 DEBUG "result": [ 22:11:59 DEBUG 22:11:59 DEBUG "result": [ 22:11:59 DEBUG "resultType": "vector", 22:11:59 DEBUG "result": [ 22:11:59 DEBUG "resultType": "vector", 22:11:59 DEBUG "result": [ 22:11:59 DEBUG }, 22:11:59 DEBUG 22:11:59 DEBUG { 22:11:59 DEBUG 22:11:59 DEBUG "result": [ 22:11:59 DEBUG "status": "success", 22:11:59 DEBUG "instance": "10.142.0.107:9100" 22:11:59 DEBUG { 22:11:59 DEBUG { 22:11:59 DEBUG 22:11:59 DEBUG "result": [ 22:11:59 DEBUG }, 22:11:59 DEBUG { 22:11:59 DEBUG "value": [ 22:11:59 DEBUG 22:11:59 DEBUG 22:11:59 DEBUG [...] 22:11:59 DEBUG 22:11:59 DEBUG "metric": { 22:11:59 DEBUG "result": [ 22:11:59 DEBUG { 22:11:59 DEBUG "value": [ 22:11:59 DEBUG 22:11:59 DEBUG "metric": { 22:11:59 DEBUG 22:11:59 DEBUG { 22:11:59 DEBUG "data": { 22:11:59 DEBUG }, 22:11:59 DEBUG "metric": { 22:11:59 DEBUG "metric": { 22:11:59 DEBUG 22:11:59 DEBUG { 22:11:59 DEBUG "value": [ 22:11:59 DEBUG "metric": { 22:11:59 DEBUG 22:11:59 DEBUG { 22:11:59 DEBUG 22:11:59 DEBUG { 22:11:59 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:11:59 DEBUG { 22:11:59 DEBUG { 22:11:59 DEBUG "metric": { 22:11:59 DEBUG 22:11:59 DEBUG 22:11:59 DEBUG "pod": "ds-idrepo-1" 22:11:59 DEBUG [...] 22:11:59 DEBUG "metric": { 22:11:59 DEBUG "resultType": "vector", 22:11:59 DEBUG "value": [ 22:11:59 DEBUG "instance": "10.142.0.107:9100" 22:11:59 DEBUG "instance": "10.142.0.107:9100" 22:11:59 DEBUG 22:11:59 DEBUG "metric": { 22:11:59 DEBUG 22:11:59 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:11:59 DEBUG 22:11:59 DEBUG "metric": { 22:11:59 DEBUG 22:11:59 DEBUG "metric": { 22:11:59 DEBUG }, 22:11:59 DEBUG "metric": { 22:11:59 DEBUG "metric": { 22:11:59 DEBUG "pod": "ds-cts-0" 22:11:59 DEBUG 22:11:59 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 22:11:59 DEBUG }, 22:11:59 DEBUG 22:11:59 DEBUG "pod": "ds-idrepo-0" 22:11:59 DEBUG "result": [ 22:11:59 DEBUG 22:11:59 DEBUG }, 22:11:59 DEBUG }, 22:11:59 DEBUG [...] 22:11:59 DEBUG "pod": "ds-idrepo-0" 22:11:59 DEBUG 22:11:59 DEBUG }, 22:11:59 DEBUG [...] 22:11:59 DEBUG 22:11:59 DEBUG 22:11:59 DEBUG "pod": "ds-idrepo-1" 22:11:59 DEBUG "value": [ 22:11:59 DEBUG "pod": "am-767f6b67b5-9zw22" 22:11:59 DEBUG "pod": "am-767f6b67b5-9zw22" 22:11:59 DEBUG }, 22:11:59 DEBUG 22:11:59 DEBUG }, 22:11:59 DEBUG "value": [ 22:11:59 DEBUG 22:11:59 DEBUG }, 22:11:59 DEBUG { 22:11:59 DEBUG 22:11:59 DEBUG "value": [ 22:11:59 DEBUG "value": [ 22:11:59 DEBUG 22:11:59 DEBUG }, 22:11:59 DEBUG 22:11:59 DEBUG "value": [ 22:11:59 DEBUG 22:11:59 DEBUG "pod": "ds-idrepo-0" 22:11:59 DEBUG "pod": "ds-cts-2" 22:11:59 DEBUG }, 22:11:59 DEBUG 22:11:59 DEBUG }, 22:11:59 DEBUG }, 22:11:59 DEBUG "value": [ 22:11:59 DEBUG [...] 22:11:59 DEBUG "value": [ 22:11:59 DEBUG 22:11:59 DEBUG 22:11:59 DEBUG "value": [ 22:11:59 DEBUG "metric": { 22:11:59 DEBUG 22:11:59 DEBUG 22:11:59 DEBUG 22:11:59 DEBUG 22:11:59 DEBUG "value": [ 22:11:59 DEBUG 22:11:59 DEBUG 22:11:59 DEBUG "pod": "am-767f6b67b5-vbltz" 22:11:59 DEBUG }, 22:11:59 DEBUG }, 22:11:59 DEBUG "value": [ 22:11:59 DEBUG 22:11:59 DEBUG "value": [ 22:11:59 DEBUG "value": [ 22:11:59 DEBUG 22:11:59 DEBUG 22:11:59 DEBUG 1713647519, 22:11:59 DEBUG 22:11:59 DEBUG "instance": "10.142.0.43:9100" 22:11:59 DEBUG 22:11:59 DEBUG "pod": "am-767f6b67b5-9zw22" 22:11:59 DEBUG [...] 22:11:59 DEBUG 22:11:59 DEBUG 22:11:59 DEBUG [...] 22:11:59 DEBUG 22:11:59 DEBUG 22:11:59 DEBUG 22:11:59 DEBUG }, 22:11:59 DEBUG "value": [ 22:11:59 DEBUG "value": [ 22:11:59 DEBUG 22:11:59 DEBUG 22:11:59 DEBUG 22:11:59 DEBUG 22:11:59 DEBUG 22:11:59 DEBUG 22:11:59 DEBUG "0.00002955110278819237" 22:11:59 DEBUG 22:11:59 DEBUG }, 22:11:59 DEBUG 22:11:59 DEBUG }, 22:11:59 DEBUG 22:11:59 DEBUG [...] 22:11:59 DEBUG "pod": "am-767f6b67b5-vbltz" 22:11:59 DEBUG 22:11:59 DEBUG 22:11:59 DEBUG 22:11:59 DEBUG [...] 22:11:59 DEBUG "value": [ 22:11:59 DEBUG 22:11:59 DEBUG 1713647519, 22:11:59 DEBUG 22:11:59 DEBUG [...] 22:11:59 DEBUG 22:11:59 DEBUG 22:11:59 DEBUG 22:11:59 DEBUG 22:11:59 DEBUG ] 22:11:59 DEBUG [...] 22:11:59 DEBUG "value": [ 22:11:59 DEBUG 22:11:59 DEBUG "value": [ 22:11:59 DEBUG 22:11:59 DEBUG 22:11:59 DEBUG }, 22:11:59 DEBUG 22:11:59 DEBUG 22:11:59 DEBUG 22:11:59 DEBUG 22:11:59 DEBUG 1713647519, 22:11:59 DEBUG 22:11:59 DEBUG "32.63333333333333" 22:11:59 DEBUG 22:11:59 DEBUG 22:11:59 DEBUG [...] 22:11:59 DEBUG 22:11:59 DEBUG [...] 22:11:59 DEBUG "pod": "ds-cts-2" 22:11:59 DEBUG } 22:11:59 DEBUG 22:11:59 DEBUG 1713647519, 22:11:59 DEBUG [...] 22:11:59 DEBUG 22:11:59 DEBUG 22:11:59 DEBUG 22:11:59 DEBUG "value": [ 22:11:59 DEBUG 22:11:59 DEBUG "pod": "am-767f6b67b5-vbltz" 22:11:59 DEBUG 22:11:59 DEBUG [...] 22:11:59 DEBUG "2560.5" 22:11:59 DEBUG 22:11:59 DEBUG ] 22:11:59 DEBUG [...] 22:11:59 DEBUG 22:11:59 DEBUG 22:11:59 DEBUG 22:11:59 DEBUG 22:11:59 DEBUG 22:11:59 DEBUG ] 22:11:59 DEBUG 22:11:59 DEBUG "4090448845824" 22:11:59 DEBUG 22:11:59 DEBUG 22:11:59 DEBUG "instance": "10.142.0.43:9100" 22:11:59 DEBUG 22:11:59 DEBUG 1713647519, 22:11:59 DEBUG }, 22:11:59 DEBUG [...] 22:11:59 DEBUG 22:11:59 DEBUG 22:11:59 DEBUG ] 22:11:59 DEBUG [...] 22:11:59 DEBUG } 22:11:59 DEBUG 22:11:59 DEBUG 22:11:59 DEBUG 22:11:59 DEBUG [...] 22:11:59 DEBUG 22:11:59 DEBUG }, 22:11:59 DEBUG } 22:11:59 DEBUG } 22:11:59 DEBUG ] 22:11:59 DEBUG 22:11:59 DEBUG 22:11:59 DEBUG }, 22:11:59 DEBUG "instance": "10.142.0.43:9100" 22:11:59 DEBUG "80.72999999999999" 22:11:59 DEBUG "value": [ 22:11:59 DEBUG 22:11:59 DEBUG "instance": "10.142.0.43:9100" 22:11:59 DEBUG 22:11:59 DEBUG } 22:11:59 DEBUG 22:11:59 DEBUG ] 22:11:59 INFO [loop_until]: OK (rc = 0) 22:11:59 DEBUG 22:11:59 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:11:59 DEBUG 22:11:59 DEBUG 22:11:59 DEBUG "pod": "ds-cts-2" 22:11:59 DEBUG "value": [ 22:11:59 DEBUG 22:11:59 DEBUG } 22:11:59 DEBUG 22:11:59 DEBUG [...] 22:11:59 DEBUG "value": [ 22:11:59 DEBUG }, 22:11:59 DEBUG ] 22:11:59 DEBUG 1713647519, 22:11:59 DEBUG 22:11:59 DEBUG }, 22:11:59 DEBUG 22:11:59 DEBUG ] 22:11:59 DEBUG 22:11:59 DEBUG 22:11:59 DEBUG --- stdout --- 22:11:59 DEBUG 22:11:59 DEBUG }, 22:11:59 DEBUG "pod": "am-767f6b67b5-vbltz" 22:11:59 DEBUG 22:11:59 DEBUG }, 22:11:59 DEBUG "value": [ 22:11:59 DEBUG 1713647519, 22:11:59 DEBUG ] 22:11:59 DEBUG "pod": "ds-idrepo-2" 22:11:59 DEBUG }, 22:11:59 DEBUG 1713647519, 22:11:59 DEBUG "value": [ 22:11:59 DEBUG } 22:11:59 DEBUG "588943.5" 22:11:59 DEBUG 22:11:59 DEBUG "value": [ 22:11:59 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:11:59 DEBUG }, 22:11:59 DEBUG } 22:11:59 DEBUG "pod": "ds-idrepo-2" 22:11:59 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 6212Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8609m 28% 7685Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 19159m 64% 11575Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 81m 0% 3374Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 19537m 65% 15543Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 136m 0% 5182Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 15887m 53% 11901Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12958m 43% 3053Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 22107m 74% 11224Mi 10% gke-xlou-cdm-ds-0f601751-j488 8519m 28% 7287Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 18980m 63% 17354Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 8903m 29% 7952Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20379m 68% 15160Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 8131m 51% 7995Mi 13% 22:11:59 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 22:11:59 DEBUG "value": [ 22:11:59 DEBUG }, 22:11:59 DEBUG 22:11:59 DEBUG "0.3337333333333333" 22:11:59 DEBUG 1713647519, 22:11:59 DEBUG } 22:11:59 DEBUG 22:11:59 DEBUG "value": [ 22:11:59 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 22:11:59 DEBUG "101600820.86666667" 22:11:59 DEBUG 1713647519, 22:11:59 DEBUG ] 22:11:59 DEBUG ] 22:11:59 DEBUG "pod": "ds-idrepo-2" 22:11:59 DEBUG 1713647519, 22:11:59 DEBUG } 22:11:59 DEBUG "value": [ 22:11:59 DEBUG } 22:11:59 DEBUG }, 22:11:59 DEBUG --- stderr --- 22:11:59 DEBUG }, 22:11:59 DEBUG 1713647519, 22:11:59 DEBUG "value": [ 22:11:59 DEBUG "pod": "am-767f6b67b5-vbltz" 22:11:59 DEBUG ] 22:11:59 DEBUG "0" 22:11:59 DEBUG } 22:11:59 DEBUG 22:11:59 DEBUG 1713647519, 22:11:59 DEBUG }, 22:11:59 DEBUG "value": [ 22:11:59 DEBUG "0.32313333333334243" 22:11:59 DEBUG } 22:11:59 DEBUG } 22:11:59 DEBUG ] 22:11:59 DEBUG "0.05366666666665007" 22:11:59 DEBUG } 22:11:59 DEBUG 1713647519, 22:11:59 DEBUG "value": [ 22:11:59 DEBUG 22:11:59 DEBUG "value": [ 22:11:59 DEBUG "1280380928" 22:11:59 DEBUG 1713647519, 22:11:59 DEBUG }, 22:11:59 DEBUG "value": [ 22:11:59 DEBUG ] 22:11:59 DEBUG 22:11:59 DEBUG "0" 22:11:59 DEBUG 1713647519, 22:11:59 DEBUG ] 22:11:59 DEBUG } 22:11:59 DEBUG ] 22:11:59 DEBUG }, 22:11:59 DEBUG } 22:11:59 DEBUG ] 22:11:59 DEBUG "11.867723285963963" 22:11:59 DEBUG 1713647519, 22:11:59 DEBUG 1713647519, 22:11:59 DEBUG ] 22:11:59 DEBUG "0" 22:11:59 DEBUG 1713647519, 22:11:59 DEBUG } 22:11:59 DEBUG } 22:11:59 DEBUG "pod": "am-767f6b67b5-vbltz" 22:11:59 DEBUG ] 22:11:59 DEBUG "0" 22:11:59 DEBUG } 22:11:59 DEBUG } 22:11:59 DEBUG "value": [ 22:11:59 DEBUG } 22:11:59 DEBUG } 22:11:59 DEBUG ] 22:11:59 DEBUG "0" 22:11:59 DEBUG "0" 22:11:59 DEBUG } 22:11:59 DEBUG ] 22:11:59 DEBUG "4166.333333333333" 22:11:59 DEBUG ] 22:11:59 DEBUG ] 22:11:59 DEBUG }, 22:11:59 DEBUG } 22:11:59 DEBUG ] 22:11:59 DEBUG ] 22:11:59 DEBUG ] 22:11:59 DEBUG 1713647519, 22:11:59 DEBUG ] 22:11:59 DEBUG } 22:11:59 DEBUG ] 22:11:59 DEBUG ] 22:11:59 DEBUG ] 22:11:59 DEBUG } 22:11:59 DEBUG ] 22:11:59 DEBUG } 22:11:59 DEBUG } 22:11:59 DEBUG "value": [ 22:11:59 DEBUG ] 22:11:59 DEBUG } 22:11:59 DEBUG } 22:11:59 DEBUG } 22:11:59 DEBUG "0" 22:11:59 DEBUG } 22:11:59 DEBUG ] 22:11:59 DEBUG } 22:11:59 DEBUG } 22:11:59 DEBUG } 22:11:59 DEBUG ] 22:11:59 DEBUG } 22:11:59 DEBUG } 22:11:59 DEBUG } 22:11:59 DEBUG 1713647519, 22:11:59 DEBUG } 22:11:59 DEBUG ] 22:11:59 DEBUG } 22:11:59 DEBUG } 22:11:59 DEBUG ] 22:11:59 DEBUG } 22:11:59 DEBUG } 22:11:59 DEBUG ] 22:11:59 DEBUG ] 22:11:59 DEBUG } 22:11:59 DEBUG } 22:11:59 DEBUG ] 22:11:59 DEBUG "2566.633333333333" 22:11:59 DEBUG } 22:11:59 DEBUG } 22:11:59 DEBUG } 22:11:59 DEBUG } 22:11:59 DEBUG } 22:11:59 DEBUG } 22:11:59 DEBUG } 22:11:59 DEBUG } 22:11:59 DEBUG } 22:11:59 DEBUG } 22:11:59 DEBUG ] 22:11:59 DEBUG } 22:11:59 DEBUG } 22:11:59 DEBUG ] 22:11:59 DEBUG } 22:11:59 DEBUG } 22:11:59 DEBUG } 22:11:59 DEBUG ] 22:11:59 DEBUG } 22:11:59 DEBUG } 22:12:08 WARNING File /tmp/lodemon_data-20240420_213952/node_disk_read_bytes_total.json does not exist. Skipping average value computation... 127.0.0.1 - - [20/Apr/2024 22:12:08] "GET /monitoring/average?start_time=20240420_204254&stop_time=20240420_211007 HTTP/1.1" 200 - 22:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713647574" 22:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647574" 22:12:54 ERROR [http_cmd]: ERROR 22:12:54 ERROR --- status code --- 22:12:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:12:54 ERROR --- http response --- 22:12:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:12:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:12:54 INFO [http_cmd]: http status code OK 22:12:54 DEBUG --- status code --- 22:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:54 DEBUG --- http response --- 22:12:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:12:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647575" 22:12:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713647575" 22:12:55 INFO [http_cmd]: http status code OK 22:12:55 DEBUG --- status code --- 22:12:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:55 DEBUG --- http response --- 22:12:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:12:55 DEBUG ----- output ----- 22:12:55 DEBUG { 22:12:55 DEBUG "status": "success", 22:12:55 DEBUG "data": { 22:12:55 DEBUG "resultType": "vector", 22:12:55 DEBUG "result": [ 22:12:55 DEBUG { 22:12:55 DEBUG "metric": { 22:12:55 DEBUG "pod": "ds-idrepo-0" 22:12:55 DEBUG }, 22:12:55 DEBUG "value": [ 22:12:55 DEBUG 22:12:55 DEBUG 22:12:55 DEBUG 22:12:55 DEBUG [...] 22:12:55 DEBUG 22:12:55 DEBUG 22:12:55 DEBUG 22:12:55 DEBUG "pod": "ds-cts-2" 22:12:55 DEBUG }, 22:12:55 DEBUG "value": [ 22:12:55 DEBUG 1713647575, 22:12:55 DEBUG "0" 22:12:55 DEBUG ] 22:12:55 DEBUG } 22:12:55 DEBUG ] 22:12:55 DEBUG } 22:12:55 DEBUG } 22:12:55 INFO [http_cmd]: http status code OK 22:12:55 DEBUG --- status code --- 22:12:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:55 DEBUG --- http response --- 22:12:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 22:12:55 DEBUG ----- output ----- 22:12:55 DEBUG { 22:12:55 DEBUG "status": "success", 22:12:55 DEBUG "data": { 22:12:55 DEBUG "resultType": "vector", 22:12:55 DEBUG "result": [ 22:12:55 DEBUG { 22:12:55 DEBUG "metric": { 22:12:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:12:55 DEBUG }, 22:12:55 DEBUG "value": [ 22:12:55 DEBUG 22:12:55 DEBUG 22:12:55 DEBUG 22:12:55 DEBUG [...] 22:12:55 DEBUG 22:12:55 DEBUG 22:12:55 DEBUG 22:12:55 DEBUG "pod": "overseer-0-967d957f7-zk66v" 22:12:55 DEBUG }, 22:12:55 DEBUG "value": [ 22:12:55 DEBUG 1713647575, 22:12:55 DEBUG "11964245.333333332" 22:12:55 DEBUG ] 22:12:55 DEBUG } 22:12:55 DEBUG ] 22:12:55 DEBUG } 22:12:55 DEBUG } 22:12:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647576" 22:12:56 INFO [http_cmd]: http status code OK 22:12:56 DEBUG --- status code --- 22:12:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:56 DEBUG --- http response --- 22:12:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:12:56 DEBUG ----- output ----- 22:12:56 DEBUG { 22:12:56 DEBUG "status": "success", 22:12:56 DEBUG "data": { 22:12:56 DEBUG "resultType": "vector", 22:12:56 DEBUG "result": [ 22:12:56 DEBUG { 22:12:56 DEBUG "metric": { 22:12:56 DEBUG "pod": "am-767f6b67b5-9zw22" 22:12:56 DEBUG }, 22:12:56 DEBUG "value": [ 22:12:56 DEBUG 22:12:56 DEBUG 22:12:56 DEBUG 22:12:56 DEBUG [...] 22:12:56 DEBUG 22:12:56 DEBUG 22:12:56 DEBUG 22:12:56 DEBUG "pod": "am-767f6b67b5-vbltz" 22:12:56 DEBUG }, 22:12:56 DEBUG "value": [ 22:12:56 DEBUG 1713647576, 22:12:56 DEBUG "1015.6666666666666" 22:12:56 DEBUG ] 22:12:56 DEBUG } 22:12:56 DEBUG ] 22:12:56 DEBUG } 22:12:56 DEBUG } 22:12:57 INFO 22:12:57 INFO [loop_until]: kubectl --namespace=xlou top pods 22:12:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:12:57 INFO [loop_until]: OK (rc = 0) 22:12:57 DEBUG --- stdout --- 22:12:57 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 6841m 8809Mi am-767f6b67b5-qcqz2 5662m 8943Mi am-767f6b67b5-vbltz 9412m 8946Mi ds-cts-0 19450m 12314Mi ds-cts-1 18833m 12280Mi ds-cts-2 18954m 12469Mi ds-idrepo-0 3505m 3921Mi ds-idrepo-1 2277m 3966Mi ds-idrepo-2 2194m 3877Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-7dd48676f6-tdxl9 3m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-967d957f7-zk66v 5451m 715Mi 22:12:57 DEBUG --- stderr --- 22:12:57 DEBUG 22:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647579" 22:12:59 INFO [http_cmd]: http status code OK 22:12:59 DEBUG --- status code --- 22:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:59 DEBUG --- http response --- 22:12:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:12:59 DEBUG ----- output ----- 22:12:59 DEBUG { 22:12:59 DEBUG "status": "success", 22:12:59 DEBUG "data": { 22:12:59 DEBUG "resultType": "vector", 22:12:59 DEBUG "result": [ 22:12:59 DEBUG { 22:12:59 DEBUG "metric": { 22:12:59 DEBUG "instance": "10.142.0.107:9100" 22:12:59 DEBUG }, 22:12:59 DEBUG "value": [ 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG [...] 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG "instance": "10.142.0.43:9100" 22:12:59 DEBUG }, 22:12:59 DEBUG "value": [ 22:12:59 DEBUG 1713647579, 22:12:59 DEBUG "28788743.1" 22:12:59 DEBUG ] 22:12:59 DEBUG } 22:12:59 DEBUG ] 22:12:59 DEBUG } 22:12:59 DEBUG } 22:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647579" 22:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647579" 22:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647579" 22:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647579" 22:12:59 INFO 22:12:59 INFO [loop_until]: kubectl --namespace=xlou top node 22:12:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713647579" 22:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647579" 22:12:59 INFO [http_cmd]: http status code OK 22:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647579" 22:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647579" 22:12:59 DEBUG --- status code --- 22:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647579" 22:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713647579" 22:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647579" 22:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647579" 22:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713647579" 22:12:59 INFO [http_cmd]: http status code OK 22:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647579" 22:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647579" 22:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647579" 22:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647579" 22:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647579" 22:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647579" 22:12:59 INFO [http_cmd]: http status code OK 22:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647579" 22:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:59 INFO [http_cmd]: http status code OK 22:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647579" 22:12:59 INFO [http_cmd]: http status code OK 22:12:59 INFO [http_cmd]: http status code OK 22:12:59 INFO [http_cmd]: http status code OK 22:12:59 INFO [http_cmd]: http status code OK 22:12:59 DEBUG --- status code --- 22:12:59 INFO [http_cmd]: http status code OK 22:12:59 INFO [http_cmd]: http status code OK 22:12:59 INFO [http_cmd]: http status code OK 22:12:59 INFO [http_cmd]: http status code OK 22:12:59 DEBUG --- status code --- 22:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:59 INFO [http_cmd]: http status code OK 22:12:59 INFO [http_cmd]: http status code OK 22:12:59 DEBUG --- status code --- 22:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:59 DEBUG --- http response --- 22:12:59 DEBUG --- status code --- 22:12:59 DEBUG --- status code --- 22:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:59 INFO [http_cmd]: http status code OK 22:12:59 DEBUG --- status code --- 22:12:59 DEBUG --- status code --- 22:12:59 DEBUG --- status code --- 22:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:59 DEBUG --- status code --- 22:12:59 DEBUG --- status code --- 22:12:59 DEBUG --- status code --- 22:12:59 DEBUG --- http response --- 22:12:59 INFO [http_cmd]: http status code OK 22:12:59 DEBUG --- status code --- 22:12:59 DEBUG --- status code --- 22:12:59 INFO [http_cmd]: http status code OK 22:12:59 INFO [http_cmd]: http status code OK 22:12:59 DEBUG --- http response --- 22:12:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:59 DEBUG --- http response --- 22:12:59 INFO [http_cmd]: http status code OK 22:12:59 DEBUG --- status code --- 22:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:59 DEBUG --- http response --- 22:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:59 DEBUG --- http response --- 22:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:12:59 DEBUG --- status code --- 22:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:59 DEBUG --- status code --- 22:12:59 DEBUG --- status code --- 22:12:59 DEBUG ----- output ----- 22:12:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:12:59 DEBUG --- http response --- 22:12:59 DEBUG --- status code --- 22:12:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:12:59 DEBUG --- http response --- 22:12:59 INFO [http_cmd]: http status code OK 22:12:59 DEBUG --- status code --- 22:12:59 DEBUG --- http response --- 22:12:59 DEBUG --- http response --- 22:12:59 DEBUG --- http response --- 22:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:12:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:12:59 DEBUG --- http response --- 22:12:59 DEBUG ----- output ----- 22:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:59 DEBUG --- http response --- 22:12:59 DEBUG --- http response --- 22:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:59 DEBUG { 22:12:59 DEBUG ----- output ----- 22:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:12:59 DEBUG ----- output ----- 22:12:59 INFO [http_cmd]: http status code OK 22:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:12:59 DEBUG ----- output ----- 22:12:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:12:59 DEBUG --- http response --- 22:12:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:12:59 DEBUG ----- output ----- 22:12:59 DEBUG ----- output ----- 22:12:59 DEBUG { 22:12:59 DEBUG { 22:12:59 DEBUG --- http response --- 22:12:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:12:59 DEBUG ----- output ----- 22:12:59 DEBUG --- http response --- 22:12:59 DEBUG --- http response --- 22:12:59 DEBUG "status": "success", 22:12:59 DEBUG { 22:12:59 DEBUG --- http response --- 22:12:59 DEBUG ----- output ----- 22:12:59 DEBUG { 22:12:59 DEBUG --- status code --- 22:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:59 DEBUG --- http response --- 22:12:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:12:59 DEBUG ----- output ----- 22:12:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:12:59 DEBUG ----- output ----- 22:12:59 DEBUG { 22:12:59 DEBUG "status": "success", 22:12:59 DEBUG "data": { 22:12:59 DEBUG "status": "success", 22:12:59 DEBUG { 22:12:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:12:59 DEBUG ----- output ----- 22:12:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:12:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:12:59 DEBUG "data": { 22:12:59 DEBUG "status": "success", 22:12:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:12:59 DEBUG { 22:12:59 DEBUG "status": "success", 22:12:59 DEBUG --- http response --- 22:12:59 DEBUG { 22:12:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:12:59 DEBUG ----- output ----- 22:12:59 DEBUG { 22:12:59 DEBUG ----- output ----- 22:12:59 DEBUG { 22:12:59 DEBUG "status": "success", 22:12:59 DEBUG "resultType": "vector", 22:12:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:12:59 DEBUG "data": { 22:12:59 DEBUG "status": "success", 22:12:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:12:59 DEBUG { 22:12:59 DEBUG ----- output ----- 22:12:59 DEBUG ----- output ----- 22:12:59 DEBUG "resultType": "vector", 22:12:59 DEBUG "data": { 22:12:59 DEBUG ----- output ----- 22:12:59 DEBUG "status": "success", 22:12:59 DEBUG "data": { 22:12:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:12:59 DEBUG "status": "success", 22:12:59 DEBUG ----- output ----- 22:12:59 DEBUG { 22:12:59 DEBUG "status": "success", 22:12:59 DEBUG { 22:12:59 DEBUG "status": "success", 22:12:59 DEBUG "data": { 22:12:59 DEBUG "resultType": "vector", 22:12:59 DEBUG ----- output ----- 22:12:59 DEBUG "resultType": "vector", 22:12:59 DEBUG "result": [ 22:12:59 DEBUG ----- output ----- 22:12:59 DEBUG "status": "success", 22:12:59 DEBUG { 22:12:59 DEBUG "status": "success", 22:12:59 DEBUG "result": [ 22:12:59 DEBUG "resultType": "vector", 22:12:59 DEBUG { 22:12:59 DEBUG "data": { 22:12:59 DEBUG "resultType": "vector", 22:12:59 DEBUG ----- output ----- 22:12:59 DEBUG "data": { 22:12:59 DEBUG { 22:12:59 DEBUG "status": "success", 22:12:59 DEBUG "data": { 22:12:59 DEBUG "status": "success", 22:12:59 DEBUG "result": [ 22:12:59 DEBUG "data": { 22:12:59 DEBUG "result": [ 22:12:59 DEBUG { 22:12:59 DEBUG { 22:12:59 DEBUG "metric": { 22:12:59 DEBUG { 22:12:59 DEBUG "data": { 22:12:59 DEBUG { 22:12:59 DEBUG "data": { 22:12:59 DEBUG { 22:12:59 DEBUG "result": [ 22:12:59 DEBUG { 22:12:59 DEBUG "resultType": "vector", 22:12:59 DEBUG "result": [ 22:12:59 DEBUG { 22:12:59 DEBUG "resultType": "vector", 22:12:59 DEBUG "status": "success", 22:12:59 DEBUG "data": { 22:12:59 DEBUG "resultType": "vector", 22:12:59 DEBUG "data": { 22:12:59 DEBUG { 22:12:59 DEBUG "resultType": "vector", 22:12:59 DEBUG { 22:12:59 DEBUG "status": "success", 22:12:59 DEBUG "instance": "10.142.0.107:9100" 22:12:59 DEBUG "data": { 22:12:59 DEBUG "status": "success", 22:12:59 DEBUG "resultType": "vector", 22:12:59 DEBUG "status": "success", 22:12:59 DEBUG "resultType": "vector", 22:12:59 DEBUG "metric": { 22:12:59 DEBUG "metric": { 22:12:59 DEBUG "status": "success", 22:12:59 DEBUG "result": [ 22:12:59 DEBUG { 22:12:59 DEBUG "status": "success", 22:12:59 DEBUG "result": [ 22:12:59 DEBUG "data": { 22:12:59 DEBUG "resultType": "vector", 22:12:59 DEBUG "result": [ 22:12:59 DEBUG "resultType": "vector", 22:12:59 DEBUG "metric": { 22:12:59 DEBUG "result": [ 22:12:59 DEBUG { 22:12:59 DEBUG "data": { 22:12:59 DEBUG }, 22:12:59 DEBUG "resultType": "vector", 22:12:59 DEBUG "data": { 22:12:59 DEBUG "result": [ 22:12:59 DEBUG "data": { 22:12:59 DEBUG "result": [ 22:12:59 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:12:59 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:12:59 DEBUG "data": { 22:12:59 DEBUG { 22:12:59 DEBUG "metric": { 22:12:59 DEBUG "data": { 22:12:59 DEBUG { 22:12:59 DEBUG "resultType": "vector", 22:12:59 DEBUG "result": [ 22:12:59 DEBUG { 22:12:59 DEBUG "result": [ 22:12:59 DEBUG "pod": "ds-cts-0" 22:12:59 DEBUG }, 22:12:59 DEBUG "metric": { 22:12:59 DEBUG "resultType": "vector", 22:12:59 DEBUG "value": [ 22:12:59 DEBUG "result": [ 22:12:59 DEBUG "resultType": "vector", 22:12:59 DEBUG { 22:12:59 DEBUG "resultType": "vector", 22:12:59 DEBUG { 22:12:59 DEBUG }, 22:12:59 DEBUG }, 22:12:59 DEBUG "resultType": "vector", 22:12:59 DEBUG "metric": { 22:12:59 DEBUG "instance": "10.142.0.107:9100" 22:12:59 DEBUG "resultType": "vector", 22:12:59 DEBUG "metric": { 22:12:59 DEBUG "result": [ 22:12:59 DEBUG { 22:12:59 DEBUG "metric": { 22:12:59 DEBUG { 22:12:59 DEBUG "value": [ 22:12:59 DEBUG "metric": { 22:12:59 DEBUG "pod": "ds-cts-0" 22:12:59 DEBUG "result": [ 22:12:59 DEBUG 22:12:59 DEBUG { 22:12:59 DEBUG "result": [ 22:12:59 DEBUG "metric": { 22:12:59 DEBUG "result": [ 22:12:59 DEBUG "metric": { 22:12:59 DEBUG "value": [ 22:12:59 DEBUG "value": [ 22:12:59 DEBUG "result": [ 22:12:59 DEBUG "pod": "am-767f6b67b5-9zw22" 22:12:59 DEBUG }, 22:12:59 DEBUG "result": [ 22:12:59 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:12:59 DEBUG { 22:12:59 DEBUG "pod": "ds-cts-0" 22:12:59 DEBUG "metric": { 22:12:59 DEBUG "metric": { 22:12:59 DEBUG 22:12:59 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:12:59 DEBUG }, 22:12:59 DEBUG { 22:12:59 DEBUG 22:12:59 DEBUG { 22:12:59 DEBUG "metric": { 22:12:59 DEBUG "instance": "10.142.0.107:9100" 22:12:59 DEBUG { 22:12:59 DEBUG "pod": "ds-idrepo-0" 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG { 22:12:59 DEBUG "value": [ 22:12:59 DEBUG }, 22:12:59 DEBUG { 22:12:59 DEBUG }, 22:12:59 DEBUG "metric": { 22:12:59 DEBUG "pod": "am-767f6b67b5-9zw22" 22:12:59 DEBUG "pod": "am-767f6b67b5-9zw22" 22:12:59 DEBUG }, 22:12:59 DEBUG 22:12:59 DEBUG "pod": "ds-idrepo-0" 22:12:59 DEBUG }, 22:12:59 DEBUG "metric": { 22:12:59 DEBUG 22:12:59 DEBUG "metric": { 22:12:59 DEBUG "pod": "am-767f6b67b5-9zw22" 22:12:59 DEBUG }, 22:12:59 DEBUG "value": [ 22:12:59 DEBUG }, 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG "value": [ 22:12:59 DEBUG "metric": { 22:12:59 DEBUG "value": [ 22:12:59 DEBUG 22:12:59 DEBUG }, 22:12:59 DEBUG "value": [ 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG "value": [ 22:12:59 DEBUG "value": [ 22:12:59 DEBUG [...] 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG "instance": "10.142.0.43:9100" 22:12:59 DEBUG }, 22:12:59 DEBUG "value": [ 22:12:59 DEBUG }, 22:12:59 DEBUG 22:12:59 DEBUG "metric": { 22:12:59 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:12:59 DEBUG 22:12:59 DEBUG "pod": "ds-idrepo-0" 22:12:59 DEBUG }, 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG [...] 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG "metric": { 22:12:59 DEBUG 22:12:59 DEBUG "instance": "10.142.0.107:9100" 22:12:59 DEBUG "value": [ 22:12:59 DEBUG 22:12:59 DEBUG [...] 22:12:59 DEBUG [...] 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 INFO [loop_until]: OK (rc = 0) 22:12:59 DEBUG }, 22:12:59 DEBUG 22:12:59 DEBUG }, 22:12:59 DEBUG "value": [ 22:12:59 DEBUG [...] 22:12:59 DEBUG "value": [ 22:12:59 DEBUG 22:12:59 DEBUG }, 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG 1713647579, 22:12:59 DEBUG "pod": "am-767f6b67b5-9zw22" 22:12:59 DEBUG 22:12:59 DEBUG }, 22:12:59 DEBUG 22:12:59 DEBUG "value": [ 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG "pod": "am-767f6b67b5-9zw22" 22:12:59 DEBUG 22:12:59 DEBUG --- stdout --- 22:12:59 DEBUG "value": [ 22:12:59 DEBUG 22:12:59 DEBUG "value": [ 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG [...] 22:12:59 DEBUG "value": [ 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG [...] 22:12:59 DEBUG "0.09299999999999307" 22:12:59 DEBUG }, 22:12:59 DEBUG "value": [ 22:12:59 DEBUG "value": [ 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG }, 22:12:59 DEBUG [...] 22:12:59 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 6206Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 2760m 9% 7680Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 8055m 26% 11545Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 81m 0% 3365Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 19416m 65% 15555Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 149m 0% 5181Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 11302m 37% 11860Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 7064m 23% 3042Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 8953m 29% 11190Mi 10% gke-xlou-cdm-ds-0f601751-j488 3351m 11% 7293Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 18632m 62% 17356Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 3006m 10% 7964Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 18593m 62% 15173Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 5730m 36% 8604Mi 14% 22:12:59 DEBUG 22:12:59 DEBUG [...] 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG "pod": "ds-cts-2" 22:12:59 DEBUG [...] 22:12:59 DEBUG ] 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG [...] 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG "pod": "overseer-0-967d957f7-zk66v" 22:12:59 DEBUG "value": [ 22:12:59 DEBUG 22:12:59 DEBUG --- stderr --- 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG "pod": "overseer-0-967d957f7-zk66v" 22:12:59 DEBUG }, 22:12:59 DEBUG 22:12:59 DEBUG }, 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG } 22:12:59 DEBUG 22:12:59 DEBUG [...] 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG [...] 22:12:59 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG [...] 22:12:59 DEBUG 22:12:59 DEBUG "value": [ 22:12:59 DEBUG [...] 22:12:59 DEBUG 22:12:59 DEBUG "value": [ 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG ] 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG }, 22:12:59 DEBUG 22:12:59 DEBUG }, 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG [...] 22:12:59 DEBUG 22:12:59 DEBUG [...] 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG 1713647579, 22:12:59 DEBUG "1703.578792383379" 22:12:59 DEBUG [...] 22:12:59 DEBUG 22:12:59 DEBUG 1713647579, 22:12:59 DEBUG 22:12:59 DEBUG } 22:12:59 DEBUG [...] 22:12:59 DEBUG 22:12:59 DEBUG [...] 22:12:59 DEBUG 22:12:59 DEBUG "value": [ 22:12:59 DEBUG 22:12:59 DEBUG "value": [ 22:12:59 DEBUG 22:12:59 DEBUG "pod": "am-767f6b67b5-vbltz" 22:12:59 DEBUG 22:12:59 DEBUG "instance": "10.142.0.43:9100" 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG "pod": "am-767f6b67b5-vbltz" 22:12:59 DEBUG ] 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG "27.8" 22:12:59 DEBUG "pod": "ds-idrepo-2" 22:12:59 DEBUG } 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG "pod": "ds-idrepo-2" 22:12:59 DEBUG 1713647579, 22:12:59 DEBUG 22:12:59 DEBUG 1713647579, 22:12:59 DEBUG [...] 22:12:59 DEBUG }, 22:12:59 DEBUG 22:12:59 DEBUG }, 22:12:59 DEBUG "pod": "am-767f6b67b5-vbltz" 22:12:59 DEBUG }, 22:12:59 DEBUG "value": [ 22:12:59 DEBUG } 22:12:59 DEBUG 22:12:59 DEBUG 22:12:59 DEBUG ] 22:12:59 DEBUG }, 22:12:59 DEBUG 22:12:59 DEBUG "pod": "am-767f6b67b5-vbltz" 22:12:59 DEBUG 22:12:59 DEBUG }, 22:12:59 DEBUG "0.7112861297468767" 22:12:59 DEBUG "instance": "10.142.0.43:9100" 22:12:59 DEBUG 22:12:59 DEBUG "1.1912070518080582" 22:12:59 DEBUG 22:12:59 DEBUG "value": [ 22:12:59 DEBUG 1713647579, 22:12:59 DEBUG "value": [ 22:12:59 DEBUG }, 22:12:59 DEBUG 22:12:59 DEBUG 1713647579, 22:12:59 DEBUG ] 22:12:59 DEBUG "pod": "ds-idrepo-2" 22:12:59 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:12:59 DEBUG }, 22:12:59 DEBUG "value": [ 22:12:59 DEBUG 22:12:59 DEBUG }, 22:12:59 DEBUG 22:12:59 DEBUG "value": [ 22:12:59 DEBUG ] 22:12:59 DEBUG }, 22:12:59 DEBUG 22:12:59 DEBUG ] 22:12:59 DEBUG 22:12:59 DEBUG "84.26249999999999" 22:12:59 DEBUG 22:12:59 DEBUG 1713647579, 22:12:59 DEBUG "4094826228736" 22:12:59 DEBUG "pod": "ds-cts-2" 22:12:59 DEBUG }, 22:12:59 DEBUG } 22:12:59 DEBUG }, 22:12:59 DEBUG "value": [ 22:12:59 DEBUG "value": [ 22:12:59 DEBUG 1713647579, 22:12:59 DEBUG "pod": "am-767f6b67b5-vbltz" 22:12:59 DEBUG "value": [ 22:12:59 DEBUG "instance": "10.142.0.43:9100" 22:12:59 DEBUG 1713647579, 22:12:59 DEBUG } 22:12:59 DEBUG "value": [ 22:12:59 DEBUG 22:12:59 DEBUG } 22:12:59 DEBUG 22:12:59 DEBUG ] 22:12:59 DEBUG "pod": "overseer-0-967d957f7-zk66v" 22:12:59 DEBUG ] 22:12:59 DEBUG "442823.5" 22:12:59 DEBUG "value": [ 22:12:59 DEBUG } 22:12:59 DEBUG } 22:12:59 DEBUG "value": [ 22:12:59 DEBUG 1713647579, 22:12:59 DEBUG 1713647579, 22:12:59 DEBUG "0" 22:12:59 DEBUG }, 22:12:59 DEBUG 1713647579, 22:12:59 DEBUG "1017.0666666666667" 22:12:59 DEBUG "0" 22:12:59 DEBUG ] 22:12:59 DEBUG } 22:12:59 DEBUG "pod": "ds-cts-2" 22:12:59 DEBUG ] 22:12:59 DEBUG "pod": "am-767f6b67b5-vbltz" 22:12:59 DEBUG } 22:12:59 DEBUG }, 22:12:59 DEBUG } 22:12:59 DEBUG ] 22:12:59 DEBUG 1713647579, 22:12:59 DEBUG ] 22:12:59 DEBUG } 22:12:59 DEBUG 1713647579, 22:12:59 DEBUG "647114752" 22:12:59 DEBUG ] 22:12:59 DEBUG "value": [ 22:12:59 DEBUG }, 22:12:59 DEBUG ] 22:12:59 DEBUG ] 22:12:59 DEBUG } 22:12:59 DEBUG 1713647579, 22:12:59 DEBUG }, 22:12:59 DEBUG } 22:12:59 DEBUG }, 22:12:59 DEBUG ] 22:12:59 DEBUG "value": [ 22:12:59 DEBUG ] 22:12:59 DEBUG } 22:12:59 DEBUG "0.13063333333333277" 22:12:59 DEBUG "1014.0333333333333" 22:12:59 DEBUG } 22:12:59 DEBUG "0" 22:12:59 DEBUG ] 22:12:59 DEBUG } 22:12:59 DEBUG 1713647579, 22:12:59 DEBUG "value": [ 22:12:59 DEBUG } 22:12:59 DEBUG ] 22:12:59 DEBUG "28795726.133333333" 22:12:59 DEBUG "value": [ 22:12:59 DEBUG } 22:12:59 DEBUG "value": [ 22:12:59 DEBUG } 22:12:59 DEBUG 1713647579, 22:12:59 DEBUG } 22:12:59 DEBUG ] 22:12:59 DEBUG ] 22:12:59 DEBUG ] 22:12:59 DEBUG ] 22:12:59 DEBUG } 22:12:59 DEBUG ] 22:12:59 DEBUG "0" 22:12:59 DEBUG 1713647579, 22:12:59 DEBUG } 22:12:59 DEBUG } 22:12:59 DEBUG ] 22:12:59 DEBUG 1713647579, 22:12:59 DEBUG 1713647579, 22:12:59 DEBUG } 22:12:59 DEBUG "0" 22:12:59 DEBUG } 22:12:59 DEBUG } 22:12:59 DEBUG } 22:12:59 DEBUG } 22:12:59 DEBUG } 22:12:59 DEBUG ] 22:12:59 DEBUG } 22:12:59 DEBUG ] 22:12:59 DEBUG "0.4846333333333954" 22:12:59 DEBUG } 22:12:59 DEBUG ] 22:12:59 DEBUG } 22:12:59 DEBUG "0" 22:12:59 DEBUG "4154.099999999999" 22:12:59 DEBUG ] 22:12:59 DEBUG } 22:12:59 DEBUG ] 22:12:59 DEBUG ] 22:12:59 DEBUG ] 22:12:59 DEBUG } 22:12:59 DEBUG } 22:12:59 DEBUG } 22:12:59 DEBUG ] 22:12:59 DEBUG ] 22:12:59 DEBUG ] 22:12:59 DEBUG ] 22:12:59 DEBUG ] 22:12:59 DEBUG } 22:12:59 DEBUG } 22:12:59 DEBUG } 22:12:59 DEBUG } 22:12:59 DEBUG } 22:12:59 DEBUG } 22:12:59 DEBUG } 22:12:59 DEBUG } 22:12:59 DEBUG } 22:12:59 DEBUG } 22:12:59 DEBUG } 22:12:59 DEBUG ] 22:12:59 DEBUG } 22:12:59 DEBUG } 22:12:59 DEBUG } 22:12:59 DEBUG } 22:12:59 DEBUG } 22:12:59 DEBUG ] 22:12:59 DEBUG } 22:12:59 DEBUG ] 22:12:59 DEBUG ] 22:12:59 DEBUG } 22:12:59 DEBUG } 22:12:59 DEBUG } 22:12:59 DEBUG } 22:12:59 DEBUG } 22:12:59 DEBUG } 22:12:59 DEBUG } 22:12:59 DEBUG } 22:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713647634" 22:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647634" 22:13:54 ERROR [http_cmd]: ERROR 22:13:54 ERROR --- status code --- 22:13:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:13:54 ERROR --- http response --- 22:13:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:13:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:13:54 INFO [http_cmd]: http status code OK 22:13:54 DEBUG --- status code --- 22:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:54 DEBUG --- http response --- 22:13:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:13:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713647635" 22:13:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647635" 22:13:55 INFO [http_cmd]: http status code OK 22:13:55 DEBUG --- status code --- 22:13:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:55 DEBUG --- http response --- 22:13:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:13:55 DEBUG ----- output ----- 22:13:55 DEBUG { 22:13:55 DEBUG "status": "success", 22:13:55 DEBUG "data": { 22:13:55 DEBUG "resultType": "vector", 22:13:55 DEBUG "result": [ 22:13:55 DEBUG { 22:13:55 DEBUG "metric": { 22:13:55 DEBUG "pod": "ds-idrepo-0" 22:13:55 DEBUG }, 22:13:55 DEBUG "value": [ 22:13:55 DEBUG 22:13:55 DEBUG 22:13:55 DEBUG 22:13:55 DEBUG [...] 22:13:55 DEBUG 22:13:55 DEBUG 22:13:55 DEBUG 22:13:55 DEBUG "pod": "ds-cts-2" 22:13:55 DEBUG }, 22:13:55 DEBUG "value": [ 22:13:55 DEBUG 1713647635, 22:13:55 DEBUG "0" 22:13:55 DEBUG ] 22:13:55 DEBUG } 22:13:55 DEBUG ] 22:13:55 DEBUG } 22:13:55 DEBUG } 22:13:55 INFO [http_cmd]: http status code OK 22:13:55 DEBUG --- status code --- 22:13:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:55 DEBUG --- http response --- 22:13:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 22:13:55 DEBUG ----- output ----- 22:13:55 DEBUG { 22:13:55 DEBUG "status": "success", 22:13:55 DEBUG "data": { 22:13:55 DEBUG "resultType": "vector", 22:13:55 DEBUG "result": [ 22:13:55 DEBUG { 22:13:55 DEBUG "metric": { 22:13:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:13:55 DEBUG }, 22:13:55 DEBUG "value": [ 22:13:55 DEBUG 22:13:55 DEBUG 22:13:55 DEBUG 22:13:55 DEBUG [...] 22:13:55 DEBUG 22:13:55 DEBUG 22:13:55 DEBUG 22:13:55 DEBUG "pod": "overseer-0-967d957f7-zk66v" 22:13:55 DEBUG }, 22:13:55 DEBUG "value": [ 22:13:55 DEBUG 1713647635, 22:13:55 DEBUG "0" 22:13:55 DEBUG ] 22:13:55 DEBUG } 22:13:55 DEBUG ] 22:13:55 DEBUG } 22:13:55 DEBUG } 22:13:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647636" 22:13:56 INFO [http_cmd]: http status code OK 22:13:56 DEBUG --- status code --- 22:13:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:56 DEBUG --- http response --- 22:13:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:13:56 DEBUG ----- output ----- 22:13:56 DEBUG { 22:13:56 DEBUG "status": "success", 22:13:56 DEBUG "data": { 22:13:56 DEBUG "resultType": "vector", 22:13:56 DEBUG "result": [ 22:13:56 DEBUG { 22:13:56 DEBUG "metric": { 22:13:56 DEBUG "pod": "am-767f6b67b5-9zw22" 22:13:56 DEBUG }, 22:13:56 DEBUG "value": [ 22:13:56 DEBUG 22:13:56 DEBUG 22:13:56 DEBUG 22:13:56 DEBUG [...] 22:13:56 DEBUG 22:13:56 DEBUG 22:13:56 DEBUG 22:13:56 DEBUG "pod": "am-767f6b67b5-vbltz" 22:13:56 DEBUG }, 22:13:56 DEBUG "value": [ 22:13:56 DEBUG 1713647636, 22:13:56 DEBUG "3472.4666666666667" 22:13:56 DEBUG ] 22:13:56 DEBUG } 22:13:56 DEBUG ] 22:13:56 DEBUG } 22:13:56 DEBUG } 22:13:57 INFO 22:13:57 INFO [loop_until]: kubectl --namespace=xlou top pods 22:13:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:13:57 INFO [loop_until]: OK (rc = 0) 22:13:57 DEBUG --- stdout --- 22:13:57 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 24599m 8827Mi am-767f6b67b5-qcqz2 24632m 8973Mi am-767f6b67b5-vbltz 24682m 8982Mi ds-cts-0 20409m 12323Mi ds-cts-1 19248m 12285Mi ds-cts-2 20420m 12474Mi ds-idrepo-0 10724m 3923Mi ds-idrepo-1 10225m 3966Mi ds-idrepo-2 10512m 3878Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 5m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-7dd48676f6-tdxl9 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-967d957f7-zk66v 13606m 2198Mi 22:13:57 DEBUG --- stderr --- 22:13:57 DEBUG 22:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647639" 22:13:59 INFO [http_cmd]: http status code OK 22:13:59 DEBUG --- status code --- 22:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:59 DEBUG --- http response --- 22:13:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:13:59 DEBUG ----- output ----- 22:13:59 DEBUG { 22:13:59 DEBUG "status": "success", 22:13:59 DEBUG "data": { 22:13:59 DEBUG "resultType": "vector", 22:13:59 DEBUG "result": [ 22:13:59 DEBUG { 22:13:59 DEBUG "metric": { 22:13:59 DEBUG "instance": "10.142.0.107:9100" 22:13:59 DEBUG }, 22:13:59 DEBUG "value": [ 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG [...] 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG "instance": "10.142.0.43:9100" 22:13:59 DEBUG }, 22:13:59 DEBUG "value": [ 22:13:59 DEBUG 1713647639, 22:13:59 DEBUG "107195317.33333334" 22:13:59 DEBUG ] 22:13:59 DEBUG } 22:13:59 DEBUG ] 22:13:59 DEBUG } 22:13:59 DEBUG } 22:13:59 INFO 22:13:59 INFO [loop_until]: kubectl --namespace=xlou top node 22:13:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647639" 22:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647639" 22:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647639" 22:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647639" 22:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713647639" 22:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713647639" 22:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647639" 22:13:59 INFO [http_cmd]: http status code OK 22:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647639" 22:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647639" 22:13:59 DEBUG --- status code --- 22:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647639" 22:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713647639" 22:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647639" 22:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647639" 22:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647639" 22:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647639" 22:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647639" 22:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647639" 22:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647639" 22:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647639" 22:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647639" 22:13:59 INFO [http_cmd]: http status code OK 22:13:59 INFO [http_cmd]: http status code OK 22:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647639" 22:13:59 INFO [http_cmd]: http status code OK 22:13:59 INFO [http_cmd]: http status code OK 22:13:59 INFO [http_cmd]: http status code OK 22:13:59 DEBUG --- status code --- 22:13:59 INFO [http_cmd]: http status code OK 22:13:59 INFO [http_cmd]: http status code OK 22:13:59 INFO [http_cmd]: http status code OK 22:13:59 DEBUG --- http response --- 22:13:59 INFO [http_cmd]: http status code OK 22:13:59 INFO [http_cmd]: http status code OK 22:13:59 INFO [http_cmd]: http status code OK 22:13:59 DEBUG --- status code --- 22:13:59 DEBUG --- status code --- 22:13:59 INFO [http_cmd]: http status code OK 22:13:59 INFO [http_cmd]: http status code OK 22:13:59 DEBUG --- status code --- 22:13:59 DEBUG --- status code --- 22:13:59 DEBUG --- status code --- 22:13:59 INFO [http_cmd]: http status code OK 22:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:59 INFO [http_cmd]: http status code OK 22:13:59 DEBUG --- status code --- 22:13:59 DEBUG --- status code --- 22:13:59 DEBUG --- status code --- 22:13:59 DEBUG --- status code --- 22:13:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:13:59 INFO [http_cmd]: http status code OK 22:13:59 DEBUG --- status code --- 22:13:59 DEBUG --- status code --- 22:13:59 INFO [http_cmd]: http status code OK 22:13:59 INFO [http_cmd]: http status code OK 22:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:59 DEBUG --- status code --- 22:13:59 DEBUG --- status code --- 22:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:59 DEBUG --- http response --- 22:13:59 INFO [http_cmd]: http status code OK 22:13:59 DEBUG --- http response --- 22:13:59 DEBUG --- status code --- 22:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:59 DEBUG ----- output ----- 22:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:59 DEBUG --- status code --- 22:13:59 INFO [http_cmd]: http status code OK 22:13:59 DEBUG --- status code --- 22:13:59 DEBUG --- http response --- 22:13:59 DEBUG --- http response --- 22:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:59 DEBUG --- http response --- 22:13:59 DEBUG --- http response --- 22:13:59 DEBUG --- status code --- 22:13:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:13:59 DEBUG --- status code --- 22:13:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:13:59 DEBUG --- http response --- 22:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:59 DEBUG --- http response --- 22:13:59 DEBUG --- http response --- 22:13:59 DEBUG --- http response --- 22:13:59 DEBUG { 22:13:59 DEBUG --- http response --- 22:13:59 DEBUG --- http response --- 22:13:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:13:59 DEBUG --- status code --- 22:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:13:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:13:59 INFO [loop_until]: OK (rc = 0) 22:13:59 DEBUG --- http response --- 22:13:59 DEBUG --- http response --- 22:13:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:13:59 DEBUG ----- output ----- 22:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:59 DEBUG ----- output ----- 22:13:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:13:59 DEBUG --- http response --- 22:13:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:13:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:13:59 DEBUG "status": "success", 22:13:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:13:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:13:59 DEBUG ----- output ----- 22:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:59 DEBUG --- http response --- 22:13:59 DEBUG ----- output ----- 22:13:59 DEBUG ----- output ----- 22:13:59 DEBUG --- stdout --- 22:13:59 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 6214Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 10818m 36% 7681Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 24839m 83% 11572Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 78m 0% 3357Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 20150m 67% 15556Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 152m 0% 5211Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 24784m 83% 11937Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 24137m 80% 3338Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 25790m 86% 11290Mi 10% gke-xlou-cdm-ds-0f601751-j488 10892m 36% 7300Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19591m 65% 17371Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 10711m 35% 7954Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20400m 68% 15177Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 13958m 87% 10633Mi 18% 22:13:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:13:59 DEBUG ----- output ----- 22:13:59 DEBUG --- http response --- 22:13:59 DEBUG ----- output ----- 22:13:59 DEBUG { 22:13:59 DEBUG --- http response --- 22:13:59 DEBUG { 22:13:59 DEBUG ----- output ----- 22:13:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:13:59 DEBUG ----- output ----- 22:13:59 DEBUG ----- output ----- 22:13:59 DEBUG { 22:13:59 DEBUG ----- output ----- 22:13:59 DEBUG ----- output ----- 22:13:59 DEBUG { 22:13:59 DEBUG --- http response --- 22:13:59 DEBUG --- http response --- 22:13:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:13:59 DEBUG { 22:13:59 DEBUG { 22:13:59 DEBUG "status": "success", 22:13:59 DEBUG "data": { 22:13:59 DEBUG { 22:13:59 DEBUG ----- output ----- 22:13:59 DEBUG { 22:13:59 DEBUG "status": "success", 22:13:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:13:59 DEBUG ----- output ----- 22:13:59 DEBUG "status": "success", 22:13:59 DEBUG { 22:13:59 DEBUG ----- output ----- 22:13:59 DEBUG { 22:13:59 DEBUG "status": "success", 22:13:59 DEBUG "data": { 22:13:59 DEBUG { 22:13:59 DEBUG "status": "success", 22:13:59 DEBUG "status": "success", 22:13:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:13:59 DEBUG ----- output ----- 22:13:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:13:59 DEBUG "status": "success", 22:13:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:13:59 DEBUG ----- output ----- 22:13:59 DEBUG --- stderr --- 22:13:59 DEBUG 22:13:59 DEBUG { 22:13:59 DEBUG "status": "success", 22:13:59 DEBUG "data": { 22:13:59 DEBUG { 22:13:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:13:59 DEBUG "data": { 22:13:59 DEBUG "status": "success", 22:13:59 DEBUG { 22:13:59 DEBUG "status": "success", 22:13:59 DEBUG "data": { 22:13:59 DEBUG "resultType": "vector", 22:13:59 DEBUG "data": { 22:13:59 DEBUG { 22:13:59 DEBUG "status": "success", 22:13:59 DEBUG ----- output ----- 22:13:59 DEBUG { 22:13:59 DEBUG ----- output ----- 22:13:59 DEBUG "data": { 22:13:59 DEBUG "resultType": "vector", 22:13:59 DEBUG "resultType": "vector", 22:13:59 DEBUG "status": "success", 22:13:59 DEBUG "status": "success", 22:13:59 DEBUG "data": { 22:13:59 DEBUG "resultType": "vector", 22:13:59 DEBUG "status": "success", 22:13:59 DEBUG ----- output ----- 22:13:59 DEBUG "resultType": "vector", 22:13:59 DEBUG "data": { 22:13:59 DEBUG "status": "success", 22:13:59 DEBUG "data": { 22:13:59 DEBUG "resultType": "vector", 22:13:59 DEBUG "result": [ 22:13:59 DEBUG "resultType": "vector", 22:13:59 DEBUG "data": { 22:13:59 DEBUG "data": { 22:13:59 DEBUG { 22:13:59 DEBUG "status": "success", 22:13:59 DEBUG { 22:13:59 DEBUG "status": "success", 22:13:59 DEBUG "result": [ 22:13:59 DEBUG "result": [ 22:13:59 DEBUG "data": { 22:13:59 DEBUG "data": { 22:13:59 DEBUG "resultType": "vector", 22:13:59 DEBUG "result": [ 22:13:59 DEBUG "data": { 22:13:59 DEBUG "resultType": "vector", 22:13:59 DEBUG "result": [ 22:13:59 DEBUG "resultType": "vector", 22:13:59 DEBUG "data": { 22:13:59 DEBUG "resultType": "vector", 22:13:59 DEBUG "result": [ 22:13:59 DEBUG { 22:13:59 DEBUG "result": [ 22:13:59 DEBUG "resultType": "vector", 22:13:59 DEBUG "resultType": "vector", 22:13:59 DEBUG "status": "success", 22:13:59 DEBUG "data": { 22:13:59 DEBUG { 22:13:59 DEBUG "data": { 22:13:59 DEBUG { 22:13:59 DEBUG { 22:13:59 DEBUG "resultType": "vector", 22:13:59 DEBUG "resultType": "vector", 22:13:59 DEBUG "result": [ 22:13:59 DEBUG { 22:13:59 DEBUG "result": [ 22:13:59 DEBUG { 22:13:59 DEBUG { 22:13:59 DEBUG "result": [ 22:13:59 DEBUG "result": [ 22:13:59 DEBUG "resultType": "vector", 22:13:59 DEBUG { 22:13:59 DEBUG "metric": { 22:13:59 DEBUG "instance": "10.142.0.107:9100" 22:13:59 DEBUG "result": [ 22:13:59 DEBUG "result": [ 22:13:59 DEBUG "data": { 22:13:59 DEBUG "resultType": "vector", 22:13:59 DEBUG "status": "success", 22:13:59 DEBUG "resultType": "vector", 22:13:59 DEBUG "metric": { 22:13:59 DEBUG "metric": { 22:13:59 DEBUG "result": [ 22:13:59 DEBUG "result": [ 22:13:59 DEBUG { 22:13:59 DEBUG "metric": { 22:13:59 DEBUG { 22:13:59 DEBUG "status": "success", 22:13:59 DEBUG "data": { 22:13:59 DEBUG { 22:13:59 DEBUG { 22:13:59 DEBUG "result": [ 22:13:59 DEBUG { 22:13:59 DEBUG }, 22:13:59 DEBUG { 22:13:59 DEBUG { 22:13:59 DEBUG { 22:13:59 DEBUG "resultType": "vector", 22:13:59 DEBUG "result": [ 22:13:59 DEBUG "data": { 22:13:59 DEBUG "result": [ 22:13:59 DEBUG "pod": "ds-cts-0" 22:13:59 DEBUG }, 22:13:59 DEBUG { 22:13:59 DEBUG "metric": { 22:13:59 DEBUG { 22:13:59 DEBUG "pod": "am-767f6b67b5-9zw22" 22:13:59 DEBUG "metric": { 22:13:59 DEBUG "resultType": "vector", 22:13:59 DEBUG "metric": { 22:13:59 DEBUG "metric": { 22:13:59 DEBUG "metric": { 22:13:59 DEBUG "metric": { 22:13:59 DEBUG "metric": { 22:13:59 DEBUG "value": [ 22:13:59 DEBUG "metric": { 22:13:59 DEBUG "metric": { 22:13:59 DEBUG "metric": { 22:13:59 DEBUG "result": [ 22:13:59 DEBUG { 22:13:59 DEBUG "resultType": "vector", 22:13:59 DEBUG { 22:13:59 DEBUG "value": [ 22:13:59 DEBUG 22:13:59 DEBUG "pod": "am-767f6b67b5-9zw22" 22:13:59 DEBUG "metric": { 22:13:59 DEBUG "metric": { 22:13:59 DEBUG "pod": "ds-cts-0" 22:13:59 DEBUG "instance": "10.142.0.107:9100" 22:13:59 DEBUG "result": [ 22:13:59 DEBUG "pod": "am-767f6b67b5-9zw22" 22:13:59 DEBUG }, 22:13:59 DEBUG "pod": "ds-idrepo-0" 22:13:59 DEBUG "pod": "am-767f6b67b5-9zw22" 22:13:59 DEBUG "instance": "10.142.0.107:9100" 22:13:59 DEBUG 22:13:59 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:13:59 DEBUG }, 22:13:59 DEBUG "pod": "ds-idrepo-0" 22:13:59 DEBUG { 22:13:59 DEBUG "metric": { 22:13:59 DEBUG "metric": { 22:13:59 DEBUG "metric": { 22:13:59 DEBUG "instance": "10.142.0.107:9100" 22:13:59 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:13:59 DEBUG }, 22:13:59 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:13:59 DEBUG }, 22:13:59 DEBUG }, 22:13:59 DEBUG }, 22:13:59 DEBUG { 22:13:59 DEBUG "value": [ 22:13:59 DEBUG }, 22:13:59 DEBUG }, 22:13:59 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:13:59 DEBUG }, 22:13:59 DEBUG 22:13:59 DEBUG "value": [ 22:13:59 DEBUG "pod": "am-767f6b67b5-9zw22" 22:13:59 DEBUG }, 22:13:59 DEBUG "value": [ 22:13:59 DEBUG "result": [ 22:13:59 DEBUG "pod": "ds-idrepo-0" 22:13:59 DEBUG 22:13:59 DEBUG }, 22:13:59 DEBUG }, 22:13:59 DEBUG "value": [ 22:13:59 DEBUG }, 22:13:59 DEBUG "value": [ 22:13:59 DEBUG 22:13:59 DEBUG "value": [ 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG "value": [ 22:13:59 DEBUG "value": [ 22:13:59 DEBUG 22:13:59 DEBUG "value": [ 22:13:59 DEBUG }, 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:13:59 DEBUG { 22:13:59 DEBUG "metric": { 22:13:59 DEBUG "pod": "ds-cts-0" 22:13:59 DEBUG }, 22:13:59 DEBUG }, 22:13:59 DEBUG "value": [ 22:13:59 DEBUG 22:13:59 DEBUG "value": [ 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG "metric": { 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG "value": [ 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG }, 22:13:59 DEBUG "value": [ 22:13:59 DEBUG "value": [ 22:13:59 DEBUG }, 22:13:59 DEBUG "value": [ 22:13:59 DEBUG 22:13:59 DEBUG "value": [ 22:13:59 DEBUG 22:13:59 DEBUG "value": [ 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG "pod": "am-767f6b67b5-9zw22" 22:13:59 DEBUG }, 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG [...] 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG [...] 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG [...] 22:13:59 DEBUG [...] 22:13:59 DEBUG "value": [ 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG [...] 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG "value": [ 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG [...] 22:13:59 DEBUG 22:13:59 DEBUG [...] 22:13:59 DEBUG [...] 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG [...] 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG [...] 22:13:59 DEBUG 22:13:59 DEBUG [...] 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG [...] 22:13:59 DEBUG 22:13:59 DEBUG [...] 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG "instance": "10.142.0.43:9100" 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG [...] 22:13:59 DEBUG 22:13:59 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:13:59 DEBUG 22:13:59 DEBUG [...] 22:13:59 DEBUG [...] 22:13:59 DEBUG [...] 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG }, 22:13:59 DEBUG 22:13:59 DEBUG [...] 22:13:59 DEBUG 22:13:59 DEBUG "instance": "10.142.0.43:9100" 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG }, 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG [...] 22:13:59 DEBUG "pod": "am-767f6b67b5-qcqz2" 22:13:59 DEBUG [...] 22:13:59 DEBUG "value": [ 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG "pod": "am-767f6b67b5-vbltz" 22:13:59 DEBUG }, 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG "value": [ 22:13:59 DEBUG [...] 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG "pod": "ds-cts-2" 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG "pod": "am-767f6b67b5-vbltz" 22:13:59 DEBUG "pod": "ds-cts-2" 22:13:59 DEBUG 22:13:59 DEBUG }, 22:13:59 DEBUG 22:13:59 DEBUG 1713647639, 22:13:59 DEBUG "pod": "ds-idrepo-2" 22:13:59 DEBUG 22:13:59 DEBUG }, 22:13:59 DEBUG "value": [ 22:13:59 DEBUG "instance": "10.142.0.43:9100" 22:13:59 DEBUG 22:13:59 DEBUG "pod": "am-767f6b67b5-vbltz" 22:13:59 DEBUG 1713647639, 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG }, 22:13:59 DEBUG "pod": "ds-idrepo-2" 22:13:59 DEBUG "pod": "overseer-0-967d957f7-zk66v" 22:13:59 DEBUG }, 22:13:59 DEBUG }, 22:13:59 DEBUG 22:13:59 DEBUG "value": [ 22:13:59 DEBUG 22:13:59 DEBUG "107199246.46666667" 22:13:59 DEBUG }, 22:13:59 DEBUG "value": [ 22:13:59 DEBUG 1713647639, 22:13:59 DEBUG 1713647639, 22:13:59 DEBUG "0.07600000000000856" 22:13:59 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:13:59 DEBUG }, 22:13:59 DEBUG "2033561600" 22:13:59 DEBUG 22:13:59 DEBUG 22:13:59 DEBUG "pod": "ds-cts-2" 22:13:59 DEBUG "pod": "ds-idrepo-2" 22:13:59 DEBUG }, 22:13:59 DEBUG }, 22:13:59 DEBUG }, 22:13:59 DEBUG "value": [ 22:13:59 DEBUG 1713647639, 22:13:59 DEBUG 22:13:59 DEBUG 1713647639, 22:13:59 DEBUG 22:13:59 DEBUG ] 22:13:59 DEBUG 22:13:59 DEBUG "0" 22:13:59 DEBUG "value": [ 22:13:59 DEBUG ] 22:13:59 DEBUG }, 22:13:59 DEBUG }, 22:13:59 DEBUG "value": [ 22:13:59 DEBUG ] 22:13:59 DEBUG "pod": "am-767f6b67b5-qcqz2" 22:13:59 DEBUG "instance": "10.142.0.43:9100" 22:13:59 DEBUG }, 22:13:59 DEBUG "value": [ 22:13:59 DEBUG "value": [ 22:13:59 DEBUG "value": [ 22:13:59 DEBUG "value": [ 22:13:59 DEBUG "0" 22:13:59 DEBUG "pod": "am-767f6b67b5-vbltz" 22:13:59 DEBUG "543.9936250747062" 22:13:59 DEBUG "value": [ 22:13:59 DEBUG "pod": "am-767f6b67b5-vbltz" 22:13:59 DEBUG } 22:13:59 DEBUG "pod": "am-767f6b67b5-vbltz" 22:13:59 DEBUG ] 22:13:59 DEBUG 1713647639, 22:13:59 DEBUG } 22:13:59 DEBUG "value": [ 22:13:59 DEBUG "value": [ 22:13:59 DEBUG 1713647639, 22:13:59 DEBUG } 22:13:59 DEBUG }, 22:13:59 DEBUG "value": [ 22:13:59 DEBUG 1713647639, 22:13:59 DEBUG 1713647639, 22:13:59 DEBUG 1713647639, 22:13:59 DEBUG }, 22:13:59 DEBUG 1713647639, 22:13:59 DEBUG ] 22:13:59 DEBUG }, 22:13:59 DEBUG ] 22:13:59 DEBUG 1713647639, 22:13:59 DEBUG }, 22:13:59 DEBUG ] 22:13:59 DEBUG }, 22:13:59 DEBUG } 22:13:59 DEBUG "3472.266666666667" 22:13:59 DEBUG ] 22:13:59 DEBUG 1713647639, 22:13:59 DEBUG 1713647639, 22:13:59 DEBUG "3471.366666666667" 22:13:59 DEBUG ] 22:13:59 DEBUG "value": [ 22:13:59 DEBUG 1713647639, 22:13:59 DEBUG "0.6016999999999996" 22:13:59 DEBUG "0" 22:13:59 DEBUG "0" 22:13:59 DEBUG "value": [ 22:13:59 DEBUG "0" 22:13:59 DEBUG } 22:13:59 DEBUG "value": [ 22:13:59 DEBUG } 22:13:59 DEBUG "29.366666666666667" 22:13:59 DEBUG "value": [ 22:13:59 DEBUG } 22:13:59 DEBUG "value": [ 22:13:59 DEBUG ] 22:13:59 DEBUG ] 22:13:59 DEBUG } 22:13:59 DEBUG } 22:13:59 DEBUG "13.458806578753215" 22:13:59 DEBUG ] 22:13:59 DEBUG } 22:13:59 DEBUG 1713647639, 22:13:59 DEBUG "0.3701666666666919" 22:13:59 DEBUG ] 22:13:59 DEBUG ] 22:13:59 DEBUG ] 22:13:59 DEBUG 1713647639, 22:13:59 DEBUG ] 22:13:59 DEBUG ] 22:13:59 DEBUG 1713647639, 22:13:59 DEBUG ] 22:13:59 DEBUG ] 22:13:59 DEBUG 1713647639, 22:13:59 DEBUG } 22:13:59 DEBUG 1713647639, 22:13:59 DEBUG } 22:13:59 DEBUG } 22:13:59 DEBUG "4099717604352" 22:13:59 DEBUG ] 22:13:59 DEBUG } 22:13:59 DEBUG } 22:13:59 DEBUG "0" 22:13:59 DEBUG ] 22:13:59 DEBUG } 22:13:59 DEBUG } 22:13:59 DEBUG } 22:13:59 DEBUG "0" 22:13:59 DEBUG } 22:13:59 DEBUG } 22:13:59 DEBUG "87.4605" 22:13:59 DEBUG } 22:13:59 DEBUG } 22:13:59 DEBUG ] 22:13:59 DEBUG "411030" 22:13:59 DEBUG ] 22:13:59 DEBUG ] 22:13:59 DEBUG ] 22:13:59 DEBUG } 22:13:59 DEBUG ] 22:13:59 DEBUG ] 22:13:59 DEBUG } 22:13:59 DEBUG ] 22:13:59 DEBUG ] 22:13:59 DEBUG ] 22:13:59 DEBUG ] 22:13:59 DEBUG ] 22:13:59 DEBUG } 22:13:59 DEBUG ] 22:13:59 DEBUG } 22:13:59 DEBUG "3472.0333333333333" 22:13:59 DEBUG } 22:13:59 DEBUG } 22:13:59 DEBUG } 22:13:59 DEBUG } 22:13:59 DEBUG } 22:13:59 DEBUG ] 22:13:59 DEBUG } 22:13:59 DEBUG } 22:13:59 DEBUG ] 22:13:59 DEBUG } 22:13:59 DEBUG } 22:13:59 DEBUG } 22:13:59 DEBUG } 22:13:59 DEBUG ] 22:13:59 DEBUG } 22:13:59 DEBUG } 22:13:59 DEBUG } 22:13:59 DEBUG } 22:13:59 DEBUG } 22:13:59 DEBUG ] 22:13:59 DEBUG } 22:13:59 DEBUG } 22:13:59 DEBUG } 22:13:59 DEBUG } 22:13:59 DEBUG } 22:13:59 DEBUG } 22:13:59 DEBUG } 22:13:59 DEBUG ] 22:13:59 DEBUG ] 22:13:59 DEBUG ] 22:13:59 DEBUG } 22:13:59 DEBUG } 22:13:59 DEBUG } 22:13:59 DEBUG } 22:13:59 DEBUG ] 22:13:59 DEBUG } 22:13:59 DEBUG } 22:13:59 DEBUG } 22:13:59 DEBUG } 22:13:59 DEBUG } 22:13:59 DEBUG } 22:13:59 DEBUG ] 22:13:59 DEBUG } 22:13:59 DEBUG } 22:13:59 DEBUG } 22:13:59 DEBUG } 22:14:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713647695" 22:14:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647695" 22:14:55 ERROR [http_cmd]: ERROR 22:14:55 ERROR --- status code --- 22:14:55 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:14:55 ERROR --- http response --- 22:14:55 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:14:55 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:14:55 INFO [http_cmd]: http status code OK 22:14:55 DEBUG --- status code --- 22:14:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:55 DEBUG --- http response --- 22:14:55 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:14:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713647695" 22:14:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647695" 22:14:55 INFO [http_cmd]: http status code OK 22:14:55 INFO [http_cmd]: http status code OK 22:14:55 DEBUG --- status code --- 22:14:55 DEBUG --- status code --- 22:14:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:55 DEBUG --- http response --- 22:14:55 DEBUG --- http response --- 22:14:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:14:55 DEBUG ----- output ----- 22:14:55 DEBUG { 22:14:55 DEBUG "status": "success", 22:14:55 DEBUG "data": { 22:14:55 DEBUG "resultType": "vector", 22:14:55 DEBUG "result": [ 22:14:55 DEBUG { 22:14:55 DEBUG "metric": { 22:14:55 DEBUG "pod": "ds-idrepo-0" 22:14:55 DEBUG }, 22:14:55 DEBUG "value": [ 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG [...] 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG "pod": "ds-cts-2" 22:14:55 DEBUG }, 22:14:55 DEBUG "value": [ 22:14:55 DEBUG 1713647695, 22:14:55 DEBUG "0" 22:14:55 DEBUG ] 22:14:55 DEBUG } 22:14:55 DEBUG ] 22:14:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 22:14:55 DEBUG } 22:14:55 DEBUG ----- output ----- 22:14:55 DEBUG } 22:14:55 DEBUG { 22:14:55 DEBUG "status": "success", 22:14:55 DEBUG "data": { 22:14:55 DEBUG "resultType": "vector", 22:14:55 DEBUG "result": [ 22:14:55 DEBUG { 22:14:55 DEBUG "metric": { 22:14:55 DEBUG "pod": "ds-idrepo-1" 22:14:55 DEBUG }, 22:14:55 DEBUG "value": [ 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG [...] 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 22:14:55 DEBUG }, 22:14:55 DEBUG "value": [ 22:14:55 DEBUG 1713647695, 22:14:55 DEBUG "0" 22:14:55 DEBUG ] 22:14:55 DEBUG } 22:14:55 DEBUG ] 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647696" 22:14:56 INFO [http_cmd]: http status code OK 22:14:56 DEBUG --- status code --- 22:14:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:56 DEBUG --- http response --- 22:14:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:14:56 DEBUG ----- output ----- 22:14:56 DEBUG { 22:14:56 DEBUG "status": "success", 22:14:56 DEBUG "data": { 22:14:56 DEBUG "resultType": "vector", 22:14:56 DEBUG "result": [ 22:14:56 DEBUG { 22:14:56 DEBUG "metric": { 22:14:56 DEBUG "pod": "am-767f6b67b5-9zw22" 22:14:56 DEBUG }, 22:14:56 DEBUG "value": [ 22:14:56 DEBUG 22:14:56 DEBUG 22:14:56 DEBUG 22:14:56 DEBUG [...] 22:14:56 DEBUG 22:14:56 DEBUG 22:14:56 DEBUG 22:14:56 DEBUG "pod": "am-767f6b67b5-vbltz" 22:14:56 DEBUG }, 22:14:56 DEBUG "value": [ 22:14:56 DEBUG 1713647696, 22:14:56 DEBUG "2147.903752582817" 22:14:56 DEBUG ] 22:14:56 DEBUG } 22:14:56 DEBUG ] 22:14:56 DEBUG } 22:14:56 DEBUG } 22:14:57 INFO 22:14:57 INFO [loop_until]: kubectl --namespace=xlou top pods 22:14:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:14:57 INFO [loop_until]: OK (rc = 0) 22:14:57 DEBUG --- stdout --- 22:14:57 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 15639m 8824Mi am-767f6b67b5-qcqz2 14049m 8981Mi am-767f6b67b5-vbltz 15678m 8944Mi ds-cts-0 19790m 12311Mi ds-cts-1 18661m 12273Mi ds-cts-2 20451m 12468Mi ds-idrepo-0 6113m 3921Mi ds-idrepo-1 6293m 3966Mi ds-idrepo-2 6335m 3877Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-7dd48676f6-tdxl9 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-967d957f7-zk66v 14878m 6083Mi 22:14:57 DEBUG --- stderr --- 22:14:57 DEBUG 22:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647699" 22:14:59 INFO [http_cmd]: http status code OK 22:14:59 DEBUG --- status code --- 22:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:59 DEBUG --- http response --- 22:14:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:14:59 DEBUG ----- output ----- 22:14:59 DEBUG { 22:14:59 DEBUG "status": "success", 22:14:59 DEBUG "data": { 22:14:59 DEBUG "resultType": "vector", 22:14:59 DEBUG "result": [ 22:14:59 DEBUG { 22:14:59 DEBUG "metric": { 22:14:59 DEBUG "instance": "10.142.0.107:9100" 22:14:59 DEBUG }, 22:14:59 DEBUG "value": [ 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG [...] 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG "instance": "10.142.0.43:9100" 22:14:59 DEBUG }, 22:14:59 DEBUG "value": [ 22:14:59 DEBUG 1713647699, 22:14:59 DEBUG "74869538.53333333" 22:14:59 DEBUG ] 22:14:59 DEBUG } 22:14:59 DEBUG ] 22:14:59 DEBUG } 22:14:59 DEBUG } 22:14:59 INFO 22:14:59 INFO [loop_until]: kubectl --namespace=xlou top node 22:14:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647699" 22:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647699" 22:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647699" 22:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647699" 22:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713647699" 22:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647699" 22:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647699" 22:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647699" 22:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647699" 22:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647699" 22:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647699" 22:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647699" 22:14:59 INFO [http_cmd]: http status code OK 22:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647699" 22:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713647699" 22:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647699" 22:14:59 INFO [http_cmd]: http status code OK 22:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647699" 22:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713647699" 22:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647699" 22:14:59 INFO [http_cmd]: http status code OK 22:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647699" 22:14:59 DEBUG --- status code --- 22:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647699" 22:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647699" 22:14:59 INFO [http_cmd]: http status code OK 22:14:59 INFO [http_cmd]: http status code OK 22:14:59 INFO [http_cmd]: http status code OK 22:14:59 INFO [http_cmd]: http status code OK 22:14:59 INFO [loop_until]: OK (rc = 0) 22:14:59 INFO [http_cmd]: http status code OK 22:14:59 DEBUG --- status code --- 22:14:59 INFO [http_cmd]: http status code OK 22:14:59 INFO [http_cmd]: http status code OK 22:14:59 DEBUG --- status code --- 22:14:59 INFO [http_cmd]: http status code OK 22:14:59 DEBUG --- status code --- 22:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:59 INFO [http_cmd]: http status code OK 22:14:59 INFO [http_cmd]: http status code OK 22:14:59 DEBUG --- status code --- 22:14:59 DEBUG --- status code --- 22:14:59 DEBUG --- status code --- 22:14:59 INFO [http_cmd]: http status code OK 22:14:59 DEBUG --- status code --- 22:14:59 INFO [http_cmd]: http status code OK 22:14:59 INFO [http_cmd]: http status code OK 22:14:59 DEBUG --- stdout --- 22:14:59 INFO [http_cmd]: http status code OK 22:14:59 INFO [http_cmd]: http status code OK 22:14:59 INFO [http_cmd]: http status code OK 22:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:59 DEBUG --- status code --- 22:14:59 DEBUG --- status code --- 22:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:59 DEBUG --- status code --- 22:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:59 DEBUG --- http response --- 22:14:59 DEBUG --- status code --- 22:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:59 INFO [http_cmd]: http status code OK 22:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:59 DEBUG --- status code --- 22:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:59 DEBUG --- status code --- 22:14:59 DEBUG --- status code --- 22:14:59 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 6225Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 6324m 21% 7680Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 14958m 50% 11584Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 79m 0% 3358Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 19458m 65% 15567Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 156m 0% 5187Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 14083m 47% 11908Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 13408m 44% 3926Mi 3% gke-xlou-cdm-ds-0f601751-hmm7 16794m 56% 11216Mi 10% gke-xlou-cdm-ds-0f601751-j488 6616m 22% 7289Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 18871m 63% 17294Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6584m 22% 7942Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20062m 67% 15175Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 14941m 94% 13588Mi 23% 22:14:59 DEBUG --- status code --- 22:14:59 DEBUG --- status code --- 22:14:59 DEBUG --- status code --- 22:14:59 DEBUG --- http response --- 22:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:59 DEBUG --- http response --- 22:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:59 DEBUG --- http response --- 22:14:59 DEBUG --- status code --- 22:14:59 DEBUG --- http response --- 22:14:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:14:59 INFO [http_cmd]: http status code OK 22:14:59 DEBUG --- status code --- 22:14:59 DEBUG --- http response --- 22:14:59 DEBUG --- http response --- 22:14:59 DEBUG --- http response --- 22:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:59 DEBUG --- http response --- 22:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:59 DEBUG --- stderr --- 22:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:59 DEBUG --- http response --- 22:14:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:14:59 DEBUG --- http response --- 22:14:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:14:59 DEBUG --- http response --- 22:14:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:14:59 DEBUG ----- output ----- 22:14:59 DEBUG --- status code --- 22:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:14:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:14:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 22:14:59 DEBUG --- http response --- 22:14:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:14:59 DEBUG --- http response --- 22:14:59 DEBUG --- http response --- 22:14:59 DEBUG 22:14:59 DEBUG --- http response --- 22:14:59 DEBUG --- http response --- 22:14:59 DEBUG --- http response --- 22:14:59 DEBUG ----- output ----- 22:14:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:14:59 DEBUG ----- output ----- 22:14:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:14:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:14:59 DEBUG ----- output ----- 22:14:59 DEBUG --- http response --- 22:14:59 DEBUG ----- output ----- 22:14:59 DEBUG { 22:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:59 DEBUG --- http response --- 22:14:59 DEBUG ----- output ----- 22:14:59 DEBUG ----- output ----- 22:14:59 DEBUG ----- output ----- 22:14:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:14:59 DEBUG ----- output ----- 22:14:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:14:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:14:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:14:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:14:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:14:59 DEBUG { 22:14:59 DEBUG ----- output ----- 22:14:59 DEBUG { 22:14:59 DEBUG ----- output ----- 22:14:59 DEBUG ----- output ----- 22:14:59 DEBUG { 22:14:59 DEBUG { 22:14:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:14:59 DEBUG "status": "success", 22:14:59 DEBUG --- http response --- 22:14:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:14:59 DEBUG ----- output ----- 22:14:59 DEBUG { 22:14:59 DEBUG { 22:14:59 DEBUG ----- output ----- 22:14:59 DEBUG { 22:14:59 DEBUG ----- output ----- 22:14:59 DEBUG ----- output ----- 22:14:59 DEBUG ----- output ----- 22:14:59 DEBUG ----- output ----- 22:14:59 DEBUG ----- output ----- 22:14:59 DEBUG "status": "success", 22:14:59 DEBUG { 22:14:59 DEBUG "status": "success", 22:14:59 DEBUG { 22:14:59 DEBUG { 22:14:59 DEBUG "status": "success", 22:14:59 DEBUG "status": "success", 22:14:59 DEBUG ----- output ----- 22:14:59 DEBUG "data": { 22:14:59 DEBUG "status": "success", 22:14:59 DEBUG { 22:14:59 DEBUG "status": "success", 22:14:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 22:14:59 DEBUG "status": "success", 22:14:59 DEBUG { 22:14:59 DEBUG "status": "success", 22:14:59 DEBUG { 22:14:59 DEBUG { 22:14:59 DEBUG "status": "success", 22:14:59 DEBUG "data": { 22:14:59 DEBUG { 22:14:59 DEBUG { 22:14:59 DEBUG "status": "success", 22:14:59 DEBUG "data": { 22:14:59 DEBUG "resultType": "vector", 22:14:59 DEBUG "data": { 22:14:59 DEBUG "resultType": "vector", 22:14:59 DEBUG "data": { 22:14:59 DEBUG { 22:14:59 DEBUG "resultType": "vector", 22:14:59 DEBUG "data": { 22:14:59 DEBUG { 22:14:59 DEBUG "data": { 22:14:59 DEBUG "resultType": "vector", 22:14:59 DEBUG "data": { 22:14:59 DEBUG "status": "success", 22:14:59 DEBUG "data": { 22:14:59 DEBUG "resultType": "vector", 22:14:59 DEBUG "resultType": "vector", 22:14:59 DEBUG { 22:14:59 DEBUG "data": { 22:14:59 DEBUG "status": "success", 22:14:59 DEBUG "status": "success", 22:14:59 DEBUG "data": { 22:14:59 DEBUG "result": [ 22:14:59 DEBUG "status": "success", 22:14:59 DEBUG "status": "success", 22:14:59 DEBUG "result": [ 22:14:59 DEBUG "resultType": "vector", 22:14:59 DEBUG "status": "success", 22:14:59 DEBUG "result": [ 22:14:59 DEBUG "resultType": "vector", 22:14:59 DEBUG "status": "success", 22:14:59 DEBUG "result": [ 22:14:59 DEBUG ----- output ----- 22:14:59 DEBUG "resultType": "vector", 22:14:59 DEBUG "data": { 22:14:59 DEBUG "status": "success", 22:14:59 DEBUG "result": [ 22:14:59 DEBUG "result": [ 22:14:59 DEBUG "status": "success", 22:14:59 DEBUG "resultType": "vector", 22:14:59 DEBUG "data": { 22:14:59 DEBUG "data": { 22:14:59 DEBUG "resultType": "vector", 22:14:59 DEBUG { 22:14:59 DEBUG "data": { 22:14:59 DEBUG "data": { 22:14:59 DEBUG { 22:14:59 DEBUG "result": [ 22:14:59 DEBUG "data": { 22:14:59 DEBUG { 22:14:59 DEBUG "result": [ 22:14:59 DEBUG "data": { 22:14:59 DEBUG { 22:14:59 DEBUG { 22:14:59 DEBUG "status": "success", 22:14:59 DEBUG "result": [ 22:14:59 DEBUG "data": { 22:14:59 DEBUG { 22:14:59 DEBUG "metric": { 22:14:59 DEBUG "data": { 22:14:59 DEBUG "result": [ 22:14:59 DEBUG "resultType": "vector", 22:14:59 DEBUG "resultType": "vector", 22:14:59 DEBUG "result": [ 22:14:59 DEBUG "metric": { 22:14:59 DEBUG "resultType": "vector", 22:14:59 DEBUG "resultType": "vector", 22:14:59 DEBUG "result": [ 22:14:59 DEBUG { 22:14:59 DEBUG "resultType": "vector", 22:14:59 DEBUG "metric": { 22:14:59 DEBUG { 22:14:59 DEBUG "resultType": "vector", 22:14:59 DEBUG "metric": { 22:14:59 DEBUG "resultType": "vector", 22:14:59 DEBUG "data": { 22:14:59 DEBUG { 22:14:59 DEBUG "resultType": "vector", 22:14:59 DEBUG "pod": "am-767f6b67b5-9zw22" 22:14:59 DEBUG "resultType": "vector", 22:14:59 DEBUG { 22:14:59 DEBUG { 22:14:59 DEBUG "metric": { 22:14:59 DEBUG "result": [ 22:14:59 DEBUG "result": [ 22:14:59 DEBUG "pod": "ds-idrepo-0" 22:14:59 DEBUG "result": [ 22:14:59 DEBUG "metric": { 22:14:59 DEBUG { 22:14:59 DEBUG "metric": { 22:14:59 DEBUG "result": [ 22:14:59 DEBUG "pod": "ds-idrepo-0" 22:14:59 DEBUG "metric": { 22:14:59 DEBUG "result": [ 22:14:59 DEBUG "instance": "10.142.0.107:9100" 22:14:59 DEBUG "result": [ 22:14:59 DEBUG "resultType": "vector", 22:14:59 DEBUG "metric": { 22:14:59 DEBUG "result": [ 22:14:59 DEBUG }, 22:14:59 DEBUG "result": [ 22:14:59 DEBUG { 22:14:59 DEBUG "metric": { 22:14:59 DEBUG { 22:14:59 DEBUG { 22:14:59 DEBUG { 22:14:59 DEBUG }, 22:14:59 DEBUG { 22:14:59 DEBUG "instance": "10.142.0.107:9100" 22:14:59 DEBUG "metric": { 22:14:59 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:14:59 DEBUG { 22:14:59 DEBUG }, 22:14:59 DEBUG "pod": "ds-cts-0" 22:14:59 DEBUG { 22:14:59 DEBUG }, 22:14:59 DEBUG { 22:14:59 DEBUG "result": [ 22:14:59 DEBUG "pod": "ds-idrepo-0" 22:14:59 DEBUG { 22:14:59 DEBUG "value": [ 22:14:59 DEBUG "pod": "ds-cts-0" 22:14:59 DEBUG "pod": "am-767f6b67b5-9zw22" 22:14:59 DEBUG "metric": { 22:14:59 DEBUG "metric": { 22:14:59 DEBUG "metric": { 22:14:59 DEBUG "metric": { 22:14:59 DEBUG "value": [ 22:14:59 DEBUG "metric": { 22:14:59 DEBUG }, 22:14:59 DEBUG "instance": "10.142.0.107:9100" 22:14:59 DEBUG }, 22:14:59 DEBUG "metric": { 22:14:59 DEBUG "value": [ 22:14:59 DEBUG }, 22:14:59 DEBUG "metric": { 22:14:59 DEBUG "value": [ 22:14:59 DEBUG "metric": { 22:14:59 DEBUG { 22:14:59 DEBUG }, 22:14:59 DEBUG "metric": { 22:14:59 DEBUG 22:14:59 DEBUG }, 22:14:59 DEBUG }, 22:14:59 DEBUG "pod": "am-767f6b67b5-9zw22" 22:14:59 DEBUG "pod": "am-767f6b67b5-9zw22" 22:14:59 DEBUG "instance": "10.142.0.107:9100" 22:14:59 DEBUG "pod": "ds-idrepo-0" 22:14:59 DEBUG 22:14:59 DEBUG "pod": "am-767f6b67b5-9zw22" 22:14:59 DEBUG "value": [ 22:14:59 DEBUG }, 22:14:59 DEBUG "value": [ 22:14:59 DEBUG "pod": "ds-cts-0" 22:14:59 DEBUG 22:14:59 DEBUG "value": [ 22:14:59 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:14:59 DEBUG 22:14:59 DEBUG "pod": "ds-idrepo-0" 22:14:59 DEBUG "metric": { 22:14:59 DEBUG "value": [ 22:14:59 DEBUG "pod": "am-767f6b67b5-9zw22" 22:14:59 DEBUG 22:14:59 DEBUG "value": [ 22:14:59 DEBUG "value": [ 22:14:59 DEBUG }, 22:14:59 DEBUG }, 22:14:59 DEBUG }, 22:14:59 DEBUG }, 22:14:59 DEBUG 22:14:59 DEBUG }, 22:14:59 DEBUG 22:14:59 DEBUG "value": [ 22:14:59 DEBUG 22:14:59 DEBUG }, 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG }, 22:14:59 DEBUG 22:14:59 DEBUG }, 22:14:59 DEBUG "pod": "ds-idrepo-0" 22:14:59 DEBUG 22:14:59 DEBUG }, 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG "value": [ 22:14:59 DEBUG "value": [ 22:14:59 DEBUG "value": [ 22:14:59 DEBUG 22:14:59 DEBUG "value": [ 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG "value": [ 22:14:59 DEBUG 22:14:59 DEBUG [...] 22:14:59 DEBUG "value": [ 22:14:59 DEBUG 22:14:59 DEBUG "value": [ 22:14:59 DEBUG }, 22:14:59 DEBUG 22:14:59 DEBUG "value": [ 22:14:59 DEBUG [...] 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG [...] 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG [...] 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG [...] 22:14:59 DEBUG 22:14:59 DEBUG "value": [ 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG [...] 22:14:59 DEBUG 22:14:59 DEBUG [...] 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG [...] 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG [...] 22:14:59 DEBUG "value": [ 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG [...] 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG [...] 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG [...] 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG [...] 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG [...] 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG [...] 22:14:59 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 22:14:59 DEBUG 22:14:59 DEBUG [...] 22:14:59 DEBUG 22:14:59 DEBUG [...] 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG [...] 22:14:59 DEBUG "pod": "am-767f6b67b5-vbltz" 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG [...] 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG "pod": "ds-idrepo-2" 22:14:59 DEBUG }, 22:14:59 DEBUG "value": [ 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG }, 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG "instance": "10.142.0.43:9100" 22:14:59 DEBUG 22:14:59 DEBUG [...] 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG }, 22:14:59 DEBUG "pod": "am-767f6b67b5-vbltz" 22:14:59 DEBUG 22:14:59 DEBUG "pod": "am-767f6b67b5-vbltz" 22:14:59 DEBUG 1713647699, 22:14:59 DEBUG "0" 22:14:59 DEBUG 22:14:59 DEBUG [...] 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG "value": [ 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG }, 22:14:59 DEBUG "value": [ 22:14:59 DEBUG 1713647699, 22:14:59 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 22:14:59 DEBUG 22:14:59 DEBUG "value": [ 22:14:59 DEBUG }, 22:14:59 DEBUG "value": [ 22:14:59 DEBUG 1713647699, 22:14:59 DEBUG ] 22:14:59 DEBUG } 22:14:59 DEBUG ] 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG "pod": "ds-idrepo-2" 22:14:59 DEBUG }, 22:14:59 DEBUG "value": [ 22:14:59 DEBUG "pod": "ds-cts-2" 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG "0.0786666666666728" 22:14:59 DEBUG 22:14:59 DEBUG "pod": "ds-cts-2" 22:14:59 DEBUG }, 22:14:59 DEBUG "value": [ 22:14:59 DEBUG 1713647699, 22:14:59 DEBUG "0" 22:14:59 DEBUG } 22:14:59 DEBUG } 22:14:59 DEBUG 1713647699, 22:14:59 DEBUG 22:14:59 DEBUG 22:14:59 DEBUG }, 22:14:59 DEBUG 22:14:59 DEBUG 1713647699, 22:14:59 DEBUG "0" 22:14:59 DEBUG "pod": "am-767f6b67b5-vbltz" 22:14:59 DEBUG }, 22:14:59 DEBUG }, 22:14:59 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:14:59 DEBUG 22:14:59 DEBUG ] 22:14:59 DEBUG "90.463" 22:14:59 DEBUG 22:14:59 DEBUG ] 22:14:59 DEBUG }, 22:14:59 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 22:14:59 DEBUG 22:14:59 DEBUG "instance": "10.142.0.43:9100" 22:14:59 DEBUG "0" 22:14:59 DEBUG "instance": "10.142.0.43:9100" 22:14:59 DEBUG "pod": "am-767f6b67b5-vbltz" 22:14:59 DEBUG "value": [ 22:14:59 DEBUG 22:14:59 DEBUG ] 22:14:59 DEBUG 1713647699, 22:14:59 DEBUG "value": [ 22:14:59 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:14:59 DEBUG "value": [ 22:14:59 DEBUG }, 22:14:59 DEBUG "pod": "ds-idrepo-2" 22:14:59 DEBUG } 22:14:59 DEBUG ] 22:14:59 DEBUG 22:14:59 DEBUG } 22:14:59 DEBUG "value": [ 22:14:59 DEBUG }, 22:14:59 DEBUG "instance": "10.142.0.43:9100" 22:14:59 DEBUG }, 22:14:59 DEBUG ] 22:14:59 DEBUG }, 22:14:59 DEBUG }, 22:14:59 DEBUG 1713647699, 22:14:59 DEBUG "pod": "ds-cts-2" 22:14:59 DEBUG } 22:14:59 DEBUG ] 22:14:59 DEBUG 1713647699, 22:14:59 DEBUG }, 22:14:59 DEBUG 1713647699, 22:14:59 DEBUG "value": [ 22:14:59 DEBUG }, 22:14:59 DEBUG ] 22:14:59 DEBUG } 22:14:59 DEBUG 22:14:59 DEBUG ] 22:15:00 DEBUG } 22:14:59 DEBUG "value": [ 22:15:00 DEBUG 1713647699, 22:15:00 DEBUG "0" 22:15:00 DEBUG ] 22:14:59 DEBUG "value": [ 22:14:59 DEBUG "value": [ 22:14:59 DEBUG "2902.7527827767785" 22:14:59 DEBUG }, 22:14:59 DEBUG } 22:14:59 DEBUG "0.000020176934338627234" 22:14:59 DEBUG "2146.3373991868298" 22:14:59 DEBUG "value": [ 22:15:00 DEBUG "0.031099999999999756" 22:15:00 DEBUG 1713647699, 22:15:00 DEBUG "value": [ 22:15:00 DEBUG } 22:15:00 DEBUG ] 22:15:00 DEBUG "pod": "am-767f6b67b5-vbltz" 22:15:00 DEBUG } 22:14:59 DEBUG 1713647699, 22:15:00 DEBUG } 22:14:59 DEBUG "value": [ 22:14:59 DEBUG }, 22:15:00 DEBUG 1713647699, 22:15:00 DEBUG 1713647699, 22:14:59 DEBUG } 22:15:00 DEBUG ] 22:15:00 DEBUG "value": [ 22:15:00 DEBUG } 22:15:00 DEBUG ] 22:15:00 DEBUG ] 22:15:00 DEBUG 1713647699, 22:15:00 DEBUG ] 22:15:00 DEBUG "4448759808" 22:15:00 DEBUG 1713647699, 22:15:00 DEBUG } 22:15:00 DEBUG } 22:15:00 DEBUG }, 22:15:00 DEBUG "value": [ 22:15:00 DEBUG 1713647699, 22:15:00 DEBUG 1713647699, 22:15:00 DEBUG "4104362933248" 22:15:00 DEBUG "74874320.33333334" 22:15:00 DEBUG "381668.5" 22:15:00 DEBUG ] 22:15:00 DEBUG } 22:15:00 DEBUG 1713647699, 22:15:00 DEBUG } 22:15:00 DEBUG } 22:15:00 DEBUG "14.861866580610517" 22:15:00 DEBUG ] 22:15:00 DEBUG } 22:15:00 DEBUG ] 22:15:00 DEBUG } 22:15:00 DEBUG "0" 22:15:00 DEBUG "2149.1701659668065" 22:15:00 DEBUG ] 22:15:00 DEBUG ] 22:15:00 DEBUG } 22:15:00 DEBUG ] 22:15:00 DEBUG } 22:15:00 DEBUG ] 22:15:00 DEBUG ] 22:15:00 DEBUG "30.633333333333333" 22:15:00 DEBUG ] 22:15:00 DEBUG ] 22:15:00 DEBUG } 22:15:00 DEBUG } 22:15:00 DEBUG "0" 22:15:00 DEBUG ] 22:15:00 DEBUG ] 22:15:00 DEBUG } 22:15:00 DEBUG } 22:15:00 DEBUG ] 22:15:00 DEBUG } 22:15:00 DEBUG ] 22:15:00 DEBUG "value": [ 22:15:00 DEBUG } 22:15:00 DEBUG } 22:15:00 DEBUG ] 22:15:00 DEBUG } 22:15:00 DEBUG } 22:15:00 DEBUG } 22:15:00 DEBUG } 22:15:00 DEBUG } 22:15:00 DEBUG ] 22:15:00 DEBUG ] 22:15:00 DEBUG } 22:15:00 DEBUG } 22:15:00 DEBUG ] 22:15:00 DEBUG } 22:15:00 DEBUG 1713647699, 22:15:00 DEBUG } 22:15:00 DEBUG } 22:15:00 DEBUG } 22:15:00 DEBUG ] 22:15:00 DEBUG ] 22:15:00 DEBUG } 22:15:00 DEBUG ] 22:15:00 DEBUG } 22:15:00 DEBUG ] 22:15:00 DEBUG } 22:15:00 DEBUG } 22:15:00 DEBUG } 22:15:00 DEBUG } 22:15:00 DEBUG "0.42926666666658475" 22:15:00 DEBUG ] 22:15:00 DEBUG } 22:15:00 DEBUG } 22:15:00 DEBUG } 22:15:00 DEBUG } 22:15:00 DEBUG } 22:15:00 DEBUG } 22:15:00 DEBUG ] 22:15:00 DEBUG } 22:15:00 DEBUG ] 22:15:00 DEBUG } 22:15:00 DEBUG } 22:15:00 DEBUG } 22:15:00 DEBUG } 22:15:00 DEBUG } 22:15:00 DEBUG } 22:15:00 DEBUG ] 22:15:00 DEBUG } 22:15:00 DEBUG } 22:15:00 DEBUG } 22:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713647755" 22:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647755" 22:15:55 ERROR [http_cmd]: ERROR 22:15:55 ERROR --- status code --- 22:15:55 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:15:55 ERROR --- http response --- 22:15:55 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:15:55 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:15:55 INFO [http_cmd]: http status code OK 22:15:55 DEBUG --- status code --- 22:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:15:55 DEBUG --- http response --- 22:15:55 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713647755" 22:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647755" 22:15:55 INFO [http_cmd]: http status code OK 22:15:55 DEBUG --- status code --- 22:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:15:55 DEBUG --- http response --- 22:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:15:55 DEBUG ----- output ----- 22:15:55 DEBUG { 22:15:55 DEBUG "status": "success", 22:15:55 DEBUG "data": { 22:15:55 DEBUG "resultType": "vector", 22:15:55 DEBUG "result": [ 22:15:55 DEBUG { 22:15:55 DEBUG "metric": { 22:15:55 DEBUG "pod": "ds-idrepo-0" 22:15:55 DEBUG }, 22:15:55 DEBUG "value": [ 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG [...] 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG "pod": "ds-cts-2" 22:15:55 DEBUG }, 22:15:55 DEBUG "value": [ 22:15:55 DEBUG 1713647755, 22:15:55 DEBUG "0" 22:15:55 DEBUG ] 22:15:55 DEBUG } 22:15:55 DEBUG ] 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 INFO [http_cmd]: http status code OK 22:15:55 DEBUG --- status code --- 22:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:15:55 DEBUG --- http response --- 22:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 22:15:55 DEBUG ----- output ----- 22:15:55 DEBUG { 22:15:55 DEBUG "status": "success", 22:15:55 DEBUG "data": { 22:15:55 DEBUG "resultType": "vector", 22:15:55 DEBUG "result": [ 22:15:55 DEBUG { 22:15:55 DEBUG "metric": { 22:15:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:15:55 DEBUG }, 22:15:55 DEBUG "value": [ 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG [...] 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG "pod": "overseer-0-967d957f7-zk66v" 22:15:55 DEBUG }, 22:15:55 DEBUG "value": [ 22:15:55 DEBUG 1713647755, 22:15:55 DEBUG "0" 22:15:55 DEBUG ] 22:15:55 DEBUG } 22:15:55 DEBUG ] 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647756" 22:15:56 INFO [http_cmd]: http status code OK 22:15:56 DEBUG --- status code --- 22:15:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:15:56 DEBUG --- http response --- 22:15:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:15:56 DEBUG ----- output ----- 22:15:56 DEBUG { 22:15:56 DEBUG "status": "success", 22:15:56 DEBUG "data": { 22:15:56 DEBUG "resultType": "vector", 22:15:56 DEBUG "result": [ 22:15:56 DEBUG { 22:15:56 DEBUG "metric": { 22:15:56 DEBUG "pod": "am-767f6b67b5-9zw22" 22:15:56 DEBUG }, 22:15:56 DEBUG "value": [ 22:15:56 DEBUG 22:15:56 DEBUG 22:15:56 DEBUG 22:15:56 DEBUG [...] 22:15:56 DEBUG 22:15:56 DEBUG 22:15:56 DEBUG 22:15:56 DEBUG "pod": "am-767f6b67b5-vbltz" 22:15:56 DEBUG }, 22:15:56 DEBUG "value": [ 22:15:56 DEBUG 1713647756, 22:15:56 DEBUG "1821.5333333333333" 22:15:56 DEBUG ] 22:15:56 DEBUG } 22:15:56 DEBUG ] 22:15:56 DEBUG } 22:15:56 DEBUG } 22:15:57 INFO 22:15:57 INFO [loop_until]: kubectl --namespace=xlou top pods 22:15:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:15:57 INFO [loop_until]: OK (rc = 0) 22:15:57 DEBUG --- stdout --- 22:15:57 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 9597m 8810Mi am-767f6b67b5-qcqz2 11566m 8967Mi am-767f6b67b5-vbltz 11588m 8933Mi ds-cts-0 19068m 12314Mi ds-cts-1 18446m 12284Mi ds-cts-2 18641m 12470Mi ds-idrepo-0 4984m 3922Mi ds-idrepo-1 4756m 3966Mi ds-idrepo-2 4715m 3877Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 4m 1239Mi lodemon-7dd48676f6-tdxl9 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-967d957f7-zk66v 14535m 10257Mi 22:15:57 DEBUG --- stderr --- 22:15:57 DEBUG 22:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647759" 22:15:59 INFO [http_cmd]: http status code OK 22:15:59 DEBUG --- status code --- 22:15:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:15:59 DEBUG --- http response --- 22:15:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:15:59 DEBUG ----- output ----- 22:15:59 DEBUG { 22:15:59 DEBUG "status": "success", 22:15:59 DEBUG "data": { 22:15:59 DEBUG "resultType": "vector", 22:15:59 DEBUG "result": [ 22:15:59 DEBUG { 22:15:59 DEBUG "metric": { 22:15:59 DEBUG "instance": "10.142.0.107:9100" 22:15:59 DEBUG }, 22:15:59 DEBUG "value": [ 22:15:59 DEBUG 22:15:59 DEBUG 22:15:59 DEBUG 22:15:59 DEBUG [...] 22:15:59 DEBUG 22:15:59 DEBUG 22:15:59 DEBUG 22:15:59 DEBUG "instance": "10.142.0.43:9100" 22:15:59 DEBUG }, 22:15:59 DEBUG "value": [ 22:15:59 DEBUG 1713647759, 22:15:59 DEBUG "53857753.099999994" 22:15:59 DEBUG ] 22:15:59 DEBUG } 22:15:59 DEBUG ] 22:15:59 DEBUG } 22:15:59 DEBUG } 22:16:00 INFO 22:16:00 INFO [loop_until]: kubectl --namespace=xlou top node 22:16:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647760" 22:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647760" 22:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647760" 22:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647760" 22:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713647760" 22:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713647760" 22:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647760" 22:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647760" 22:16:00 INFO [http_cmd]: http status code OK 22:16:00 DEBUG --- status code --- 22:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647760" 22:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647760" 22:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647760" 22:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647760" 22:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647760" 22:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647760" 22:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647760" 22:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647760" 22:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647760" 22:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713647760" 22:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647760" 22:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647760" 22:16:00 INFO [http_cmd]: http status code OK 22:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647760" 22:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:00 INFO [http_cmd]: http status code OK 22:16:00 INFO [http_cmd]: http status code OK 22:16:00 INFO [http_cmd]: http status code OK 22:16:00 INFO [http_cmd]: http status code OK 22:16:00 INFO [http_cmd]: http status code OK 22:16:00 INFO [http_cmd]: http status code OK 22:16:00 INFO [http_cmd]: http status code OK 22:16:00 INFO [http_cmd]: http status code OK 22:16:00 DEBUG --- status code --- 22:16:00 INFO [http_cmd]: http status code OK 22:16:00 INFO [loop_until]: OK (rc = 0) 22:16:00 INFO [http_cmd]: http status code OK 22:16:00 INFO [http_cmd]: http status code OK 22:16:00 DEBUG --- status code --- 22:16:00 INFO [http_cmd]: http status code OK 22:16:00 DEBUG --- status code --- 22:16:00 INFO [http_cmd]: http status code OK 22:16:00 DEBUG --- http response --- 22:16:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:16:00 DEBUG --- status code --- 22:16:00 DEBUG --- status code --- 22:16:00 DEBUG --- status code --- 22:16:00 DEBUG --- status code --- 22:16:00 DEBUG --- status code --- 22:16:00 DEBUG --- status code --- 22:16:00 INFO [http_cmd]: http status code OK 22:16:00 INFO [http_cmd]: http status code OK 22:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:00 INFO [http_cmd]: http status code OK 22:16:00 DEBUG --- status code --- 22:16:00 DEBUG --- stdout --- 22:16:00 DEBUG --- status code --- 22:16:00 INFO [http_cmd]: http status code OK 22:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:00 DEBUG --- http response --- 22:16:00 DEBUG --- status code --- 22:16:00 DEBUG --- status code --- 22:16:00 DEBUG --- status code --- 22:16:00 DEBUG ----- output ----- 22:16:00 DEBUG { 22:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:00 DEBUG --- http response --- 22:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:00 DEBUG --- http response --- 22:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:00 DEBUG --- status code --- 22:16:00 DEBUG --- http response --- 22:16:00 DEBUG --- status code --- 22:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:00 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6232Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 4999m 16% 7684Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 11290m 37% 11559Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 85m 0% 3358Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 19010m 63% 15553Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 138m 0% 5191Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 11091m 37% 11884Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 10553m 35% 4059Mi 3% gke-xlou-cdm-ds-0f601751-hmm7 13031m 43% 11191Mi 10% gke-xlou-cdm-ds-0f601751-j488 5123m 17% 7297Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 18506m 61% 17369Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 5056m 16% 7949Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 19617m 65% 15166Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 14568m 91% 17428Mi 29% 22:16:00 DEBUG --- stderr --- 22:16:00 DEBUG --- status code --- 22:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:00 INFO [http_cmd]: http status code OK 22:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:00 DEBUG "status": "success", 22:16:00 INFO [http_cmd]: http status code OK 22:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:16:00 DEBUG ----- output ----- 22:16:00 DEBUG --- http response --- 22:16:00 DEBUG --- status code --- 22:16:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:16:00 DEBUG --- http response --- 22:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:00 DEBUG --- http response --- 22:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:00 DEBUG --- http response --- 22:16:00 DEBUG 22:16:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:16:00 DEBUG --- http response --- 22:16:00 DEBUG --- http response --- 22:16:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:16:00 DEBUG --- status code --- 22:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:00 DEBUG "data": { 22:16:00 DEBUG --- status code --- 22:16:00 DEBUG --- http response --- 22:16:00 DEBUG { 22:16:00 DEBUG --- http response --- 22:16:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:00 DEBUG ----- output ----- 22:16:00 DEBUG --- http response --- 22:16:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:16:00 DEBUG ----- output ----- 22:16:00 DEBUG --- http response --- 22:16:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:16:00 DEBUG ----- output ----- 22:16:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 22:16:00 DEBUG ----- output ----- 22:16:00 DEBUG { 22:16:00 DEBUG "status": "success", 22:16:00 DEBUG ----- output ----- 22:16:00 DEBUG --- http response --- 22:16:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:16:00 DEBUG ----- output ----- 22:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:16:00 DEBUG ----- output ----- 22:16:00 DEBUG { 22:16:00 DEBUG ----- output ----- 22:16:00 DEBUG --- http response --- 22:16:00 DEBUG { 22:16:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:16:00 DEBUG ----- output ----- 22:16:00 DEBUG { 22:16:00 DEBUG "status": "success", 22:16:00 DEBUG "data": { 22:16:00 DEBUG "resultType": "vector", 22:16:00 DEBUG "result": [ 22:16:00 DEBUG --- http response --- 22:16:00 DEBUG { 22:16:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:16:00 DEBUG { 22:16:00 DEBUG "resultType": "vector", 22:16:00 DEBUG { 22:16:00 DEBUG --- http response --- 22:16:00 DEBUG --- http response --- 22:16:00 DEBUG "status": "success", 22:16:00 DEBUG { 22:16:00 DEBUG "status": "success", 22:16:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 22:16:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:16:00 DEBUG "status": "success", 22:16:00 DEBUG "data": { 22:16:00 DEBUG "resultType": "vector", 22:16:00 DEBUG "result": [ 22:16:00 DEBUG { 22:16:00 DEBUG { 22:16:00 DEBUG ----- output ----- 22:16:00 DEBUG { 22:16:00 DEBUG "data": { 22:16:00 DEBUG "status": "success", 22:16:00 DEBUG ----- output ----- 22:16:00 DEBUG "status": "success", 22:16:00 DEBUG "result": [ 22:16:00 DEBUG { 22:16:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:16:00 DEBUG ----- output ----- 22:16:00 DEBUG "data": { 22:16:00 DEBUG "status": "success", 22:16:00 DEBUG "data": { 22:16:00 DEBUG "resultType": "vector", 22:16:00 DEBUG ----- output ----- 22:16:00 DEBUG "metric": { 22:16:00 DEBUG "pod": "am-767f6b67b5-9zw22" 22:16:00 DEBUG }, 22:16:00 DEBUG "value": [ 22:16:00 DEBUG ----- output ----- 22:16:00 DEBUG "status": "success", 22:16:00 DEBUG ----- output ----- 22:16:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:16:00 DEBUG ----- output ----- 22:16:00 DEBUG "data": { 22:16:00 DEBUG { 22:16:00 DEBUG "status": "success", 22:16:00 DEBUG "metric": { 22:16:00 DEBUG "pod": "ds-idrepo-0" 22:16:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:16:00 DEBUG { 22:16:00 DEBUG "resultType": "vector", 22:16:00 DEBUG "data": { 22:16:00 DEBUG "resultType": "vector", 22:16:00 DEBUG "result": [ 22:16:00 DEBUG { 22:16:00 DEBUG "metric": { 22:16:00 DEBUG "pod": "ds-idrepo-0" 22:16:00 DEBUG }, 22:16:00 DEBUG "value": [ 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG { 22:16:00 DEBUG { 22:16:00 DEBUG "resultType": "vector", 22:16:00 DEBUG "resultType": "vector", 22:16:00 DEBUG "data": { 22:16:00 DEBUG "data": { 22:16:00 DEBUG }, 22:16:00 DEBUG "status": "success", 22:16:00 DEBUG ----- output ----- 22:16:00 DEBUG { 22:16:00 DEBUG "status": "success", 22:16:00 DEBUG "data": { 22:16:00 DEBUG "result": [ 22:16:00 DEBUG "data": { 22:16:00 DEBUG { 22:16:00 DEBUG --- http response --- 22:16:00 DEBUG "result": [ 22:16:00 DEBUG "status": "success", 22:16:00 DEBUG "data": { 22:16:00 DEBUG "status": "success", 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG { 22:16:00 DEBUG "result": [ 22:16:00 DEBUG { 22:16:00 DEBUG "resultType": "vector", 22:16:00 DEBUG "resultType": "vector", 22:16:00 DEBUG "value": [ 22:16:00 DEBUG "data": { 22:16:00 DEBUG "resultType": "vector", 22:16:00 DEBUG "resultType": "vector", 22:16:00 DEBUG 22:16:00 DEBUG { 22:16:00 DEBUG "status": "success", 22:16:00 DEBUG "data": { 22:16:00 DEBUG "metric": { 22:16:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:16:00 DEBUG { 22:16:00 DEBUG "resultType": "vector", 22:16:00 DEBUG "result": [ 22:16:00 DEBUG "data": { 22:16:00 DEBUG "resultType": "vector", 22:16:00 DEBUG 22:16:00 DEBUG "status": "success", 22:16:00 DEBUG "metric": { 22:16:00 DEBUG "result": [ 22:16:00 DEBUG "result": [ 22:16:00 DEBUG "result": [ 22:16:00 DEBUG 22:16:00 DEBUG "result": [ 22:16:00 DEBUG { 22:16:00 DEBUG "result": [ 22:16:00 DEBUG { 22:16:00 DEBUG "metric": { 22:16:00 DEBUG "pod": "am-767f6b67b5-9zw22" 22:16:00 DEBUG { 22:16:00 DEBUG ----- output ----- 22:16:00 DEBUG { 22:16:00 DEBUG "metric": { 22:16:00 DEBUG { 22:16:00 DEBUG "metric": { 22:16:00 DEBUG "result": [ 22:16:00 DEBUG [...] 22:16:00 DEBUG "status": "success", 22:16:00 DEBUG "data": { 22:16:00 DEBUG "resultType": "vector", 22:16:00 DEBUG "data": { 22:16:00 DEBUG { 22:16:00 DEBUG "metric": { 22:16:00 DEBUG 22:16:00 DEBUG "metric": { 22:16:00 DEBUG "pod": "am-767f6b67b5-9zw22" 22:16:00 DEBUG }, 22:16:00 DEBUG "value": [ 22:16:00 DEBUG "resultType": "vector", 22:16:00 DEBUG "result": [ 22:16:00 DEBUG "metric": { 22:16:00 DEBUG "status": "success", 22:16:00 DEBUG "data": { 22:16:00 DEBUG "resultType": "vector", 22:16:00 DEBUG "instance": "10.142.0.107:9100" 22:16:00 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:16:00 DEBUG }, 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG "result": [ 22:16:00 DEBUG "resultType": "vector", 22:16:00 DEBUG "result": [ 22:16:00 DEBUG "pod": "ds-cts-0" 22:16:00 DEBUG }, 22:16:00 DEBUG "instance": "10.142.0.107:9100" 22:16:00 DEBUG }, 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG { 22:16:00 DEBUG [...] 22:16:00 DEBUG "result": [ 22:16:00 DEBUG { 22:16:00 DEBUG "metric": { 22:16:00 DEBUG "pod": "ds-idrepo-0" 22:16:00 DEBUG "value": [ 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG }, 22:16:00 DEBUG { 22:16:00 DEBUG "metric": { 22:16:00 DEBUG "pod": "am-767f6b67b5-9zw22" 22:16:00 DEBUG { 22:16:00 DEBUG "metric": { 22:16:00 DEBUG "pod": "ds-cts-0" 22:16:00 DEBUG }, 22:16:00 DEBUG 22:16:00 DEBUG [...] 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG }, 22:16:00 DEBUG "value": [ 22:16:00 DEBUG "resultType": "vector", 22:16:00 DEBUG "instance": "10.142.0.107:9100" 22:16:00 DEBUG }, 22:16:00 DEBUG "value": [ 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG "value": [ 22:16:00 DEBUG 22:16:00 DEBUG }, 22:16:00 DEBUG { 22:16:00 DEBUG "value": [ 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG [...] 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG "instance": "10.142.0.107:9100" 22:16:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:16:00 DEBUG "result": [ 22:16:00 DEBUG 22:16:00 DEBUG [...] 22:16:00 DEBUG [...] 22:16:00 DEBUG 22:16:00 DEBUG "value": [ 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG [...] 22:16:00 DEBUG }, 22:16:00 DEBUG "value": [ 22:16:00 DEBUG 22:16:00 DEBUG "value": [ 22:16:00 DEBUG "metric": { 22:16:00 DEBUG "status": "success", 22:16:00 DEBUG 22:16:00 DEBUG "pod": "am-767f6b67b5-vbltz" 22:16:00 DEBUG 22:16:00 DEBUG ----- output ----- 22:16:00 DEBUG { 22:16:00 DEBUG { 22:16:00 DEBUG 22:16:00 DEBUG "pod": "am-767f6b67b5-vbltz" 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG { 22:16:00 DEBUG "instance": "10.142.0.43:9100" 22:16:00 DEBUG 22:16:00 DEBUG "value": [ 22:16:00 DEBUG "metric": { 22:16:00 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:16:00 DEBUG }, 22:16:00 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:16:00 DEBUG }, 22:16:00 DEBUG }, 22:16:00 DEBUG }, 22:16:00 DEBUG [...] 22:16:00 DEBUG { 22:16:00 DEBUG "metric": { 22:16:00 DEBUG "metric": { 22:16:00 DEBUG 22:16:00 DEBUG }, 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG "metric": { 22:16:00 DEBUG }, 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG "value": [ 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG "value": [ 22:16:00 DEBUG "data": { 22:16:00 DEBUG "value": [ 22:16:00 DEBUG "value": [ 22:16:00 DEBUG 1713647760, 22:16:00 DEBUG "92.59" 22:16:00 DEBUG ] 22:16:00 DEBUG } 22:16:00 DEBUG 22:16:00 DEBUG "value": [ 22:16:00 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:16:00 DEBUG [...] 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG "status": "success", 22:16:00 DEBUG "data": { 22:16:00 DEBUG "resultType": "vector", 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG [...] 22:16:00 DEBUG 22:16:00 DEBUG "resultType": "vector", 22:16:00 DEBUG 22:16:00 DEBUG ] 22:16:00 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG 1713647760, 22:16:00 DEBUG }, 22:16:00 DEBUG 22:16:00 DEBUG "pod": "ds-idrepo-2" 22:16:00 DEBUG "pod": "ds-cts-0" 22:16:00 DEBUG 22:16:00 DEBUG "pod": "am-767f6b67b5-vbltz" 22:16:00 DEBUG "value": [ 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG [...] 22:16:00 DEBUG 22:16:00 DEBUG "result": [ 22:16:00 DEBUG 22:16:00 DEBUG } 22:16:00 DEBUG }, 22:16:00 DEBUG "pod": "ds-idrepo-2" 22:16:00 DEBUG "pod": "am-767f6b67b5-9zw22" 22:16:00 DEBUG "instance": "10.142.0.43:9100" 22:16:00 DEBUG "1823.1333333333332" 22:16:00 DEBUG "value": [ 22:16:00 DEBUG 22:16:00 DEBUG }, 22:16:00 DEBUG }, 22:16:00 DEBUG }, 22:16:00 DEBUG "result": [ 22:16:00 DEBUG 1713647760, 22:16:00 DEBUG "0.44726666666663467" 22:16:00 DEBUG [...] 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG { 22:16:00 DEBUG 22:16:00 DEBUG } 22:16:00 DEBUG "value": [ 22:16:00 DEBUG }, 22:16:00 DEBUG }, 22:16:00 DEBUG }, 22:16:00 DEBUG ] 22:16:00 DEBUG 1713647760, 22:16:00 DEBUG "instance": "10.142.0.43:9100" 22:16:00 DEBUG "value": [ 22:16:00 DEBUG "value": [ 22:16:00 DEBUG "value": [ 22:16:00 DEBUG { 22:16:00 DEBUG ] 22:16:00 DEBUG [...] 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG "pod": "ds-idrepo-2" 22:16:00 DEBUG }, 22:16:00 DEBUG "value": [ 22:16:00 DEBUG 1713647760, 22:16:00 DEBUG [...] 22:16:00 DEBUG 22:16:00 DEBUG "value": [ 22:16:00 DEBUG "value": [ 22:16:00 DEBUG "value": [ 22:16:00 DEBUG } 22:16:00 DEBUG "14.803240552034405" 22:16:00 DEBUG }, 22:16:00 DEBUG 1713647760, 22:16:00 DEBUG "0" 22:16:00 DEBUG 1713647760, 22:16:00 DEBUG "metric": { 22:16:00 DEBUG } 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG [...] 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG 1713647760, 22:16:00 DEBUG 22:16:00 DEBUG 1713647760, 22:16:00 DEBUG ] 22:16:00 DEBUG } 22:16:00 DEBUG } 22:16:00 DEBUG 22:16:00 DEBUG ] 22:16:00 DEBUG "446837" 22:16:00 DEBUG ] 22:16:00 DEBUG } 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG ] 22:16:00 DEBUG "metric": { 22:16:00 DEBUG 22:16:00 DEBUG "pod": "overseer-0-967d957f7-zk66v" 22:16:00 DEBUG "0" 22:16:00 DEBUG 22:16:00 DEBUG [...] 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG "53862714.13333333" 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG "value": [ 22:16:00 DEBUG } 22:16:00 DEBUG ] 22:16:00 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG "pod": "ds-cts-2" 22:16:00 DEBUG } 22:16:00 DEBUG "pod": "am-767f6b67b5-9zw22" 22:16:00 DEBUG }, 22:16:00 DEBUG 22:16:00 DEBUG "instance": "10.142.0.43:9100" 22:16:00 DEBUG ] 22:16:00 DEBUG "0" 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG ] 22:16:00 DEBUG } 22:16:00 DEBUG [...] 22:16:00 DEBUG 1713647760, 22:16:00 DEBUG ] 22:16:00 DEBUG } 22:16:00 DEBUG }, 22:16:00 DEBUG "pod": "am-767f6b67b5-vbltz" 22:16:00 DEBUG [...] 22:16:00 DEBUG }, 22:16:00 DEBUG } 22:16:00 DEBUG }, 22:16:00 DEBUG "value": [ 22:16:00 DEBUG 22:16:00 DEBUG }, 22:16:00 DEBUG } 22:16:00 DEBUG ] 22:16:00 DEBUG [...] 22:16:00 DEBUG "pod": "ds-idrepo-2" 22:16:00 DEBUG }, 22:16:00 DEBUG "value": [ 22:16:00 DEBUG 22:16:00 DEBUG "4108402193408" 22:16:00 DEBUG } 22:16:00 DEBUG } 22:16:00 DEBUG "value": [ 22:16:00 DEBUG }, 22:16:00 DEBUG "value": [ 22:16:00 DEBUG 1713647760, 22:16:00 DEBUG "value": [ 22:16:00 DEBUG 1713647760, 22:16:00 DEBUG 22:16:00 DEBUG "value": [ 22:16:00 DEBUG ] 22:16:00 DEBUG } 22:16:00 DEBUG 22:16:00 DEBUG ] 22:16:00 DEBUG 1713647760, 22:16:00 DEBUG ] 22:16:00 DEBUG 22:16:00 DEBUG ] 22:16:00 DEBUG } 22:16:00 DEBUG 22:16:00 DEBUG "value": [ 22:16:00 DEBUG 1713647760, 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG "0" 22:16:00 DEBUG 22:16:00 DEBUG "pod": "ds-idrepo-2" 22:16:00 DEBUG }, 22:16:00 DEBUG "value": [ 22:16:00 DEBUG 1713647760, 22:16:00 DEBUG } 22:16:00 DEBUG ] 22:16:00 DEBUG } 22:16:00 DEBUG } 22:16:00 DEBUG } 22:16:00 DEBUG 22:16:00 DEBUG "28.808433333333333" 22:16:00 DEBUG "1345.3666666666666" 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG ] 22:16:00 DEBUG "0" 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG } 22:16:00 DEBUG 22:16:00 DEBUG } 22:16:00 DEBUG "4303.987277064637" 22:16:00 DEBUG } 22:16:00 DEBUG } 22:16:00 DEBUG ] 22:16:00 DEBUG 22:16:00 DEBUG ] 22:16:00 DEBUG } 22:16:00 DEBUG ] 22:16:00 DEBUG 22:16:00 DEBUG [...] 22:16:00 DEBUG ] 22:16:00 DEBUG } 22:16:00 DEBUG "pod": "am-767f6b67b5-vbltz" 22:16:00 DEBUG "pod": "ds-cts-2" 22:16:00 DEBUG ] 22:16:00 DEBUG ] 22:16:00 DEBUG } 22:16:00 DEBUG [...] 22:16:00 DEBUG } 22:16:00 DEBUG } 22:16:00 DEBUG } 22:16:00 DEBUG ] 22:16:00 DEBUG 22:16:00 DEBUG ] 22:16:00 DEBUG } 22:16:00 DEBUG 1713647760, 22:16:00 DEBUG "0.07566666666666606" 22:16:00 DEBUG } 22:16:00 DEBUG } 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG } 22:16:00 DEBUG } 22:16:00 DEBUG } 22:16:00 DEBUG ] 22:16:00 DEBUG } 22:16:00 DEBUG }, 22:16:00 DEBUG ] 22:16:00 DEBUG } 22:16:00 DEBUG ] 22:16:00 DEBUG 22:16:00 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:16:00 DEBUG }, 22:16:00 DEBUG 22:16:00 DEBUG 22:16:00 DEBUG "pod": "am-767f6b67b5-vbltz" 22:16:00 DEBUG "value": [ 22:16:00 DEBUG } 22:16:00 DEBUG } 22:16:00 DEBUG "value": [ 22:16:00 DEBUG 22:16:00 DEBUG "pod": "ds-cts-2" 22:16:00 DEBUG } 22:16:00 DEBUG 1713647760, 22:16:00 DEBUG "1821.7333333333333" 22:16:00 DEBUG ] 22:16:00 DEBUG }, 22:16:00 DEBUG "value": [ 22:16:00 DEBUG }, 22:16:00 DEBUG "value": [ 22:16:00 DEBUG 1713647760, 22:16:00 DEBUG } 22:16:00 DEBUG }, 22:16:00 DEBUG "value": [ 22:16:00 DEBUG 1713647760, 22:16:00 DEBUG 1713647760, 22:16:00 DEBUG "0" 22:16:00 DEBUG ] 22:16:00 DEBUG ] 22:16:00 DEBUG "28.2" 22:16:00 DEBUG 1713647760, 22:16:00 DEBUG "0" 22:16:00 DEBUG ] 22:16:00 DEBUG } 22:16:00 DEBUG ] 22:16:00 DEBUG } 22:16:00 DEBUG } 22:16:00 DEBUG ] 22:16:00 DEBUG "8425496576" 22:16:00 DEBUG } 22:16:00 DEBUG } 22:16:00 DEBUG ] 22:16:00 DEBUG } 22:16:00 DEBUG ] 22:16:00 DEBUG } 22:16:00 DEBUG } 22:16:00 DEBUG } 22:16:00 DEBUG } 22:16:00 DEBUG } 22:16:00 DEBUG ] 22:16:00 DEBUG ] 22:16:00 DEBUG } 22:16:00 DEBUG } 22:16:00 DEBUG } 22:16:00 DEBUG } 22:16:00 DEBUG ] 22:16:00 DEBUG ] 22:16:00 DEBUG } 22:16:00 DEBUG } 22:16:00 DEBUG } 22:16:00 DEBUG } 22:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713647815" 22:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647815" 22:16:55 ERROR [http_cmd]: ERROR 22:16:55 ERROR --- status code --- 22:16:55 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:16:55 ERROR --- http response --- 22:16:55 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:16:55 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:16:55 INFO [http_cmd]: http status code OK 22:16:55 DEBUG --- status code --- 22:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:55 DEBUG --- http response --- 22:16:55 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713647815" 22:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647815" 22:16:55 INFO [http_cmd]: http status code OK 22:16:55 INFO [http_cmd]: http status code OK 22:16:55 DEBUG --- status code --- 22:16:55 DEBUG --- status code --- 22:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:55 DEBUG --- http response --- 22:16:55 DEBUG --- http response --- 22:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:16:55 DEBUG ----- output ----- 22:16:55 DEBUG ----- output ----- 22:16:55 DEBUG { 22:16:55 DEBUG { 22:16:55 DEBUG "status": "success", 22:16:55 DEBUG "status": "success", 22:16:55 DEBUG "data": { 22:16:55 DEBUG "data": { 22:16:55 DEBUG "resultType": "vector", 22:16:55 DEBUG "resultType": "vector", 22:16:55 DEBUG "result": [ 22:16:55 DEBUG { 22:16:55 DEBUG "result": [ 22:16:55 DEBUG "metric": { 22:16:55 DEBUG { 22:16:55 DEBUG "pod": "ds-idrepo-1" 22:16:55 DEBUG "metric": { 22:16:55 DEBUG }, 22:16:55 DEBUG "pod": "ds-idrepo-0" 22:16:55 DEBUG "value": [ 22:16:55 DEBUG }, 22:16:55 DEBUG 22:16:55 DEBUG "value": [ 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG [...] 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG [...] 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG "pod": "overseer-0-967d957f7-zk66v" 22:16:55 DEBUG 22:16:55 DEBUG }, 22:16:55 DEBUG "pod": "ds-cts-2" 22:16:55 DEBUG "value": [ 22:16:55 DEBUG }, 22:16:55 DEBUG 1713647815, 22:16:55 DEBUG "value": [ 22:16:55 DEBUG "50323947.59505268" 22:16:55 DEBUG 1713647815, 22:16:55 DEBUG ] 22:16:55 DEBUG "0" 22:16:55 DEBUG } 22:16:55 DEBUG ] 22:16:55 DEBUG ] 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG ] 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647817" 22:16:57 INFO [http_cmd]: http status code OK 22:16:57 DEBUG --- status code --- 22:16:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:57 DEBUG --- http response --- 22:16:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:16:57 DEBUG ----- output ----- 22:16:57 DEBUG { 22:16:57 DEBUG "status": "success", 22:16:57 DEBUG "data": { 22:16:57 DEBUG "resultType": "vector", 22:16:57 DEBUG "result": [ 22:16:57 DEBUG { 22:16:57 DEBUG "metric": { 22:16:57 DEBUG "pod": "am-767f6b67b5-9zw22" 22:16:57 DEBUG }, 22:16:57 DEBUG "value": [ 22:16:57 DEBUG 22:16:57 DEBUG 22:16:57 DEBUG 22:16:57 DEBUG [...] 22:16:57 DEBUG 22:16:57 DEBUG 22:16:57 DEBUG 22:16:57 DEBUG "pod": "am-767f6b67b5-vbltz" 22:16:57 DEBUG }, 22:16:57 DEBUG "value": [ 22:16:57 DEBUG 1713647817, 22:16:57 DEBUG "1997.2" 22:16:57 DEBUG ] 22:16:57 DEBUG } 22:16:57 DEBUG ] 22:16:57 DEBUG } 22:16:57 DEBUG } 22:16:57 INFO 22:16:57 INFO [loop_until]: kubectl --namespace=xlou top pods 22:16:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:16:57 INFO [loop_until]: OK (rc = 0) 22:16:57 DEBUG --- stdout --- 22:16:57 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 14877m 8816Mi am-767f6b67b5-qcqz2 14773m 8971Mi am-767f6b67b5-vbltz 15862m 8941Mi ds-cts-0 19151m 12314Mi ds-cts-1 19352m 12277Mi ds-cts-2 19944m 12468Mi ds-idrepo-0 5962m 3927Mi ds-idrepo-1 6073m 3966Mi ds-idrepo-2 5887m 3878Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-7dd48676f6-tdxl9 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-967d957f7-zk66v 14906m 13554Mi 22:16:57 DEBUG --- stderr --- 22:16:57 DEBUG 22:16:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647819" 22:16:59 INFO [http_cmd]: http status code OK 22:16:59 DEBUG --- status code --- 22:16:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:59 DEBUG --- http response --- 22:16:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:16:59 DEBUG ----- output ----- 22:16:59 DEBUG { 22:16:59 DEBUG "status": "success", 22:16:59 DEBUG "data": { 22:16:59 DEBUG "resultType": "vector", 22:16:59 DEBUG "result": [ 22:16:59 DEBUG { 22:16:59 DEBUG "metric": { 22:16:59 DEBUG "instance": "10.142.0.107:9100" 22:16:59 DEBUG }, 22:16:59 DEBUG "value": [ 22:16:59 DEBUG 22:16:59 DEBUG 22:16:59 DEBUG 22:16:59 DEBUG [...] 22:16:59 DEBUG 22:16:59 DEBUG 22:16:59 DEBUG 22:16:59 DEBUG "instance": "10.142.0.43:9100" 22:16:59 DEBUG }, 22:16:59 DEBUG "value": [ 22:16:59 DEBUG 1713647819, 22:16:59 DEBUG "70018657.63333333" 22:16:59 DEBUG ] 22:16:59 DEBUG } 22:16:59 DEBUG ] 22:16:59 DEBUG } 22:16:59 DEBUG } 22:17:00 INFO 22:17:00 INFO [loop_until]: kubectl --namespace=xlou top node 22:17:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:17:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713647820" 22:17:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647820" 22:17:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713647820" 22:17:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647820" 22:17:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647820" 22:17:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647820" 22:17:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647820" 22:17:00 INFO [http_cmd]: http status code OK 22:17:00 INFO [http_cmd]: http status code OK 22:17:00 DEBUG --- status code --- 22:17:00 DEBUG --- status code --- 22:17:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:00 DEBUG --- http response --- 22:17:00 DEBUG --- http response --- 22:17:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:17:00 INFO [http_cmd]: http status code OK 22:17:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:17:00 DEBUG ----- output ----- 22:17:00 DEBUG --- status code --- 22:17:00 DEBUG ----- output ----- 22:17:00 DEBUG { 22:17:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:00 DEBUG { 22:17:00 DEBUG "status": "success", 22:17:00 DEBUG --- http response --- 22:17:00 DEBUG "status": "success", 22:17:00 DEBUG "data": { 22:17:00 DEBUG "data": { 22:17:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:17:00 DEBUG "resultType": "vector", 22:17:00 DEBUG "resultType": "vector", 22:17:00 INFO [http_cmd]: http status code OK 22:17:00 DEBUG ----- output ----- 22:17:00 DEBUG "result": [ 22:17:00 DEBUG "result": [ 22:17:00 DEBUG --- status code --- 22:17:00 DEBUG { 22:17:00 DEBUG { 22:17:00 DEBUG { 22:17:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:00 DEBUG "status": "success", 22:17:00 DEBUG "metric": { 22:17:00 DEBUG "metric": { 22:17:00 DEBUG --- http response --- 22:17:00 DEBUG "data": { 22:17:00 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:17:00 DEBUG "pod": "am-767f6b67b5-9zw22" 22:17:00 DEBUG "resultType": "vector", 22:17:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:17:00 DEBUG }, 22:17:00 DEBUG }, 22:17:00 DEBUG "result": [ 22:17:00 DEBUG { 22:17:00 DEBUG ----- output ----- 22:17:00 DEBUG "value": [ 22:17:00 DEBUG "value": [ 22:17:00 DEBUG "metric": { 22:17:00 DEBUG { 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG "pod": "am-767f6b67b5-9zw22" 22:17:00 DEBUG "status": "success", 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG }, 22:17:00 DEBUG "data": { 22:17:00 DEBUG 22:17:00 DEBUG [...] 22:17:00 DEBUG "value": [ 22:17:00 DEBUG "resultType": "vector", 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG "result": [ 22:17:00 DEBUG [...] 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG { 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG "metric": { 22:17:00 DEBUG 22:17:00 DEBUG "pod": "am-767f6b67b5-vbltz" 22:17:00 INFO [http_cmd]: http status code OK 22:17:00 DEBUG [...] 22:17:00 DEBUG "pod": "am-767f6b67b5-9zw22" 22:17:00 INFO [http_cmd]: http status code OK 22:17:00 DEBUG 22:17:00 DEBUG }, 22:17:00 DEBUG --- status code --- 22:17:00 INFO [http_cmd]: http status code OK 22:17:00 DEBUG --- status code --- 22:17:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:00 DEBUG --- status code --- 22:17:00 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:17:00 DEBUG "value": [ 22:17:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:00 DEBUG --- http response --- 22:17:00 DEBUG 22:17:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:00 DEBUG }, 22:17:00 DEBUG 1713647820, 22:17:00 DEBUG --- http response --- 22:17:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:17:00 DEBUG 22:17:00 DEBUG --- http response --- 22:17:00 DEBUG "value": [ 22:17:00 DEBUG "1997.2333333333333" 22:17:00 DEBUG }, 22:17:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:17:00 DEBUG ----- output ----- 22:17:00 DEBUG 22:17:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:17:00 DEBUG 1713647820, 22:17:00 DEBUG ] 22:17:00 DEBUG } 22:17:00 DEBUG ----- output ----- 22:17:00 DEBUG { 22:17:00 DEBUG "pod": "am-767f6b67b5-vbltz" 22:17:00 DEBUG ----- output ----- 22:17:00 DEBUG { 22:17:00 DEBUG "status": "success", 22:17:00 DEBUG "value": [ 22:17:00 DEBUG { 22:17:00 DEBUG "status": "success", 22:17:00 DEBUG }, 22:17:00 DEBUG "14.83713628117102" 22:17:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647820" 22:17:00 DEBUG "data": { 22:17:00 DEBUG ] 22:17:00 DEBUG 22:17:00 DEBUG "status": "success", 22:17:00 DEBUG "data": { 22:17:00 DEBUG "value": [ 22:17:00 DEBUG ] 22:17:00 DEBUG "resultType": "vector", 22:17:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647820" 22:17:00 DEBUG } 22:17:00 DEBUG } 22:17:00 DEBUG "data": { 22:17:00 DEBUG "resultType": "vector", 22:17:00 DEBUG 1713647820, 22:17:00 DEBUG } 22:17:00 DEBUG "result": [ 22:17:00 DEBUG 22:17:00 DEBUG "resultType": "vector", 22:17:00 DEBUG "result": [ 22:17:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647820" 22:17:00 DEBUG "result": [ 22:17:00 DEBUG "94.081" 22:17:00 DEBUG ] 22:17:00 DEBUG { 22:17:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647820" 22:17:00 DEBUG 22:17:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647820" 22:17:00 DEBUG { 22:17:00 DEBUG { 22:17:00 DEBUG ] 22:17:00 DEBUG } 22:17:00 DEBUG } 22:17:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647820" 22:17:00 DEBUG "metric": { 22:17:00 INFO [http_cmd]: http status code OK 22:17:00 DEBUG [...] 22:17:00 DEBUG 22:17:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647820" 22:17:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647820" 22:17:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647820" 22:17:00 DEBUG "metric": { 22:17:00 DEBUG } 22:17:00 DEBUG ] 22:17:00 INFO [http_cmd]: http status code OK 22:17:00 DEBUG "instance": "10.142.0.107:9100" 22:17:00 DEBUG --- status code --- 22:17:00 DEBUG 22:17:00 DEBUG "metric": { 22:17:00 INFO [http_cmd]: http status code OK 22:17:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647820" 22:17:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647820" 22:17:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713647820" 22:17:00 INFO [loop_until]: OK (rc = 0) 22:17:00 INFO [http_cmd]: http status code OK 22:17:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647820" 22:17:00 INFO [http_cmd]: http status code OK 22:17:00 DEBUG "pod": "ds-cts-0" 22:17:00 INFO [http_cmd]: http status code OK 22:17:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647820" 22:17:00 DEBUG } 22:17:00 DEBUG --- status code --- 22:17:00 DEBUG }, 22:17:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:00 DEBUG 22:17:00 DEBUG "pod": "ds-idrepo-0" 22:17:00 DEBUG --- status code --- 22:17:00 DEBUG --- stdout --- 22:17:00 DEBUG --- status code --- 22:17:00 DEBUG --- status code --- 22:17:00 DEBUG }, 22:17:00 INFO [http_cmd]: http status code OK 22:17:00 DEBUG --- status code --- 22:17:00 DEBUG } 22:17:00 INFO [http_cmd]: http status code OK 22:17:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:00 DEBUG "value": [ 22:17:00 DEBUG --- http response --- 22:17:00 DEBUG "pod": "am-767f6b67b5-vbltz" 22:17:00 DEBUG }, 22:17:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:00 INFO [http_cmd]: http status code OK 22:17:00 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 6219Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 6227m 20% 7689Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 14900m 49% 11563Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 81m 0% 3365Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 19591m 65% 15539Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 152m 0% 5197Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 15362m 51% 11895Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12558m 42% 3928Mi 3% gke-xlou-cdm-ds-0f601751-hmm7 16544m 55% 11182Mi 10% gke-xlou-cdm-ds-0f601751-j488 6433m 21% 7297Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 18906m 63% 17368Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6365m 21% 7946Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 19999m 66% 15166Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 14838m 93% 20110Mi 34% 22:17:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:00 DEBUG "value": [ 22:17:00 DEBUG --- status code --- 22:17:00 INFO [http_cmd]: http status code OK 22:17:00 INFO [http_cmd]: http status code OK 22:17:00 DEBUG --- status code --- 22:17:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:00 INFO [http_cmd]: http status code OK 22:17:00 DEBUG --- status code --- 22:17:00 DEBUG --- http response --- 22:17:00 DEBUG 22:17:00 DEBUG }, 22:17:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:17:00 DEBUG "value": [ 22:17:00 DEBUG --- http response --- 22:17:00 DEBUG --- status code --- 22:17:00 DEBUG --- stderr --- 22:17:00 DEBUG --- http response --- 22:17:00 DEBUG --- http response --- 22:17:00 DEBUG 22:17:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:00 DEBUG --- status code --- 22:17:00 INFO [http_cmd]: http status code OK 22:17:00 INFO [http_cmd]: http status code OK 22:17:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:00 DEBUG --- http response --- 22:17:00 DEBUG --- status code --- 22:17:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:00 DEBUG --- http response --- 22:17:00 DEBUG 22:17:00 DEBUG "value": [ 22:17:00 DEBUG ----- output ----- 22:17:00 DEBUG 22:17:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:17:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:00 DEBUG 22:17:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:17:00 DEBUG 22:17:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:17:00 DEBUG --- http response --- 22:17:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:00 DEBUG --- status code --- 22:17:00 DEBUG --- status code --- 22:17:00 DEBUG --- http response --- 22:17:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:17:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:17:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:17:00 DEBUG 22:17:00 DEBUG 1713647820, 22:17:00 DEBUG { 22:17:00 DEBUG 22:17:00 DEBUG ----- output ----- 22:17:00 DEBUG --- http response --- 22:17:00 DEBUG ----- output ----- 22:17:00 DEBUG 22:17:00 DEBUG ----- output ----- 22:17:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:17:00 DEBUG --- http response --- 22:17:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:17:00 DEBUG --- http response --- 22:17:00 DEBUG ----- output ----- 22:17:00 DEBUG ----- output ----- 22:17:00 DEBUG ----- output ----- 22:17:00 DEBUG [...] 22:17:00 DEBUG "3810.7333333333336" 22:17:00 DEBUG "status": "success", 22:17:00 DEBUG 22:17:00 DEBUG { 22:17:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:17:00 DEBUG { 22:17:00 DEBUG [...] 22:17:00 DEBUG { 22:17:00 DEBUG ----- output ----- 22:17:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:17:00 DEBUG --- http response --- 22:17:00 DEBUG --- http response --- 22:17:00 DEBUG ----- output ----- 22:17:00 DEBUG { 22:17:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:17:00 DEBUG { 22:17:00 DEBUG { 22:17:00 DEBUG 22:17:00 DEBUG ] 22:17:00 DEBUG "data": { 22:17:00 DEBUG [...] 22:17:00 DEBUG "status": "success", 22:17:00 DEBUG ----- output ----- 22:17:00 DEBUG "status": "success", 22:17:00 DEBUG 22:17:00 DEBUG "status": "success", 22:17:00 DEBUG { 22:17:00 DEBUG ----- output ----- 22:17:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:17:00 DEBUG { 22:17:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:17:00 DEBUG "status": "success", 22:17:00 DEBUG ----- output ----- 22:17:00 DEBUG "status": "success", 22:17:00 DEBUG "status": "success", 22:17:00 DEBUG 22:17:00 DEBUG } 22:17:00 DEBUG "resultType": "vector", 22:17:00 DEBUG 22:17:00 DEBUG "data": { 22:17:00 DEBUG { 22:17:00 DEBUG "data": { 22:17:00 DEBUG 22:17:00 DEBUG "data": { 22:17:00 DEBUG "status": "success", 22:17:00 DEBUG { 22:17:00 DEBUG ----- output ----- 22:17:00 DEBUG "status": "success", 22:17:00 DEBUG ----- output ----- 22:17:00 DEBUG "data": { 22:17:00 DEBUG { 22:17:00 DEBUG "data": { 22:17:00 DEBUG "data": { 22:17:00 DEBUG "resultType": "vector", 22:17:00 DEBUG ] 22:17:00 DEBUG } 22:17:00 DEBUG 22:17:00 DEBUG "resultType": "vector", 22:17:00 DEBUG "status": "success", 22:17:00 DEBUG "resultType": "vector", 22:17:00 DEBUG 22:17:00 DEBUG "resultType": "vector", 22:17:00 DEBUG "data": { 22:17:00 DEBUG "status": "success", 22:17:00 DEBUG { 22:17:00 DEBUG "data": { 22:17:00 DEBUG { 22:17:00 DEBUG "resultType": "vector", 22:17:00 DEBUG "status": "success", 22:17:00 DEBUG "resultType": "vector", 22:17:00 DEBUG "result": [ 22:17:00 DEBUG 22:17:00 DEBUG } 22:17:00 DEBUG 22:17:00 DEBUG "result": [ 22:17:00 DEBUG "result": [ 22:17:00 DEBUG "data": { 22:17:00 DEBUG "result": [ 22:17:00 DEBUG "pod": "ds-cts-2" 22:17:00 DEBUG "result": [ 22:17:00 DEBUG "resultType": "vector", 22:17:00 DEBUG "data": { 22:17:00 DEBUG "status": "success", 22:17:00 DEBUG "resultType": "vector", 22:17:00 DEBUG "status": "success", 22:17:00 DEBUG "result": [ 22:17:00 DEBUG { 22:17:00 DEBUG "result": [ 22:17:00 DEBUG { 22:17:00 DEBUG "instance": "10.142.0.43:9100" 22:17:00 DEBUG "pod": "overseer-0-967d957f7-zk66v" 22:17:00 DEBUG { 22:17:00 DEBUG { 22:17:00 DEBUG "resultType": "vector", 22:17:00 DEBUG { 22:17:00 DEBUG }, 22:17:00 DEBUG { 22:17:00 DEBUG "result": [ 22:17:00 DEBUG "resultType": "vector", 22:17:00 DEBUG "data": { 22:17:00 DEBUG "result": [ 22:17:00 DEBUG "data": { 22:17:00 DEBUG "data": { 22:17:00 DEBUG "metric": { 22:17:00 DEBUG { 22:17:00 DEBUG "metric": { 22:17:00 DEBUG }, 22:17:00 DEBUG }, 22:17:00 DEBUG "metric": { 22:17:00 DEBUG "metric": { 22:17:00 DEBUG "result": [ 22:17:00 DEBUG "metric": { 22:17:00 DEBUG "value": [ 22:17:00 DEBUG "metric": { 22:17:00 DEBUG { 22:17:00 DEBUG "metric": { 22:17:00 DEBUG "resultType": "vector", 22:17:00 DEBUG { 22:17:00 DEBUG "resultType": "vector", 22:17:00 DEBUG "resultType": "vector", 22:17:00 DEBUG "pod": "ds-idrepo-0" 22:17:00 DEBUG "metric": { 22:17:00 DEBUG "pod": "ds-idrepo-0" 22:17:00 DEBUG "value": [ 22:17:00 DEBUG 1713647820, 22:17:00 DEBUG "pod": "am-767f6b67b5-9zw22" 22:17:00 DEBUG "instance": "10.142.0.107:9100" 22:17:00 DEBUG { 22:17:00 DEBUG "instance": "10.142.0.107:9100" 22:17:00 DEBUG 1713647820, 22:17:00 DEBUG "pod": "ds-idrepo-0" 22:17:00 DEBUG }, 22:17:00 DEBUG "value": [ 22:17:00 DEBUG "result": [ 22:17:00 DEBUG { 22:17:00 DEBUG "metric": { 22:17:00 DEBUG "result": [ 22:17:00 DEBUG "metric": { 22:17:00 DEBUG "pod": "ds-idrepo-0" 22:17:00 DEBUG }, 22:17:00 DEBUG "0.10900000000000125" 22:17:00 DEBUG "value": [ 22:17:00 DEBUG }, 22:17:00 DEBUG }, 22:17:00 DEBUG "metric": { 22:17:00 DEBUG }, 22:17:00 DEBUG "0.27393333333333203" 22:17:00 DEBUG 22:17:00 DEBUG "result": [ 22:17:00 DEBUG { 22:17:00 DEBUG "metric": { 22:17:00 DEBUG "pod": "am-767f6b67b5-9zw22" 22:17:00 DEBUG }, 22:17:00 DEBUG "result": [ 22:17:00 DEBUG "pod": "ds-idrepo-0" 22:17:00 DEBUG }, 22:17:00 DEBUG "value": [ 22:17:00 DEBUG ] 22:17:00 DEBUG 1713647820, 22:17:00 DEBUG "value": [ 22:17:00 DEBUG "value": [ 22:17:00 DEBUG "pod": "ds-cts-0" 22:17:00 DEBUG }, 22:17:00 DEBUG ] 22:17:00 DEBUG 22:17:00 DEBUG "instance": "10.142.0.107:9100" 22:17:00 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:17:00 DEBUG }, 22:17:00 DEBUG "value": [ 22:17:00 DEBUG { 22:17:00 DEBUG }, 22:17:00 DEBUG { 22:17:00 DEBUG "value": [ 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG [...] 22:17:00 DEBUG 22:17:00 DEBUG "value": [ 22:17:00 DEBUG "value": [ 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG }, 22:17:00 DEBUG }, 22:17:00 DEBUG "value": [ 22:17:00 DEBUG 22:17:00 DEBUG "metric": { 22:17:00 DEBUG "value": [ 22:17:00 DEBUG "metric": { 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG } 22:17:00 DEBUG "0" 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG } 22:17:00 DEBUG [...] 22:17:00 DEBUG "value": [ 22:17:00 DEBUG "value": [ 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG "pod": "ds-cts-0" 22:17:00 DEBUG 22:17:00 DEBUG "pod": "am-767f6b67b5-9zw22" 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG ] 22:17:00 DEBUG } 22:17:00 DEBUG 22:17:00 DEBUG ] 22:17:00 DEBUG 22:17:00 DEBUG ] 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG }, 22:17:00 DEBUG 22:17:00 DEBUG }, 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG } 22:17:00 DEBUG [...] 22:17:00 DEBUG 22:17:00 DEBUG } 22:17:00 DEBUG [...] 22:17:00 DEBUG } 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG [...] 22:17:00 DEBUG "value": [ 22:17:00 DEBUG 22:17:00 DEBUG "value": [ 22:17:00 DEBUG [...] 22:17:00 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:17:00 DEBUG [...] 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG ] 22:17:00 DEBUG 22:17:00 DEBUG } 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG [...] 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG [...] 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG }, 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG [...] 22:17:00 DEBUG } 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG [...] 22:17:00 DEBUG [...] 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG "value": [ 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG "instance": "10.142.0.43:9100" 22:17:00 DEBUG } 22:17:00 DEBUG "pod": "ds-cts-2" 22:17:00 DEBUG "pod": "ds-idrepo-2" 22:17:00 DEBUG }, 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG "pod": "ds-idrepo-2" 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG 1713647820, 22:17:00 DEBUG 22:17:00 DEBUG }, 22:17:00 DEBUG 22:17:00 DEBUG }, 22:17:00 DEBUG "value": [ 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG "pod": "am-767f6b67b5-vbltz" 22:17:00 DEBUG }, 22:17:00 DEBUG 22:17:00 DEBUG [...] 22:17:00 DEBUG "pod": "ds-idrepo-2" 22:17:00 DEBUG "0" 22:17:00 DEBUG "pod": "am-767f6b67b5-vbltz" 22:17:00 DEBUG "value": [ 22:17:00 DEBUG 22:17:00 DEBUG 1713647820, 22:17:00 DEBUG 22:17:00 DEBUG "value": [ 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG "instance": "10.142.0.43:9100" 22:17:00 DEBUG 22:17:00 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:17:00 DEBUG 22:17:00 DEBUG }, 22:17:00 DEBUG "value": [ 22:17:00 DEBUG }, 22:17:00 DEBUG 1713647820, 22:17:00 DEBUG 22:17:00 DEBUG "instance": "10.142.0.43:9100" 22:17:00 DEBUG "29.466666666666665" 22:17:00 DEBUG 1713647820, 22:17:00 DEBUG }, 22:17:00 DEBUG "value": [ 22:17:00 DEBUG }, 22:17:00 DEBUG "value": [ 22:17:00 DEBUG 1713647820, 22:17:00 DEBUG 22:17:00 DEBUG [...] 22:17:00 DEBUG 1713647820, 22:17:00 DEBUG "value": [ 22:17:00 DEBUG "0.43899999999998157" 22:17:00 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:17:00 DEBUG }, 22:17:00 DEBUG "value": [ 22:17:00 DEBUG 1713647820, 22:17:00 DEBUG "13365055488" 22:17:00 DEBUG 1713647820, 22:17:00 DEBUG "1995.6333333333334" 22:17:00 DEBUG }, 22:17:00 DEBUG "value": [ 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG 22:17:00 DEBUG 1713647820, 22:17:00 DEBUG ] 22:17:00 DEBUG "value": [ 22:17:00 DEBUG ] 22:17:00 DEBUG } 22:17:00 DEBUG ] 22:17:00 DEBUG ] 22:17:00 DEBUG ] 22:17:00 DEBUG }, 22:17:00 DEBUG 1713647820, 22:17:00 DEBUG 22:17:00 DEBUG "pod": "ds-cts-2" 22:17:00 DEBUG ] 22:17:00 DEBUG "0" 22:17:00 DEBUG "401579.5" 22:17:00 DEBUG } 22:17:00 DEBUG 1713647820, 22:17:00 DEBUG "0" 22:17:00 DEBUG } 22:17:00 DEBUG "4113045070848" 22:17:00 DEBUG ] 22:17:00 DEBUG } 22:17:00 DEBUG "value": [ 22:17:00 DEBUG "70020667.06666666" 22:17:00 DEBUG "pod": "am-767f6b67b5-vbltz" 22:17:00 DEBUG }, 22:17:00 DEBUG } 22:17:00 DEBUG ] 22:17:00 DEBUG ] 22:17:00 DEBUG ] 22:17:00 DEBUG "0" 22:17:00 DEBUG ] 22:17:00 DEBUG } 22:17:00 DEBUG } 22:17:00 DEBUG ] 22:17:00 DEBUG ] 22:17:00 DEBUG 1713647820, 22:17:00 DEBUG ] 22:17:00 DEBUG }, 22:17:00 DEBUG "value": [ 22:17:00 DEBUG ] 22:17:00 DEBUG } 22:17:00 DEBUG } 22:17:00 DEBUG } 22:17:00 DEBUG ] 22:17:00 DEBUG } 22:17:00 DEBUG } 22:17:00 DEBUG } 22:17:00 DEBUG } 22:17:00 DEBUG "0" 22:17:00 DEBUG } 22:17:00 DEBUG "value": [ 22:17:00 DEBUG 1713647820, 22:17:00 DEBUG } 22:17:00 DEBUG ] 22:17:00 DEBUG ] 22:17:00 DEBUG } 22:17:00 DEBUG } 22:17:00 DEBUG ] 22:17:00 DEBUG ] 22:17:00 DEBUG } 22:17:00 DEBUG } 22:17:00 DEBUG ] 22:17:00 DEBUG ] 22:17:00 DEBUG 1713647820, 22:17:00 DEBUG "0" 22:17:00 DEBUG } 22:17:00 DEBUG } 22:17:00 DEBUG } 22:17:00 DEBUG } 22:17:00 DEBUG } 22:17:00 DEBUG ] 22:17:00 DEBUG } 22:17:00 DEBUG } 22:17:00 DEBUG "0" 22:17:00 DEBUG ] 22:17:00 DEBUG } 22:17:00 DEBUG } 22:17:00 DEBUG } 22:17:00 DEBUG } 22:17:00 DEBUG } 22:17:00 DEBUG ] 22:17:00 DEBUG } 22:17:00 DEBUG ] 22:17:00 DEBUG } 22:17:00 DEBUG ] 22:17:00 DEBUG } 22:17:00 DEBUG } 22:17:00 DEBUG ] 22:17:00 DEBUG } 22:17:00 DEBUG } 22:17:00 DEBUG } 22:17:00 DEBUG } 22:17:00 DEBUG } 22:17:00 DEBUG } 22:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713647875" 22:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647875" 22:17:55 ERROR [http_cmd]: ERROR 22:17:55 ERROR --- status code --- 22:17:55 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:17:55 ERROR --- http response --- 22:17:55 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:17:55 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:17:55 INFO [http_cmd]: http status code OK 22:17:55 DEBUG --- status code --- 22:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:55 DEBUG --- http response --- 22:17:55 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713647875" 22:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647875" 22:17:55 INFO [http_cmd]: http status code OK 22:17:55 DEBUG --- status code --- 22:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:55 INFO [http_cmd]: http status code OK 22:17:55 DEBUG --- http response --- 22:17:55 DEBUG --- status code --- 22:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:55 DEBUG ----- output ----- 22:17:55 DEBUG --- http response --- 22:17:55 DEBUG { 22:17:55 DEBUG "status": "success", 22:17:55 DEBUG "data": { 22:17:55 DEBUG "resultType": "vector", 22:17:55 DEBUG "result": [ 22:17:55 DEBUG { 22:17:55 DEBUG "metric": { 22:17:55 DEBUG "pod": "ds-idrepo-0" 22:17:55 DEBUG }, 22:17:55 DEBUG "value": [ 22:17:55 DEBUG 22:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:17:55 DEBUG 22:17:55 DEBUG ----- output ----- 22:17:55 DEBUG 22:17:55 DEBUG { 22:17:55 DEBUG [...] 22:17:55 DEBUG "status": "success", 22:17:55 DEBUG 22:17:55 DEBUG "data": { 22:17:55 DEBUG 22:17:55 DEBUG "resultType": "vector", 22:17:55 DEBUG 22:17:55 DEBUG "result": [ 22:17:55 DEBUG "pod": "ds-cts-2" 22:17:55 DEBUG { 22:17:55 DEBUG }, 22:17:55 DEBUG "metric": { 22:17:55 DEBUG "value": [ 22:17:55 DEBUG "pod": "ds-idrepo-1" 22:17:55 DEBUG 1713647875, 22:17:55 DEBUG }, 22:17:55 DEBUG "0" 22:17:55 DEBUG "value": [ 22:17:55 DEBUG ] 22:17:55 DEBUG 22:17:55 DEBUG } 22:17:55 DEBUG 22:17:55 DEBUG ] 22:17:55 DEBUG 22:17:55 DEBUG } 22:17:55 DEBUG [...] 22:17:55 DEBUG } 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG "pod": "overseer-0-967d957f7-zk66v" 22:17:55 DEBUG }, 22:17:55 DEBUG "value": [ 22:17:55 DEBUG 1713647875, 22:17:55 DEBUG "30187264.311814856" 22:17:55 DEBUG ] 22:17:55 DEBUG } 22:17:55 DEBUG ] 22:17:55 DEBUG } 22:17:55 DEBUG } 22:17:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647877" 22:17:57 INFO [http_cmd]: http status code OK 22:17:57 DEBUG --- status code --- 22:17:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:57 DEBUG --- http response --- 22:17:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:17:57 DEBUG ----- output ----- 22:17:57 DEBUG { 22:17:57 DEBUG "status": "success", 22:17:57 DEBUG "data": { 22:17:57 DEBUG "resultType": "vector", 22:17:57 DEBUG "result": [ 22:17:57 DEBUG { 22:17:57 DEBUG "metric": { 22:17:57 DEBUG "pod": "am-767f6b67b5-9zw22" 22:17:57 DEBUG }, 22:17:57 DEBUG "value": [ 22:17:57 DEBUG 22:17:57 DEBUG 22:17:57 DEBUG 22:17:57 DEBUG [...] 22:17:57 DEBUG 22:17:57 DEBUG 22:17:57 DEBUG 22:17:57 DEBUG "pod": "am-767f6b67b5-vbltz" 22:17:57 DEBUG }, 22:17:57 DEBUG "value": [ 22:17:57 DEBUG 1713647877, 22:17:57 DEBUG "2177.233333333333" 22:17:57 DEBUG ] 22:17:57 DEBUG } 22:17:57 DEBUG ] 22:17:57 DEBUG } 22:17:57 DEBUG } 22:17:57 INFO 22:17:57 INFO [loop_until]: kubectl --namespace=xlou top pods 22:17:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:17:57 INFO [loop_until]: OK (rc = 0) 22:17:57 DEBUG --- stdout --- 22:17:57 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 14912m 8811Mi am-767f6b67b5-qcqz2 14348m 8972Mi am-767f6b67b5-vbltz 15881m 8939Mi ds-cts-0 18989m 12315Mi ds-cts-1 19135m 12281Mi ds-cts-2 20305m 12468Mi ds-idrepo-0 6066m 3927Mi ds-idrepo-1 6063m 3966Mi ds-idrepo-2 6039m 3880Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-7dd48676f6-tdxl9 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-967d957f7-zk66v 14928m 15137Mi 22:17:57 DEBUG --- stderr --- 22:17:57 DEBUG 22:17:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647879" 22:17:59 INFO [http_cmd]: http status code OK 22:17:59 DEBUG --- status code --- 22:17:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:59 DEBUG --- http response --- 22:17:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:17:59 DEBUG ----- output ----- 22:17:59 DEBUG { 22:17:59 DEBUG "status": "success", 22:17:59 DEBUG "data": { 22:17:59 DEBUG "resultType": "vector", 22:17:59 DEBUG "result": [ 22:17:59 DEBUG { 22:17:59 DEBUG "metric": { 22:17:59 DEBUG "instance": "10.142.0.107:9100" 22:17:59 DEBUG }, 22:17:59 DEBUG "value": [ 22:17:59 DEBUG 22:17:59 DEBUG 22:17:59 DEBUG 22:17:59 DEBUG [...] 22:17:59 DEBUG 22:17:59 DEBUG 22:17:59 DEBUG 22:17:59 DEBUG "instance": "10.142.0.43:9100" 22:17:59 DEBUG }, 22:17:59 DEBUG "value": [ 22:17:59 DEBUG 1713647879, 22:17:59 DEBUG "69386741.56666666" 22:17:59 DEBUG ] 22:17:59 DEBUG } 22:17:59 DEBUG ] 22:17:59 DEBUG } 22:17:59 DEBUG } 22:18:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647880" 22:18:00 INFO [http_cmd]: http status code OK 22:18:00 DEBUG --- status code --- 22:18:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:00 DEBUG --- http response --- 22:18:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:18:00 DEBUG ----- output ----- 22:18:00 DEBUG { 22:18:00 DEBUG "status": "success", 22:18:00 DEBUG "data": { 22:18:00 DEBUG "resultType": "vector", 22:18:00 DEBUG "result": [ 22:18:00 DEBUG { 22:18:00 DEBUG "metric": { 22:18:00 DEBUG "pod": "am-767f6b67b5-9zw22" 22:18:00 DEBUG }, 22:18:00 DEBUG "value": [ 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG [...] 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG "pod": "am-767f6b67b5-vbltz" 22:18:00 DEBUG }, 22:18:00 DEBUG "value": [ 22:18:00 DEBUG 1713647880, 22:18:00 DEBUG "2175.5333333333333" 22:18:00 DEBUG ] 22:18:00 DEBUG } 22:18:00 DEBUG ] 22:18:00 DEBUG } 22:18:00 DEBUG } 22:18:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713647880" 22:18:00 INFO [http_cmd]: http status code OK 22:18:00 DEBUG --- status code --- 22:18:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:00 DEBUG --- http response --- 22:18:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:18:00 DEBUG ----- output ----- 22:18:00 DEBUG { 22:18:00 DEBUG "status": "success", 22:18:00 DEBUG "data": { 22:18:00 DEBUG "resultType": "vector", 22:18:00 DEBUG "result": [ 22:18:00 DEBUG { 22:18:00 DEBUG "metric": { 22:18:00 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:18:00 DEBUG }, 22:18:00 DEBUG "value": [ 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG [...] 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:18:00 DEBUG }, 22:18:00 DEBUG "value": [ 22:18:00 DEBUG 1713647880, 22:18:00 DEBUG "14.915677654089048" 22:18:00 DEBUG ] 22:18:00 DEBUG } 22:18:00 DEBUG ] 22:18:00 DEBUG } 22:18:00 DEBUG } 22:18:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713647880" 22:18:00 INFO 22:18:00 INFO [loop_until]: kubectl --namespace=xlou top node 22:18:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:18:00 INFO [http_cmd]: http status code OK 22:18:00 DEBUG --- status code --- 22:18:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:00 DEBUG --- http response --- 22:18:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:18:00 DEBUG ----- output ----- 22:18:00 DEBUG { 22:18:00 DEBUG "status": "success", 22:18:00 DEBUG "data": { 22:18:00 DEBUG "resultType": "vector", 22:18:00 DEBUG "result": [ 22:18:00 DEBUG { 22:18:00 DEBUG "metric": { 22:18:00 DEBUG "pod": "am-767f6b67b5-9zw22" 22:18:00 DEBUG }, 22:18:00 DEBUG "value": [ 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG [...] 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG "pod": "am-767f6b67b5-vbltz" 22:18:00 DEBUG }, 22:18:00 DEBUG "value": [ 22:18:00 DEBUG 1713647880, 22:18:00 DEBUG "96.245" 22:18:00 DEBUG ] 22:18:00 DEBUG } 22:18:00 DEBUG ] 22:18:00 DEBUG } 22:18:00 DEBUG } 22:18:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647880" 22:18:00 INFO [http_cmd]: http status code OK 22:18:00 DEBUG --- status code --- 22:18:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:00 DEBUG --- http response --- 22:18:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:18:00 DEBUG ----- output ----- 22:18:00 DEBUG { 22:18:00 DEBUG "status": "success", 22:18:00 DEBUG "data": { 22:18:00 DEBUG "resultType": "vector", 22:18:00 DEBUG "result": [ 22:18:00 DEBUG { 22:18:00 DEBUG "metric": { 22:18:00 DEBUG "pod": "am-767f6b67b5-9zw22" 22:18:00 DEBUG }, 22:18:00 DEBUG "value": [ 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG [...] 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG "pod": "am-767f6b67b5-vbltz" 22:18:00 DEBUG }, 22:18:00 DEBUG "value": [ 22:18:00 DEBUG 1713647880, 22:18:00 DEBUG "2476.8" 22:18:00 DEBUG ] 22:18:00 DEBUG } 22:18:00 DEBUG ] 22:18:00 DEBUG } 22:18:00 DEBUG } 22:18:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647880" 22:18:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647880" 22:18:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647880" 22:18:00 INFO [http_cmd]: http status code OK 22:18:00 DEBUG --- status code --- 22:18:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:00 DEBUG --- http response --- 22:18:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:18:00 DEBUG ----- output ----- 22:18:00 DEBUG { 22:18:00 DEBUG "status": "success", 22:18:00 DEBUG "data": { 22:18:00 DEBUG "resultType": "vector", 22:18:00 DEBUG "result": [ 22:18:00 DEBUG { 22:18:00 DEBUG "metric": { 22:18:00 DEBUG "instance": "10.142.0.107:9100" 22:18:00 DEBUG }, 22:18:00 DEBUG "value": [ 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG [...] 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG "instance": "10.142.0.43:9100" 22:18:00 DEBUG }, 22:18:00 DEBUG "value": [ 22:18:00 DEBUG 1713647880, 22:18:00 DEBUG "0.07399999999998955" 22:18:00 DEBUG ] 22:18:00 DEBUG } 22:18:00 DEBUG ] 22:18:00 DEBUG } 22:18:00 INFO [http_cmd]: http status code OK 22:18:00 DEBUG } 22:18:00 INFO [http_cmd]: http status code OK 22:18:00 DEBUG --- status code --- 22:18:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713647880" 22:18:00 DEBUG --- status code --- 22:18:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:00 DEBUG --- http response --- 22:18:00 DEBUG --- http response --- 22:18:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647880" 22:18:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:18:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647880" 22:18:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647880" 22:18:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:18:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647880" 22:18:00 DEBUG ----- output ----- 22:18:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647880" 22:18:00 DEBUG ----- output ----- 22:18:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647880" 22:18:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647880" 22:18:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647880" 22:18:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647880" 22:18:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647880" 22:18:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647880" 22:18:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647880" 22:18:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647880" 22:18:00 DEBUG { 22:18:00 DEBUG { 22:18:00 INFO [http_cmd]: http status code OK 22:18:00 INFO [http_cmd]: http status code OK 22:18:00 INFO [http_cmd]: http status code OK 22:18:00 INFO [http_cmd]: http status code OK 22:18:00 INFO [http_cmd]: http status code OK 22:18:00 INFO [http_cmd]: http status code OK 22:18:00 INFO [http_cmd]: http status code OK 22:18:00 DEBUG "status": "success", 22:18:00 INFO [http_cmd]: http status code OK 22:18:00 INFO [http_cmd]: http status code OK 22:18:00 DEBUG "status": "success", 22:18:00 INFO [http_cmd]: http status code OK 22:18:00 DEBUG --- status code --- 22:18:00 INFO [loop_until]: OK (rc = 0) 22:18:00 DEBUG --- status code --- 22:18:00 INFO [http_cmd]: http status code OK 22:18:00 DEBUG --- status code --- 22:18:00 INFO [http_cmd]: http status code OK 22:18:00 DEBUG --- status code --- 22:18:00 DEBUG --- status code --- 22:18:00 DEBUG --- status code --- 22:18:00 DEBUG --- status code --- 22:18:00 DEBUG "data": { 22:18:00 DEBUG --- status code --- 22:18:00 DEBUG --- status code --- 22:18:00 DEBUG "data": { 22:18:00 DEBUG --- status code --- 22:18:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:00 INFO [http_cmd]: http status code OK 22:18:00 INFO [http_cmd]: http status code OK 22:18:00 DEBUG --- stdout --- 22:18:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:00 DEBUG --- status code --- 22:18:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:00 DEBUG --- status code --- 22:18:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:00 DEBUG "resultType": "vector", 22:18:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:00 DEBUG "resultType": "vector", 22:18:00 DEBUG "result": [ 22:18:00 DEBUG --- http response --- 22:18:00 DEBUG --- status code --- 22:18:00 DEBUG --- status code --- 22:18:00 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6228Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 6225m 20% 7698Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 14260m 47% 11557Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 3368Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 19304m 64% 15513Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 150m 0% 5190Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 13858m 46% 11888Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 13540m 45% 3909Mi 3% gke-xlou-cdm-ds-0f601751-hmm7 16225m 54% 11202Mi 10% gke-xlou-cdm-ds-0f601751-j488 6391m 21% 7295Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19062m 63% 17362Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6500m 21% 7951Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 19876m 66% 15173Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 15063m 94% 21811Mi 37% 22:18:00 DEBUG --- http response --- 22:18:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:00 DEBUG --- http response --- 22:18:00 DEBUG --- http response --- 22:18:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:00 DEBUG --- http response --- 22:18:00 DEBUG --- http response --- 22:18:00 DEBUG --- http response --- 22:18:00 DEBUG "result": [ 22:18:00 DEBUG --- http response --- 22:18:00 DEBUG --- http response --- 22:18:00 DEBUG { 22:18:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:18:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:00 DEBUG --- stderr --- 22:18:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:18:00 DEBUG --- http response --- 22:18:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:18:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:18:00 DEBUG --- http response --- 22:18:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:18:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:18:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:18:00 DEBUG { 22:18:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:18:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:18:00 DEBUG "metric": { 22:18:00 DEBUG --- http response --- 22:18:00 DEBUG ----- output ----- 22:18:00 DEBUG --- http response --- 22:18:00 DEBUG --- http response --- 22:18:00 DEBUG 22:18:00 DEBUG ----- output ----- 22:18:00 DEBUG ----- output ----- 22:18:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:18:00 DEBUG ----- output ----- 22:18:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:18:00 DEBUG ----- output ----- 22:18:00 DEBUG ----- output ----- 22:18:00 DEBUG ----- output ----- 22:18:00 DEBUG "metric": { 22:18:00 DEBUG ----- output ----- 22:18:00 DEBUG ----- output ----- 22:18:00 DEBUG "pod": "ds-idrepo-0" 22:18:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:18:00 DEBUG ----- output ----- 22:18:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:18:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:18:00 DEBUG { 22:18:00 DEBUG { 22:18:00 DEBUG ----- output ----- 22:18:00 DEBUG { 22:18:00 DEBUG ----- output ----- 22:18:00 DEBUG { 22:18:00 DEBUG { 22:18:00 DEBUG { 22:18:00 DEBUG "pod": "ds-cts-0" 22:18:00 DEBUG { 22:18:00 DEBUG "status": "success", 22:18:00 DEBUG }, 22:18:00 DEBUG { 22:18:00 DEBUG { 22:18:00 DEBUG ----- output ----- 22:18:00 DEBUG ----- output ----- 22:18:00 DEBUG "status": "success", 22:18:00 DEBUG "data": { 22:18:00 DEBUG "status": "success", 22:18:00 DEBUG { 22:18:00 DEBUG { 22:18:00 DEBUG "status": "success", 22:18:00 DEBUG "status": "success", 22:18:00 DEBUG "status": "success", 22:18:00 DEBUG }, 22:18:00 DEBUG { 22:18:00 DEBUG "data": { 22:18:00 DEBUG "value": [ 22:18:00 DEBUG "status": "success", 22:18:00 DEBUG "data": { 22:18:00 DEBUG { 22:18:00 DEBUG { 22:18:00 DEBUG "status": "success", 22:18:00 DEBUG "resultType": "vector", 22:18:00 DEBUG "data": { 22:18:00 DEBUG "status": "success", 22:18:00 DEBUG "status": "success", 22:18:00 DEBUG "data": { 22:18:00 DEBUG "data": { 22:18:00 DEBUG "data": { 22:18:00 DEBUG "value": [ 22:18:00 DEBUG "status": "success", 22:18:00 DEBUG "resultType": "vector", 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG "status": "success", 22:18:00 DEBUG "status": "success", 22:18:00 DEBUG "status": "success", 22:18:00 DEBUG "data": { 22:18:00 DEBUG "result": [ 22:18:00 DEBUG "resultType": "vector", 22:18:00 DEBUG "data": { 22:18:00 DEBUG "data": { 22:18:00 DEBUG "resultType": "vector", 22:18:00 DEBUG "resultType": "vector", 22:18:00 DEBUG "resultType": "vector", 22:18:00 DEBUG 22:18:00 DEBUG "data": { 22:18:00 DEBUG "result": [ 22:18:00 DEBUG 22:18:00 DEBUG "resultType": "vector", 22:18:00 DEBUG "data": { 22:18:00 DEBUG "data": { 22:18:00 DEBUG "data": { 22:18:00 DEBUG "resultType": "vector", 22:18:00 DEBUG { 22:18:00 DEBUG "result": [ 22:18:00 DEBUG "resultType": "vector", 22:18:00 DEBUG "resultType": "vector", 22:18:00 DEBUG "result": [ 22:18:00 DEBUG "result": [ 22:18:00 DEBUG "result": [ 22:18:00 DEBUG 22:18:00 DEBUG "resultType": "vector", 22:18:00 DEBUG "result": [ 22:18:00 DEBUG [...] 22:18:00 DEBUG "result": [ 22:18:00 DEBUG "resultType": "vector", 22:18:00 DEBUG "resultType": "vector", 22:18:00 DEBUG "resultType": "vector", 22:18:00 DEBUG "result": [ 22:18:00 DEBUG "metric": { 22:18:00 DEBUG { 22:18:00 DEBUG "result": [ 22:18:00 DEBUG "result": [ 22:18:00 DEBUG { 22:18:00 DEBUG { 22:18:00 DEBUG { 22:18:00 DEBUG 22:18:00 DEBUG { 22:18:00 DEBUG { 22:18:00 DEBUG 22:18:00 DEBUG { 22:18:00 DEBUG "result": [ 22:18:00 DEBUG "result": [ 22:18:00 DEBUG "result": [ 22:18:00 DEBUG { 22:18:00 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:18:00 DEBUG "metric": { 22:18:00 DEBUG { 22:18:00 DEBUG { 22:18:00 DEBUG "metric": { 22:18:00 DEBUG "metric": { 22:18:00 DEBUG "pod": "am-767f6b67b5-9zw22" 22:18:00 DEBUG [...] 22:18:00 DEBUG "metric": { 22:18:00 DEBUG "metric": { 22:18:00 DEBUG 22:18:00 DEBUG "metric": { 22:18:00 DEBUG { 22:18:00 DEBUG { 22:18:00 DEBUG { 22:18:00 DEBUG "metric": { 22:18:00 DEBUG }, 22:18:00 DEBUG "pod": "ds-idrepo-0" 22:18:00 DEBUG "metric": { 22:18:00 DEBUG "metric": { 22:18:00 DEBUG "pod": "ds-idrepo-0" 22:18:00 DEBUG "metric": { 22:18:00 DEBUG }, 22:18:00 DEBUG 22:18:00 DEBUG "pod": "am-767f6b67b5-9zw22" 22:18:00 DEBUG "pod": "ds-idrepo-0" 22:18:00 DEBUG 22:18:00 DEBUG "instance": "10.142.0.107:9100" 22:18:00 DEBUG "metric": { 22:18:00 DEBUG "metric": { 22:18:00 DEBUG "metric": { 22:18:00 DEBUG "pod": "ds-cts-0" 22:18:00 DEBUG "value": [ 22:18:00 DEBUG }, 22:18:00 DEBUG "pod": "ds-idrepo-0" 22:18:00 DEBUG "instance": "10.142.0.107:9100" 22:18:00 DEBUG }, 22:18:00 DEBUG "pod": "ds-idrepo-0" 22:18:00 DEBUG }, 22:18:00 DEBUG 22:18:00 DEBUG }, 22:18:00 DEBUG }, 22:18:00 DEBUG "pod": "overseer-0-967d957f7-zk66v" 22:18:00 DEBUG }, 22:18:00 DEBUG "instance": "10.142.0.107:9100" 22:18:00 DEBUG "pod": "am-767f6b67b5-9zw22" 22:18:00 DEBUG "pod": "ds-cts-0" 22:18:00 DEBUG }, 22:18:00 DEBUG 22:18:00 DEBUG "value": [ 22:18:00 DEBUG }, 22:18:00 DEBUG "value": [ 22:18:00 DEBUG "value": [ 22:18:00 DEBUG "value": [ 22:18:00 DEBUG }, 22:18:00 DEBUG 22:18:00 DEBUG "value": [ 22:18:00 DEBUG "value": [ 22:18:00 DEBUG }, 22:18:00 DEBUG "value": [ 22:18:00 DEBUG }, 22:18:00 DEBUG }, 22:18:00 DEBUG }, 22:18:00 DEBUG "value": [ 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG "value": [ 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG "value": [ 22:18:00 DEBUG "pod": "ds-cts-2" 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG "value": [ 22:18:00 DEBUG 1713647880, 22:18:00 DEBUG "value": [ 22:18:00 DEBUG "value": [ 22:18:00 DEBUG "value": [ 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG }, 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG "9.814171122280255" 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG [...] 22:18:00 DEBUG 22:18:00 DEBUG [...] 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG "value": [ 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG ] 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG [...] 22:18:00 DEBUG [...] 22:18:00 DEBUG [...] 22:18:00 DEBUG 22:18:00 DEBUG 1713647880, 22:18:00 DEBUG [...] 22:18:00 DEBUG [...] 22:18:00 DEBUG 22:18:00 DEBUG } 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG [...] 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG [...] 22:18:00 DEBUG "29.5216" 22:18:00 DEBUG ] 22:18:00 DEBUG 22:18:00 DEBUG [...] 22:18:00 DEBUG ] 22:18:00 DEBUG [...] 22:18:00 DEBUG [...] 22:18:00 DEBUG [...] 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG [...] 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG } 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG } 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:18:00 DEBUG "pod": "overseer-0-967d957f7-zk66v" 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG ] 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG } 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG }, 22:18:00 DEBUG }, 22:18:00 DEBUG 22:18:00 DEBUG "instance": "10.142.0.43:9100" 22:18:00 DEBUG "pod": "ds-idrepo-2" 22:18:00 DEBUG "pod": "am-767f6b67b5-vbltz" 22:18:00 DEBUG 22:18:00 DEBUG } 22:18:00 DEBUG 22:18:00 DEBUG "pod": "ds-idrepo-2" 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG 22:18:00 DEBUG "pod": "ds-cts-2" 22:18:00 DEBUG "value": [ 22:18:00 DEBUG "value": [ 22:18:00 DEBUG 22:18:00 DEBUG }, 22:18:00 DEBUG "value": [ 22:18:00 DEBUG }, 22:18:00 DEBUG "pod": "am-767f6b67b5-vbltz" 22:18:00 DEBUG } 22:18:00 DEBUG "pod": "overseer-0-967d957f7-zk66v" 22:18:00 DEBUG }, 22:18:00 DEBUG "instance": "10.142.0.43:9100" 22:18:00 DEBUG }, 22:18:00 DEBUG "pod": "am-767f6b67b5-vbltz" 22:18:00 DEBUG "pod": "ds-cts-2" 22:18:00 DEBUG }, 22:18:00 DEBUG 1713647880, 22:18:00 DEBUG 1713647880, 22:18:00 DEBUG "pod": "ds-idrepo-2" 22:18:00 DEBUG 1713647880, 22:18:00 DEBUG }, 22:18:00 DEBUG "value": [ 22:18:00 DEBUG }, 22:18:00 DEBUG }, 22:18:00 DEBUG "value": [ 22:18:00 DEBUG "value": [ 22:18:00 DEBUG "instance": "10.142.0.43:9100" 22:18:00 DEBUG }, 22:18:00 DEBUG }, 22:18:00 DEBUG "value": [ 22:18:00 DEBUG "14782562304" 22:18:00 DEBUG "0" 22:18:00 DEBUG }, 22:18:00 DEBUG "4117606321152" 22:18:00 DEBUG "value": [ 22:18:00 DEBUG 1713647880, 22:18:00 DEBUG "value": [ 22:18:00 DEBUG "value": [ 22:18:00 DEBUG 1713647880, 22:18:00 DEBUG 1713647880, 22:18:00 DEBUG }, 22:18:00 DEBUG "value": [ 22:18:00 DEBUG "value": [ 22:18:00 DEBUG 1713647880, 22:18:00 DEBUG ] 22:18:00 DEBUG ] 22:18:00 DEBUG "value": [ 22:18:00 DEBUG ] 22:18:00 DEBUG 1713647880, 22:18:00 DEBUG "2175.9333333333334" 22:18:00 DEBUG 1713647880, 22:18:00 DEBUG 1713647880, 22:18:00 DEBUG "0" 22:18:00 DEBUG "69391958.36666666" 22:18:00 DEBUG "value": [ 22:18:00 DEBUG 1713647880, 22:18:00 DEBUG 1713647880, 22:18:00 DEBUG "31.066666666666666" 22:18:00 DEBUG } 22:18:00 DEBUG } 22:18:00 DEBUG 1713647880, 22:18:00 DEBUG } 22:18:00 DEBUG "0" 22:18:00 DEBUG ] 22:18:00 DEBUG "371867" 22:18:00 DEBUG "0" 22:18:00 DEBUG ] 22:18:00 DEBUG ] 22:18:00 DEBUG 1713647880, 22:18:00 DEBUG "0" 22:18:00 DEBUG "0" 22:18:00 DEBUG ] 22:18:00 DEBUG ] 22:18:00 DEBUG ] 22:18:00 DEBUG "0" 22:18:00 DEBUG ] 22:18:00 DEBUG ] 22:18:00 DEBUG } 22:18:00 DEBUG ] 22:18:00 DEBUG ] 22:18:00 DEBUG } 22:18:00 DEBUG } 22:18:00 DEBUG "0.4447666666667042" 22:18:00 DEBUG ] 22:18:00 DEBUG ] 22:18:00 DEBUG } 22:18:00 DEBUG } 22:18:00 DEBUG } 22:18:00 DEBUG ] 22:18:00 DEBUG } 22:18:00 DEBUG } 22:18:00 DEBUG ] 22:18:00 DEBUG } 22:18:00 DEBUG } 22:18:00 DEBUG ] 22:18:00 DEBUG ] 22:18:00 DEBUG ] 22:18:00 DEBUG } 22:18:00 DEBUG } 22:18:00 DEBUG ] 22:18:00 DEBUG } 22:18:00 DEBUG } 22:18:00 DEBUG } 22:18:00 DEBUG } 22:18:00 DEBUG ] 22:18:00 DEBUG } 22:18:00 DEBUG ] 22:18:00 DEBUG ] 22:18:00 DEBUG } 22:18:00 DEBUG } 22:18:00 DEBUG } 22:18:00 DEBUG ] 22:18:00 DEBUG ] 22:18:00 DEBUG } 22:18:00 DEBUG } 22:18:00 DEBUG ] 22:18:00 DEBUG } 22:18:00 DEBUG } 22:18:00 DEBUG } 22:18:00 DEBUG } 22:18:00 DEBUG } 22:18:00 DEBUG } 22:18:00 DEBUG ] 22:18:00 DEBUG } 22:18:00 DEBUG } 22:18:00 DEBUG } 22:18:00 DEBUG } 22:18:00 DEBUG } 22:18:00 DEBUG } 22:18:00 DEBUG } 22:18:00 DEBUG } 22:18:00 DEBUG } 22:18:00 DEBUG } 22:18:00 DEBUG } 22:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647935" 22:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713647935" 22:18:55 INFO [http_cmd]: http status code OK 22:18:55 DEBUG --- status code --- 22:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:55 DEBUG --- http response --- 22:18:55 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:18:55 ERROR [http_cmd]: ERROR 22:18:55 ERROR --- status code --- 22:18:55 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:18:55 ERROR --- http response --- 22:18:55 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:18:55 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713647935" 22:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647935" 22:18:55 INFO [http_cmd]: http status code OK 22:18:55 INFO [http_cmd]: http status code OK 22:18:55 DEBUG --- status code --- 22:18:55 DEBUG --- status code --- 22:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:55 DEBUG --- http response --- 22:18:55 DEBUG --- http response --- 22:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:18:55 DEBUG ----- output ----- 22:18:55 DEBUG ----- output ----- 22:18:55 DEBUG { 22:18:55 DEBUG { 22:18:55 DEBUG "status": "success", 22:18:55 DEBUG "status": "success", 22:18:55 DEBUG "data": { 22:18:55 DEBUG "data": { 22:18:55 DEBUG "resultType": "vector", 22:18:55 DEBUG "resultType": "vector", 22:18:55 DEBUG "result": [ 22:18:55 DEBUG { 22:18:55 DEBUG "metric": { 22:18:55 DEBUG "pod": "ds-idrepo-0" 22:18:55 DEBUG }, 22:18:55 DEBUG "value": [ 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG [...] 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG "pod": "ds-cts-2" 22:18:55 DEBUG }, 22:18:55 DEBUG "value": [ 22:18:55 DEBUG 1713647935, 22:18:55 DEBUG "0" 22:18:55 DEBUG ] 22:18:55 DEBUG } 22:18:55 DEBUG ] 22:18:55 DEBUG } 22:18:55 DEBUG } 22:18:55 DEBUG "result": [ 22:18:55 DEBUG { 22:18:55 DEBUG "metric": { 22:18:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:18:55 DEBUG }, 22:18:55 DEBUG "value": [ 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG [...] 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG "pod": "overseer-0-967d957f7-zk66v" 22:18:55 DEBUG }, 22:18:55 DEBUG "value": [ 22:18:55 DEBUG 1713647935, 22:18:55 DEBUG "0" 22:18:55 DEBUG ] 22:18:55 DEBUG } 22:18:55 DEBUG ] 22:18:55 DEBUG } 22:18:55 DEBUG } 22:18:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647937" 22:18:57 INFO [http_cmd]: http status code OK 22:18:57 DEBUG --- status code --- 22:18:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:57 DEBUG --- http response --- 22:18:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:18:57 DEBUG ----- output ----- 22:18:57 DEBUG { 22:18:57 DEBUG "status": "success", 22:18:57 DEBUG "data": { 22:18:57 DEBUG "resultType": "vector", 22:18:57 DEBUG "result": [ 22:18:57 DEBUG { 22:18:57 DEBUG "metric": { 22:18:57 DEBUG "pod": "am-767f6b67b5-9zw22" 22:18:57 DEBUG }, 22:18:57 DEBUG "value": [ 22:18:57 DEBUG 22:18:57 DEBUG 22:18:57 DEBUG 22:18:57 DEBUG [...] 22:18:57 DEBUG 22:18:57 DEBUG 22:18:57 DEBUG 22:18:57 DEBUG "pod": "am-767f6b67b5-vbltz" 22:18:57 DEBUG }, 22:18:57 DEBUG "value": [ 22:18:57 DEBUG 1713647937, 22:18:57 DEBUG "1406.8" 22:18:57 DEBUG ] 22:18:57 DEBUG } 22:18:57 DEBUG ] 22:18:57 DEBUG } 22:18:57 DEBUG } 22:18:57 INFO 22:18:57 INFO [loop_until]: kubectl --namespace=xlou top pods 22:18:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:18:58 INFO [loop_until]: OK (rc = 0) 22:18:58 DEBUG --- stdout --- 22:18:58 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 14010m 8809Mi am-767f6b67b5-qcqz2 11580m 8973Mi am-767f6b67b5-vbltz 12028m 8940Mi ds-cts-0 18534m 12309Mi ds-cts-1 18006m 12286Mi ds-cts-2 20151m 12468Mi ds-idrepo-0 4244m 3926Mi ds-idrepo-1 3598m 3965Mi ds-idrepo-2 4136m 3880Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-7dd48676f6-tdxl9 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-967d957f7-zk66v 12565m 16383Mi 22:18:58 DEBUG --- stderr --- 22:18:58 DEBUG 22:18:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647939" 22:18:59 INFO [http_cmd]: http status code OK 22:18:59 DEBUG --- status code --- 22:18:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:59 DEBUG --- http response --- 22:18:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:18:59 DEBUG ----- output ----- 22:18:59 DEBUG { 22:18:59 DEBUG "status": "success", 22:18:59 DEBUG "data": { 22:18:59 DEBUG "resultType": "vector", 22:18:59 DEBUG "result": [ 22:18:59 DEBUG { 22:18:59 DEBUG "metric": { 22:18:59 DEBUG "instance": "10.142.0.107:9100" 22:18:59 DEBUG }, 22:18:59 DEBUG "value": [ 22:18:59 DEBUG 22:18:59 DEBUG 22:18:59 DEBUG 22:18:59 DEBUG [...] 22:18:59 DEBUG 22:18:59 DEBUG 22:18:59 DEBUG 22:18:59 DEBUG "instance": "10.142.0.43:9100" 22:18:59 DEBUG }, 22:18:59 DEBUG "value": [ 22:18:59 DEBUG 1713647939, 22:18:59 DEBUG "64870841.099999994" 22:18:59 DEBUG ] 22:18:59 DEBUG } 22:18:59 DEBUG ] 22:18:59 DEBUG } 22:18:59 DEBUG } 22:19:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647940" 22:19:00 INFO [http_cmd]: http status code OK 22:19:00 DEBUG --- status code --- 22:19:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:00 DEBUG --- http response --- 22:19:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:19:00 DEBUG ----- output ----- 22:19:00 DEBUG { 22:19:00 DEBUG "status": "success", 22:19:00 DEBUG "data": { 22:19:00 DEBUG "resultType": "vector", 22:19:00 DEBUG "result": [ 22:19:00 DEBUG { 22:19:00 DEBUG "metric": { 22:19:00 DEBUG "pod": "am-767f6b67b5-9zw22" 22:19:00 DEBUG }, 22:19:00 DEBUG "value": [ 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG [...] 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG "pod": "am-767f6b67b5-vbltz" 22:19:00 DEBUG }, 22:19:00 DEBUG "value": [ 22:19:00 DEBUG 1713647940, 22:19:00 DEBUG "1407.4" 22:19:00 DEBUG ] 22:19:00 DEBUG } 22:19:00 DEBUG ] 22:19:00 DEBUG } 22:19:00 DEBUG } 22:19:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713647940" 22:19:00 INFO [http_cmd]: http status code OK 22:19:00 DEBUG --- status code --- 22:19:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:00 DEBUG --- http response --- 22:19:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:19:00 DEBUG ----- output ----- 22:19:00 DEBUG { 22:19:00 DEBUG "status": "success", 22:19:00 DEBUG "data": { 22:19:00 DEBUG "resultType": "vector", 22:19:00 DEBUG "result": [ 22:19:00 DEBUG { 22:19:00 DEBUG "metric": { 22:19:00 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:19:00 DEBUG }, 22:19:00 DEBUG "value": [ 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG [...] 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:19:00 DEBUG }, 22:19:00 DEBUG "value": [ 22:19:00 DEBUG 1713647940, 22:19:00 DEBUG "14.875731372104017" 22:19:00 DEBUG ] 22:19:00 DEBUG } 22:19:00 DEBUG ] 22:19:00 DEBUG } 22:19:00 DEBUG } 22:19:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713647940" 22:19:00 INFO [http_cmd]: http status code OK 22:19:00 DEBUG --- status code --- 22:19:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:00 DEBUG --- http response --- 22:19:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:19:00 DEBUG ----- output ----- 22:19:00 DEBUG { 22:19:00 DEBUG "status": "success", 22:19:00 DEBUG "data": { 22:19:00 DEBUG "resultType": "vector", 22:19:00 DEBUG "result": [ 22:19:00 DEBUG { 22:19:00 DEBUG "metric": { 22:19:00 DEBUG "pod": "am-767f6b67b5-9zw22" 22:19:00 DEBUG }, 22:19:00 DEBUG "value": [ 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG [...] 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG "pod": "am-767f6b67b5-vbltz" 22:19:00 DEBUG }, 22:19:00 DEBUG "value": [ 22:19:00 DEBUG 1713647940, 22:19:00 DEBUG "98.5865" 22:19:00 DEBUG ] 22:19:00 DEBUG } 22:19:00 DEBUG ] 22:19:00 DEBUG } 22:19:00 DEBUG } 22:19:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647940" 22:19:00 INFO [http_cmd]: http status code OK 22:19:00 DEBUG --- status code --- 22:19:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:00 DEBUG --- http response --- 22:19:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:19:00 DEBUG ----- output ----- 22:19:00 DEBUG { 22:19:00 DEBUG "status": "success", 22:19:00 DEBUG "data": { 22:19:00 DEBUG "resultType": "vector", 22:19:00 DEBUG "result": [ 22:19:00 DEBUG { 22:19:00 DEBUG "metric": { 22:19:00 DEBUG "pod": "am-767f6b67b5-9zw22" 22:19:00 DEBUG }, 22:19:00 DEBUG "value": [ 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG [...] 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG "pod": "am-767f6b67b5-vbltz" 22:19:00 DEBUG }, 22:19:00 DEBUG "value": [ 22:19:00 DEBUG 1713647940, 22:19:00 DEBUG "2324.8999999999996" 22:19:00 DEBUG ] 22:19:00 DEBUG } 22:19:00 DEBUG ] 22:19:00 DEBUG } 22:19:00 DEBUG } 22:19:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647940" 22:19:00 INFO [http_cmd]: http status code OK 22:19:00 DEBUG --- status code --- 22:19:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:00 DEBUG --- http response --- 22:19:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:19:00 DEBUG ----- output ----- 22:19:00 DEBUG { 22:19:00 DEBUG "status": "success", 22:19:00 DEBUG "data": { 22:19:00 DEBUG "resultType": "vector", 22:19:00 DEBUG "result": [ 22:19:00 DEBUG { 22:19:00 DEBUG "metric": { 22:19:00 DEBUG "instance": "10.142.0.107:9100" 22:19:00 DEBUG }, 22:19:00 DEBUG "value": [ 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG [...] 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG "instance": "10.142.0.43:9100" 22:19:00 DEBUG }, 22:19:00 DEBUG "value": [ 22:19:00 DEBUG 1713647940, 22:19:00 DEBUG "0.09099999999999304" 22:19:00 DEBUG ] 22:19:00 DEBUG } 22:19:00 DEBUG ] 22:19:00 DEBUG } 22:19:00 DEBUG } 22:19:00 INFO 22:19:00 INFO [loop_until]: kubectl --namespace=xlou top node 22:19:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:19:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647940" 22:19:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647940" 22:19:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647940" 22:19:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713647940" 22:19:00 INFO [http_cmd]: http status code OK 22:19:00 INFO [http_cmd]: http status code OK 22:19:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647940" 22:19:00 DEBUG --- status code --- 22:19:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647940" 22:19:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647940" 22:19:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647940" 22:19:00 DEBUG --- status code --- 22:19:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647940" 22:19:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647940" 22:19:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647940" 22:19:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647940" 22:19:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647940" 22:19:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647940" 22:19:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647940" 22:19:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647940" 22:19:00 INFO [http_cmd]: http status code OK 22:19:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:00 INFO [http_cmd]: http status code OK 22:19:00 DEBUG --- http response --- 22:19:00 INFO [http_cmd]: http status code OK 22:19:00 INFO [http_cmd]: http status code OK 22:19:00 INFO [http_cmd]: http status code OK 22:19:00 DEBUG --- status code --- 22:19:00 INFO [loop_until]: OK (rc = 0) 22:19:00 INFO [http_cmd]: http status code OK 22:19:00 DEBUG --- http response --- 22:19:00 INFO [http_cmd]: http status code OK 22:19:00 INFO [http_cmd]: http status code OK 22:19:00 DEBUG --- status code --- 22:19:00 INFO [http_cmd]: http status code OK 22:19:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:19:00 INFO [http_cmd]: http status code OK 22:19:00 INFO [http_cmd]: http status code OK 22:19:00 DEBUG --- status code --- 22:19:00 INFO [http_cmd]: http status code OK 22:19:00 INFO [http_cmd]: http status code OK 22:19:00 DEBUG --- status code --- 22:19:00 DEBUG --- status code --- 22:19:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:00 DEBUG --- stdout --- 22:19:00 DEBUG --- status code --- 22:19:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:19:00 DEBUG --- status code --- 22:19:00 INFO [http_cmd]: http status code OK 22:19:00 DEBUG --- status code --- 22:19:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:00 DEBUG --- status code --- 22:19:00 DEBUG ----- output ----- 22:19:00 DEBUG --- status code --- 22:19:00 DEBUG --- status code --- 22:19:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:00 DEBUG --- status code --- 22:19:00 DEBUG --- status code --- 22:19:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:00 DEBUG --- http response --- 22:19:00 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6226Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 4622m 15% 7696Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 10167m 34% 11594Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 3357Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 18501m 61% 15518Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 152m 0% 5195Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 8144m 27% 11918Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 6869m 23% 3920Mi 3% gke-xlou-cdm-ds-0f601751-hmm7 12191m 40% 11219Mi 10% gke-xlou-cdm-ds-0f601751-j488 4403m 14% 7292Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17757m 59% 17345Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 4748m 15% 7945Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20082m 67% 15144Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 11811m 74% 22672Mi 38% 22:19:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:00 DEBUG ----- output ----- 22:19:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:00 DEBUG --- status code --- 22:19:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:00 DEBUG --- http response --- 22:19:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:00 DEBUG { 22:19:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:00 DEBUG --- http response --- 22:19:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:00 DEBUG --- http response --- 22:19:00 DEBUG --- http response --- 22:19:00 DEBUG --- stderr --- 22:19:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 22:19:00 DEBUG --- http response --- 22:19:00 DEBUG { 22:19:00 DEBUG --- http response --- 22:19:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:00 DEBUG --- http response --- 22:19:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:19:00 DEBUG --- http response --- 22:19:00 DEBUG "status": "success", 22:19:00 DEBUG --- http response --- 22:19:00 DEBUG --- http response --- 22:19:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:19:00 DEBUG --- http response --- 22:19:00 DEBUG --- http response --- 22:19:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 22:19:00 DEBUG 22:19:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:19:00 DEBUG ----- output ----- 22:19:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:19:00 DEBUG "status": "success", 22:19:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:19:00 DEBUG --- http response --- 22:19:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:19:00 DEBUG ----- output ----- 22:19:00 DEBUG "data": { 22:19:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:19:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:19:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:19:00 DEBUG ----- output ----- 22:19:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:19:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:19:00 DEBUG ----- output ----- 22:19:00 DEBUG ----- output ----- 22:19:00 DEBUG { 22:19:00 DEBUG ----- output ----- 22:19:00 DEBUG "data": { 22:19:00 DEBUG ----- output ----- 22:19:00 DEBUG ----- output ----- 22:19:00 DEBUG { 22:19:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:19:00 DEBUG "resultType": "vector", 22:19:00 DEBUG ----- output ----- 22:19:00 DEBUG ----- output ----- 22:19:00 DEBUG ----- output ----- 22:19:00 DEBUG { 22:19:00 DEBUG "status": "success", 22:19:00 DEBUG ----- output ----- 22:19:00 DEBUG { 22:19:00 DEBUG "status": "success", 22:19:00 DEBUG { 22:19:00 DEBUG { 22:19:00 DEBUG "resultType": "vector", 22:19:00 DEBUG { 22:19:00 DEBUG { 22:19:00 DEBUG "status": "success", 22:19:00 DEBUG ----- output ----- 22:19:00 DEBUG "result": [ 22:19:00 DEBUG { 22:19:00 DEBUG { 22:19:00 DEBUG { 22:19:00 DEBUG ----- output ----- 22:19:00 DEBUG "data": { 22:19:00 DEBUG { 22:19:00 DEBUG "status": "success", 22:19:00 DEBUG "data": { 22:19:00 DEBUG "status": "success", 22:19:00 DEBUG "status": "success", 22:19:00 DEBUG "data": { 22:19:00 DEBUG "status": "success", 22:19:00 DEBUG "data": { 22:19:00 DEBUG "data": { 22:19:00 DEBUG { 22:19:00 DEBUG { 22:19:00 DEBUG "status": "success", 22:19:00 DEBUG "status": "success", 22:19:00 DEBUG "status": "success", 22:19:00 DEBUG { 22:19:00 DEBUG "resultType": "vector", 22:19:00 DEBUG "status": "success", 22:19:00 DEBUG "data": { 22:19:00 DEBUG "resultType": "vector", 22:19:00 DEBUG "data": { 22:19:00 DEBUG "result": [ 22:19:00 DEBUG "resultType": "vector", 22:19:00 DEBUG "result": [ 22:19:00 DEBUG "status": "success", 22:19:00 DEBUG "resultType": "vector", 22:19:00 DEBUG "status": "success", 22:19:00 DEBUG "data": { 22:19:00 DEBUG "data": { 22:19:00 DEBUG "data": { 22:19:00 DEBUG "data": { 22:19:00 DEBUG "status": "success", 22:19:00 DEBUG "result": [ 22:19:00 DEBUG "data": { 22:19:00 DEBUG "resultType": "vector", 22:19:00 DEBUG "result": [ 22:19:00 DEBUG "resultType": "vector", 22:19:00 DEBUG { 22:19:00 DEBUG "resultType": "vector", 22:19:00 DEBUG { 22:19:00 DEBUG "data": { 22:19:00 DEBUG "result": [ 22:19:00 DEBUG "resultType": "vector", 22:19:00 DEBUG "metric": { 22:19:00 DEBUG "resultType": "vector", 22:19:00 DEBUG "resultType": "vector", 22:19:00 DEBUG "resultType": "vector", 22:19:00 DEBUG "data": { 22:19:00 DEBUG { 22:19:00 DEBUG "resultType": "vector", 22:19:00 DEBUG "result": [ 22:19:00 DEBUG { 22:19:00 DEBUG "result": [ 22:19:00 DEBUG "metric": { 22:19:00 DEBUG "result": [ 22:19:00 DEBUG "metric": { 22:19:00 DEBUG "resultType": "vector", 22:19:00 DEBUG { 22:19:00 DEBUG "result": [ 22:19:00 DEBUG "pod": "ds-cts-0" 22:19:00 DEBUG "result": [ 22:19:00 DEBUG "result": [ 22:19:00 DEBUG "result": [ 22:19:00 DEBUG { 22:19:00 DEBUG "metric": { 22:19:00 DEBUG "result": [ 22:19:00 DEBUG { 22:19:00 DEBUG "metric": { 22:19:00 DEBUG { 22:19:00 DEBUG "pod": "ds-idrepo-1" 22:19:00 DEBUG { 22:19:00 DEBUG "metric": { 22:19:00 DEBUG "result": [ 22:19:00 DEBUG "metric": { 22:19:00 DEBUG { 22:19:00 DEBUG }, 22:19:00 DEBUG { 22:19:00 DEBUG { 22:19:00 DEBUG "metric": { 22:19:00 DEBUG "resultType": "vector", 22:19:00 DEBUG "pod": "am-767f6b67b5-9zw22" 22:19:00 DEBUG { 22:19:00 DEBUG "metric": { 22:19:00 DEBUG "instance": "10.142.0.107:9100" 22:19:00 DEBUG "metric": { 22:19:00 DEBUG }, 22:19:00 DEBUG "pod": "ds-idrepo-1" 22:19:00 DEBUG "pod": "ds-cts-0" 22:19:00 DEBUG { 22:19:00 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:19:00 DEBUG "metric": { 22:19:00 DEBUG "value": [ 22:19:00 DEBUG "metric": { 22:19:00 DEBUG "metric": { 22:19:00 DEBUG "pod": "ds-idrepo-0" 22:19:00 DEBUG "result": [ 22:19:00 DEBUG }, 22:19:00 DEBUG "metric": { 22:19:00 DEBUG "instance": "10.142.0.107:9100" 22:19:00 DEBUG }, 22:19:00 DEBUG "instance": "10.142.0.107:9100" 22:19:00 DEBUG "value": [ 22:19:00 DEBUG }, 22:19:00 DEBUG }, 22:19:00 DEBUG "metric": { 22:19:00 DEBUG }, 22:19:00 DEBUG "pod": "ds-cts-0" 22:19:00 DEBUG 22:19:00 DEBUG "pod": "ds-idrepo-0" 22:19:00 DEBUG "pod": "am-767f6b67b5-9zw22" 22:19:00 DEBUG }, 22:19:00 DEBUG { 22:19:00 DEBUG "value": [ 22:19:00 DEBUG }, 22:19:00 DEBUG "pod": "ds-idrepo-1" 22:19:00 DEBUG "value": [ 22:19:00 DEBUG }, 22:19:00 DEBUG 22:19:00 DEBUG "value": [ 22:19:00 DEBUG "value": [ 22:19:00 DEBUG "pod": "am-767f6b67b5-9zw22" 22:19:00 DEBUG "value": [ 22:19:00 DEBUG }, 22:19:00 DEBUG 22:19:00 DEBUG }, 22:19:00 DEBUG }, 22:19:00 DEBUG "value": [ 22:19:00 DEBUG "metric": { 22:19:00 DEBUG 22:19:00 DEBUG "value": [ 22:19:00 DEBUG }, 22:19:00 DEBUG 22:19:00 DEBUG "value": [ 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG }, 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG "value": [ 22:19:00 DEBUG "value": [ 22:19:00 DEBUG 22:19:00 DEBUG "pod": "ds-idrepo-0" 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG "value": [ 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG "value": [ 22:19:00 DEBUG 22:19:00 DEBUG "value": [ 22:19:00 DEBUG [...] 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG }, 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG [...] 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG [...] 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG "value": [ 22:19:00 DEBUG [...] 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG [...] 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG [...] 22:19:00 DEBUG [...] 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG [...] 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG [...] 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG [...] 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG "pod": "ds-cts-2" 22:19:00 DEBUG 22:19:00 DEBUG [...] 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG [...] 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG [...] 22:19:00 DEBUG 22:19:00 DEBUG [...] 22:19:00 DEBUG }, 22:19:00 DEBUG [...] 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG "pod": "overseer-0-967d957f7-zk66v" 22:19:00 DEBUG }, 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG "value": [ 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG [...] 22:19:00 DEBUG "pod": "am-767f6b67b5-vbltz" 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG "instance": "10.142.0.43:9100" 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG "value": [ 22:19:00 DEBUG "pod": "ds-cts-2" 22:19:00 DEBUG 22:19:00 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:19:00 DEBUG 22:19:00 DEBUG 1713647940, 22:19:00 DEBUG 22:19:00 DEBUG 22:19:00 DEBUG "pod": "ds-idrepo-2" 22:19:00 DEBUG 22:19:00 DEBUG }, 22:19:00 DEBUG "instance": "10.142.0.43:9100" 22:19:00 DEBUG 22:19:00 DEBUG }, 22:19:00 DEBUG "instance": "10.142.0.43:9100" 22:19:00 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 22:19:00 DEBUG 1713647940, 22:19:00 DEBUG }, 22:19:00 DEBUG "pod": "am-767f6b67b5-vbltz" 22:19:00 DEBUG }, 22:19:00 DEBUG 22:19:00 DEBUG "28.9191" 22:19:00 DEBUG 22:19:00 DEBUG "pod": "am-767f6b67b5-vbltz" 22:19:00 DEBUG }, 22:19:00 DEBUG 22:19:00 DEBUG "value": [ 22:19:00 DEBUG }, 22:19:00 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 22:19:00 DEBUG "value": [ 22:19:00 DEBUG }, 22:19:00 DEBUG }, 22:19:00 DEBUG "0" 22:19:00 DEBUG "value": [ 22:19:00 DEBUG }, 22:19:00 DEBUG "value": [ 22:19:00 DEBUG "pod": "ds-cts-2" 22:19:00 DEBUG ] 22:19:00 DEBUG } 22:19:00 DEBUG "pod": "ds-idrepo-2" 22:19:00 DEBUG "value": [ 22:19:00 DEBUG 22:19:00 DEBUG "pod": "ds-idrepo-2" 22:19:00 DEBUG "value": [ 22:19:00 DEBUG }, 22:19:00 DEBUG 1713647940, 22:19:00 DEBUG "value": [ 22:19:00 DEBUG "value": [ 22:19:00 DEBUG ] 22:19:00 DEBUG 1713647940, 22:19:00 DEBUG "value": [ 22:19:00 DEBUG 1713647940, 22:19:00 DEBUG }, 22:19:00 DEBUG }, 22:19:00 DEBUG "value": [ 22:19:00 DEBUG }, 22:19:00 DEBUG 1713647940, 22:19:00 DEBUG }, 22:19:00 DEBUG 1713647940, 22:19:00 DEBUG 1713647940, 22:19:00 DEBUG "value": [ 22:19:00 DEBUG "4121874570240" 22:19:00 DEBUG 1713647940, 22:19:00 DEBUG "64875970.900000006" 22:19:00 DEBUG } 22:19:00 DEBUG "29.3" 22:19:00 DEBUG ] 22:19:00 DEBUG "16495452160" 22:19:00 DEBUG "value": [ 22:19:00 DEBUG 1713647940, 22:19:00 DEBUG "value": [ 22:19:00 DEBUG ] 22:19:00 DEBUG "0" 22:19:00 DEBUG "value": [ 22:19:00 DEBUG "1405.4666666666667" 22:19:00 DEBUG "0.4767666666666959" 22:19:00 DEBUG 1713647940, 22:19:00 DEBUG ] 22:19:00 DEBUG ] 22:19:00 DEBUG 1713647940, 22:19:00 DEBUG ] 22:19:00 DEBUG } 22:19:00 DEBUG 1713647940, 22:19:00 DEBUG ] 22:19:00 DEBUG 1713647940, 22:19:00 DEBUG "0" 22:19:00 DEBUG 1713647940, 22:19:00 DEBUG } 22:19:00 DEBUG ] 22:19:00 DEBUG 1713647940, 22:19:00 DEBUG ] 22:19:00 DEBUG ] 22:19:00 DEBUG "0" 22:19:00 DEBUG } 22:19:00 DEBUG } 22:19:00 DEBUG "0" 22:19:00 DEBUG } 22:19:00 DEBUG ] 22:19:00 DEBUG "371062" 22:19:00 DEBUG } 22:19:00 DEBUG "0" 22:19:00 DEBUG ] 22:19:00 DEBUG "0" 22:19:00 DEBUG } 22:19:00 DEBUG } 22:19:00 DEBUG "0" 22:19:00 DEBUG } 22:19:00 DEBUG } 22:19:00 DEBUG ] 22:19:00 DEBUG ] 22:19:00 DEBUG ] 22:19:00 DEBUG ] 22:19:00 DEBUG } 22:19:00 DEBUG } 22:19:00 DEBUG ] 22:19:00 DEBUG ] 22:19:00 DEBUG ] 22:19:00 DEBUG } 22:19:00 DEBUG ] 22:19:00 DEBUG ] 22:19:00 DEBUG ] 22:19:00 DEBUG ] 22:19:00 DEBUG ] 22:19:00 DEBUG } 22:19:00 DEBUG } 22:19:00 DEBUG } 22:19:00 DEBUG } 22:19:00 DEBUG } 22:19:00 DEBUG } 22:19:00 DEBUG } 22:19:00 DEBUG } 22:19:00 DEBUG ] 22:19:00 DEBUG } 22:19:00 DEBUG } 22:19:00 DEBUG } 22:19:00 DEBUG } 22:19:00 DEBUG } 22:19:00 DEBUG ] 22:19:00 DEBUG } 22:19:00 DEBUG } 22:19:00 DEBUG ] 22:19:00 DEBUG ] 22:19:00 DEBUG } 22:19:00 DEBUG ] 22:19:00 DEBUG } 22:19:00 DEBUG ] 22:19:00 DEBUG } 22:19:00 DEBUG ] 22:19:00 DEBUG } 22:19:00 DEBUG } 22:19:00 DEBUG } 22:19:00 DEBUG } 22:19:00 DEBUG } 22:19:00 DEBUG } 22:19:00 DEBUG } 22:19:00 DEBUG } 22:19:00 DEBUG } 22:19:00 DEBUG } 22:19:00 DEBUG } 22:19:00 DEBUG } 22:19:00 DEBUG } 22:19:00 DEBUG } 22:19:00 DEBUG } 22:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647995" 22:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713647995" 22:19:55 INFO [http_cmd]: http status code OK 22:19:55 DEBUG --- status code --- 22:19:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:55 DEBUG --- http response --- 22:19:55 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:19:55 ERROR [http_cmd]: ERROR 22:19:55 ERROR --- status code --- 22:19:55 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:19:55 ERROR --- http response --- 22:19:55 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:19:55 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713647995" 22:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647995" 22:19:55 INFO [http_cmd]: http status code OK 22:19:55 INFO [http_cmd]: http status code OK 22:19:55 DEBUG --- status code --- 22:19:55 DEBUG --- status code --- 22:19:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:55 DEBUG --- http response --- 22:19:55 DEBUG --- http response --- 22:19:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:19:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:19:55 DEBUG ----- output ----- 22:19:55 DEBUG ----- output ----- 22:19:55 DEBUG { 22:19:55 DEBUG { 22:19:55 DEBUG "status": "success", 22:19:55 DEBUG "status": "success", 22:19:55 DEBUG "data": { 22:19:55 DEBUG "data": { 22:19:55 DEBUG "resultType": "vector", 22:19:55 DEBUG "resultType": "vector", 22:19:55 DEBUG "result": [ 22:19:55 DEBUG "result": [ 22:19:55 DEBUG { 22:19:55 DEBUG { 22:19:55 DEBUG "metric": { 22:19:55 DEBUG "metric": { 22:19:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:19:55 DEBUG "pod": "ds-idrepo-0" 22:19:55 DEBUG }, 22:19:55 DEBUG "value": [ 22:19:55 DEBUG 22:19:55 DEBUG }, 22:19:55 DEBUG 22:19:55 DEBUG "value": [ 22:19:55 DEBUG 22:19:55 DEBUG 22:19:55 DEBUG [...] 22:19:55 DEBUG 22:19:55 DEBUG 22:19:55 DEBUG 22:19:55 DEBUG 22:19:55 DEBUG 22:19:55 DEBUG [...] 22:19:55 DEBUG 22:19:55 DEBUG "pod": "overseer-0-967d957f7-zk66v" 22:19:55 DEBUG 22:19:55 DEBUG }, 22:19:55 DEBUG 22:19:55 DEBUG "value": [ 22:19:55 DEBUG "pod": "ds-cts-2" 22:19:55 DEBUG 1713647995, 22:19:55 DEBUG }, 22:19:55 DEBUG "0" 22:19:55 DEBUG "value": [ 22:19:55 DEBUG ] 22:19:55 DEBUG 1713647995, 22:19:55 DEBUG } 22:19:55 DEBUG "0" 22:19:55 DEBUG ] 22:19:55 DEBUG ] 22:19:55 DEBUG } 22:19:55 DEBUG } 22:19:55 DEBUG } 22:19:55 DEBUG ] 22:19:55 DEBUG } 22:19:55 DEBUG } 22:19:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713647997" 22:19:57 INFO [http_cmd]: http status code OK 22:19:57 DEBUG --- status code --- 22:19:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:57 DEBUG --- http response --- 22:19:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:19:57 DEBUG ----- output ----- 22:19:57 DEBUG { 22:19:57 DEBUG "status": "success", 22:19:57 DEBUG "data": { 22:19:57 DEBUG "resultType": "vector", 22:19:57 DEBUG "result": [ 22:19:57 DEBUG { 22:19:57 DEBUG "metric": { 22:19:57 DEBUG "pod": "am-767f6b67b5-9zw22" 22:19:57 DEBUG }, 22:19:57 DEBUG "value": [ 22:19:57 DEBUG 22:19:57 DEBUG 22:19:57 DEBUG 22:19:57 DEBUG [...] 22:19:57 DEBUG 22:19:57 DEBUG 22:19:57 DEBUG 22:19:57 DEBUG "pod": "am-767f6b67b5-vbltz" 22:19:57 DEBUG }, 22:19:57 DEBUG "value": [ 22:19:57 DEBUG 1713647997, 22:19:57 DEBUG "0.06666666666666667" 22:19:57 DEBUG ] 22:19:57 DEBUG } 22:19:57 DEBUG ] 22:19:57 DEBUG } 22:19:57 DEBUG } 22:19:58 INFO 22:19:58 INFO [loop_until]: kubectl --namespace=xlou top pods 22:19:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:19:58 INFO [loop_until]: OK (rc = 0) 22:19:58 DEBUG --- stdout --- 22:19:58 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 2092m 8762Mi am-767f6b67b5-qcqz2 2573m 8850Mi am-767f6b67b5-vbltz 2754m 8864Mi ds-cts-0 17538m 12295Mi ds-cts-1 17618m 12282Mi ds-cts-2 18227m 12458Mi ds-idrepo-0 563m 3927Mi ds-idrepo-1 555m 3966Mi ds-idrepo-2 565m 3879Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-7dd48676f6-tdxl9 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-967d957f7-zk66v 2863m 16383Mi 22:19:58 DEBUG --- stderr --- 22:19:58 DEBUG 22:19:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713647999" 22:19:59 INFO [http_cmd]: http status code OK 22:19:59 DEBUG --- status code --- 22:19:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:59 DEBUG --- http response --- 22:19:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:19:59 DEBUG ----- output ----- 22:19:59 DEBUG { 22:19:59 DEBUG "status": "success", 22:19:59 DEBUG "data": { 22:19:59 DEBUG "resultType": "vector", 22:19:59 DEBUG "result": [ 22:19:59 DEBUG { 22:19:59 DEBUG "metric": { 22:19:59 DEBUG "instance": "10.142.0.107:9100" 22:19:59 DEBUG }, 22:19:59 DEBUG "value": [ 22:19:59 DEBUG 22:19:59 DEBUG 22:19:59 DEBUG 22:19:59 DEBUG [...] 22:19:59 DEBUG 22:19:59 DEBUG 22:19:59 DEBUG 22:19:59 DEBUG "instance": "10.142.0.43:9100" 22:19:59 DEBUG }, 22:19:59 DEBUG "value": [ 22:19:59 DEBUG 1713647999, 22:19:59 DEBUG "16131107.833333334" 22:19:59 DEBUG ] 22:19:59 DEBUG } 22:19:59 DEBUG ] 22:19:59 DEBUG } 22:19:59 DEBUG } 22:20:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648000" 22:20:00 INFO [http_cmd]: http status code OK 22:20:00 DEBUG --- status code --- 22:20:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:00 DEBUG --- http response --- 22:20:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:20:00 DEBUG ----- output ----- 22:20:00 DEBUG { 22:20:00 DEBUG "status": "success", 22:20:00 DEBUG "data": { 22:20:00 DEBUG "resultType": "vector", 22:20:00 DEBUG "result": [ 22:20:00 DEBUG { 22:20:00 DEBUG "metric": { 22:20:00 DEBUG "pod": "am-767f6b67b5-9zw22" 22:20:00 DEBUG }, 22:20:00 DEBUG "value": [ 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG [...] 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG "pod": "am-767f6b67b5-vbltz" 22:20:00 DEBUG }, 22:20:00 DEBUG "value": [ 22:20:00 DEBUG 1713648000, 22:20:00 DEBUG "0.06666666666666667" 22:20:00 DEBUG ] 22:20:00 DEBUG } 22:20:00 DEBUG ] 22:20:00 DEBUG } 22:20:00 DEBUG } 22:20:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713648000" 22:20:00 INFO [http_cmd]: http status code OK 22:20:00 DEBUG --- status code --- 22:20:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:00 DEBUG --- http response --- 22:20:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:20:00 DEBUG ----- output ----- 22:20:00 DEBUG { 22:20:00 DEBUG "status": "success", 22:20:00 DEBUG "data": { 22:20:00 DEBUG "resultType": "vector", 22:20:00 DEBUG "result": [ 22:20:00 DEBUG { 22:20:00 DEBUG "metric": { 22:20:00 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:20:00 DEBUG }, 22:20:00 DEBUG "value": [ 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG [...] 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:20:00 DEBUG }, 22:20:00 DEBUG "value": [ 22:20:00 DEBUG 1713648000, 22:20:00 DEBUG "3.496019730124456" 22:20:00 DEBUG ] 22:20:00 DEBUG } 22:20:00 DEBUG ] 22:20:00 DEBUG } 22:20:00 DEBUG } 22:20:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713648000" 22:20:00 INFO [http_cmd]: http status code OK 22:20:00 DEBUG --- status code --- 22:20:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:00 DEBUG --- http response --- 22:20:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:20:00 DEBUG ----- output ----- 22:20:00 DEBUG { 22:20:00 DEBUG "status": "success", 22:20:00 DEBUG "data": { 22:20:00 DEBUG "resultType": "vector", 22:20:00 DEBUG "result": [ 22:20:00 DEBUG { 22:20:00 DEBUG "metric": { 22:20:00 DEBUG "pod": "am-767f6b67b5-9zw22" 22:20:00 DEBUG }, 22:20:00 DEBUG "value": [ 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG [...] 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG "pod": "am-767f6b67b5-vbltz" 22:20:00 DEBUG }, 22:20:00 DEBUG "value": [ 22:20:00 DEBUG 1713648000, 22:20:00 DEBUG "100.11349999999999" 22:20:00 DEBUG ] 22:20:00 DEBUG } 22:20:00 DEBUG ] 22:20:00 DEBUG } 22:20:00 DEBUG } 22:20:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648000" 22:20:00 INFO [http_cmd]: http status code OK 22:20:00 DEBUG --- status code --- 22:20:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:00 DEBUG --- http response --- 22:20:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:20:00 DEBUG ----- output ----- 22:20:00 DEBUG { 22:20:00 DEBUG "status": "success", 22:20:00 DEBUG "data": { 22:20:00 DEBUG "resultType": "vector", 22:20:00 DEBUG "result": [ 22:20:00 DEBUG { 22:20:00 DEBUG "metric": { 22:20:00 DEBUG "pod": "am-767f6b67b5-9zw22" 22:20:00 DEBUG }, 22:20:00 DEBUG "value": [ 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG [...] 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG "pod": "am-767f6b67b5-vbltz" 22:20:00 DEBUG }, 22:20:00 DEBUG "value": [ 22:20:00 DEBUG 1713648000, 22:20:00 DEBUG "2836.1666666666665" 22:20:00 DEBUG ] 22:20:00 DEBUG } 22:20:00 DEBUG ] 22:20:00 DEBUG } 22:20:00 DEBUG } 22:20:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713648000" 22:20:00 INFO [http_cmd]: http status code OK 22:20:00 DEBUG --- status code --- 22:20:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:00 DEBUG --- http response --- 22:20:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:20:00 DEBUG ----- output ----- 22:20:00 DEBUG { 22:20:00 DEBUG "status": "success", 22:20:00 DEBUG "data": { 22:20:00 DEBUG "resultType": "vector", 22:20:00 DEBUG "result": [ 22:20:00 DEBUG { 22:20:00 DEBUG "metric": { 22:20:00 DEBUG "instance": "10.142.0.107:9100" 22:20:00 DEBUG }, 22:20:00 DEBUG "value": [ 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG [...] 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG "instance": "10.142.0.43:9100" 22:20:00 DEBUG }, 22:20:00 DEBUG "value": [ 22:20:00 DEBUG 1713648000, 22:20:00 DEBUG "0.09266666666666762" 22:20:00 DEBUG ] 22:20:00 DEBUG } 22:20:00 DEBUG ] 22:20:00 DEBUG } 22:20:00 DEBUG } 22:20:00 INFO 22:20:00 INFO [loop_until]: kubectl --namespace=xlou top node 22:20:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:20:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648000" 22:20:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648000" 22:20:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648000" 22:20:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713648000" 22:20:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713648000" 22:20:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713648000" 22:20:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648000" 22:20:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648000" 22:20:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648000" 22:20:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648000" 22:20:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648000" 22:20:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713648000" 22:20:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648000" 22:20:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648000" 22:20:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648000" 22:20:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648000" 22:20:00 INFO [http_cmd]: http status code OK 22:20:00 INFO [http_cmd]: http status code OK 22:20:00 INFO [http_cmd]: http status code OK 22:20:00 INFO [http_cmd]: http status code OK 22:20:00 INFO [http_cmd]: http status code OK 22:20:00 DEBUG --- status code --- 22:20:00 INFO [http_cmd]: http status code OK 22:20:00 DEBUG --- status code --- 22:20:00 INFO [http_cmd]: http status code OK 22:20:00 INFO [http_cmd]: http status code OK 22:20:00 INFO [http_cmd]: http status code OK 22:20:00 DEBUG --- status code --- 22:20:00 INFO [http_cmd]: http status code OK 22:20:00 DEBUG --- status code --- 22:20:00 INFO [http_cmd]: http status code OK 22:20:00 DEBUG --- status code --- 22:20:00 DEBUG --- status code --- 22:20:00 DEBUG --- status code --- 22:20:00 INFO [http_cmd]: http status code OK 22:20:00 INFO [http_cmd]: http status code OK 22:20:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:00 DEBUG --- status code --- 22:20:00 INFO [loop_until]: OK (rc = 0) 22:20:00 DEBUG --- status code --- 22:20:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:00 INFO [http_cmd]: http status code OK 22:20:00 DEBUG --- status code --- 22:20:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:00 DEBUG --- http response --- 22:20:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:00 DEBUG --- status code --- 22:20:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:00 DEBUG --- status code --- 22:20:00 DEBUG --- status code --- 22:20:00 DEBUG --- http response --- 22:20:00 DEBUG --- http response --- 22:20:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:00 DEBUG --- stdout --- 22:20:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:00 DEBUG --- http response --- 22:20:00 INFO [http_cmd]: http status code OK 22:20:00 DEBUG --- status code --- 22:20:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:20:00 DEBUG --- http response --- 22:20:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:00 DEBUG --- http response --- 22:20:00 DEBUG --- http response --- 22:20:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:20:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 22:20:00 DEBUG --- http response --- 22:20:00 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 6228Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 626m 2% 7697Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 2401m 8% 11506Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 81m 0% 3355Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 17987m 60% 15533Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 151m 0% 5208Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 2592m 8% 11782Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 131m 0% 3375Mi 3% gke-xlou-cdm-ds-0f601751-hmm7 2995m 10% 11141Mi 9% gke-xlou-cdm-ds-0f601751-j488 623m 2% 7287Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17762m 59% 17357Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 667m 2% 7945Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 18561m 62% 15141Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 3516m 22% 22653Mi 38% 22:20:00 DEBUG --- http response --- 22:20:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:20:00 DEBUG --- status code --- 22:20:00 INFO [http_cmd]: http status code OK 22:20:00 DEBUG --- http response --- 22:20:00 DEBUG --- http response --- 22:20:00 DEBUG ----- output ----- 22:20:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:20:00 DEBUG --- http response --- 22:20:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:20:00 DEBUG --- http response --- 22:20:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:20:00 DEBUG --- http response --- 22:20:00 DEBUG ----- output ----- 22:20:00 DEBUG ----- output ----- 22:20:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:20:00 DEBUG --- stderr --- 22:20:00 DEBUG ----- output ----- 22:20:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:20:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:00 DEBUG --- status code --- 22:20:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 22:20:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:20:00 DEBUG { 22:20:00 DEBUG ----- output ----- 22:20:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:20:00 DEBUG ----- output ----- 22:20:00 DEBUG ----- output ----- 22:20:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:20:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:20:00 DEBUG { 22:20:00 DEBUG { 22:20:00 DEBUG ----- output ----- 22:20:00 DEBUG 22:20:00 DEBUG { 22:20:00 DEBUG ----- output ----- 22:20:00 DEBUG --- http response --- 22:20:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:00 DEBUG ----- output ----- 22:20:00 DEBUG ----- output ----- 22:20:00 DEBUG "status": "success", 22:20:00 DEBUG { 22:20:00 DEBUG ----- output ----- 22:20:00 DEBUG { 22:20:00 DEBUG { 22:20:00 DEBUG ----- output ----- 22:20:00 DEBUG ----- output ----- 22:20:00 DEBUG "status": "success", 22:20:00 DEBUG "status": "success", 22:20:00 DEBUG { 22:20:00 DEBUG "status": "success", 22:20:00 DEBUG { 22:20:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:20:00 DEBUG --- http response --- 22:20:00 DEBUG { 22:20:00 DEBUG { 22:20:00 DEBUG "data": { 22:20:00 DEBUG "status": "success", 22:20:00 DEBUG { 22:20:00 DEBUG "status": "success", 22:20:00 DEBUG "status": "success", 22:20:00 DEBUG { 22:20:00 DEBUG { 22:20:00 DEBUG "data": { 22:20:00 DEBUG "data": { 22:20:00 DEBUG "status": "success", 22:20:00 DEBUG "data": { 22:20:00 DEBUG "status": "success", 22:20:00 DEBUG ----- output ----- 22:20:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:20:00 DEBUG "status": "success", 22:20:00 DEBUG "status": "success", 22:20:00 DEBUG "resultType": "vector", 22:20:00 DEBUG "data": { 22:20:00 DEBUG "status": "success", 22:20:00 DEBUG "data": { 22:20:00 DEBUG "data": { 22:20:00 DEBUG "status": "success", 22:20:00 DEBUG "status": "success", 22:20:00 DEBUG "resultType": "vector", 22:20:00 DEBUG "resultType": "vector", 22:20:00 DEBUG "data": { 22:20:00 DEBUG "resultType": "vector", 22:20:00 DEBUG "data": { 22:20:00 DEBUG { 22:20:00 DEBUG ----- output ----- 22:20:00 DEBUG "data": { 22:20:00 DEBUG "data": { 22:20:00 DEBUG "result": [ 22:20:00 DEBUG "resultType": "vector", 22:20:00 DEBUG "data": { 22:20:00 DEBUG "resultType": "vector", 22:20:00 DEBUG "resultType": "vector", 22:20:00 DEBUG "data": { 22:20:00 DEBUG "resultType": "vector", 22:20:00 DEBUG "result": [ 22:20:00 DEBUG "result": [ 22:20:00 DEBUG "resultType": "vector", 22:20:00 DEBUG "result": [ 22:20:00 DEBUG "resultType": "vector", 22:20:00 DEBUG "status": "success", 22:20:00 DEBUG { 22:20:00 DEBUG "resultType": "vector", 22:20:00 DEBUG "resultType": "vector", 22:20:00 DEBUG { 22:20:00 DEBUG "result": [ 22:20:00 DEBUG "resultType": "vector", 22:20:00 DEBUG "result": [ 22:20:00 DEBUG "result": [ 22:20:00 DEBUG "result": [ 22:20:00 DEBUG { 22:20:00 DEBUG "data": { 22:20:00 DEBUG "resultType": "vector", 22:20:00 DEBUG "result": [ 22:20:00 DEBUG { 22:20:00 DEBUG "result": [ 22:20:00 DEBUG "data": { 22:20:00 DEBUG "status": "success", 22:20:00 DEBUG "result": [ 22:20:00 DEBUG "result": [ 22:20:00 DEBUG "metric": { 22:20:00 DEBUG { 22:20:00 DEBUG "result": [ 22:20:00 DEBUG { 22:20:00 DEBUG { 22:20:00 DEBUG { 22:20:00 DEBUG "metric": { 22:20:00 DEBUG "result": [ 22:20:00 DEBUG { 22:20:00 DEBUG { 22:20:00 DEBUG "metric": { 22:20:00 DEBUG { 22:20:00 DEBUG "resultType": "vector", 22:20:00 DEBUG "data": { 22:20:00 DEBUG { 22:20:00 DEBUG { 22:20:00 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:20:00 DEBUG "metric": { 22:20:00 DEBUG { 22:20:00 DEBUG "metric": { 22:20:00 DEBUG "metric": { 22:20:00 DEBUG "metric": { 22:20:00 DEBUG "pod": "ds-idrepo-1" 22:20:00 DEBUG }, 22:20:00 DEBUG "metric": { 22:20:00 DEBUG "metric": { 22:20:00 DEBUG "instance": "10.142.0.107:9100" 22:20:00 DEBUG "metric": { 22:20:00 DEBUG "result": [ 22:20:00 DEBUG "resultType": "vector", 22:20:00 DEBUG "metric": { 22:20:00 DEBUG "metric": { 22:20:00 DEBUG }, 22:20:00 DEBUG "instance": "10.142.0.107:9100" 22:20:00 DEBUG "metric": { 22:20:00 DEBUG "instance": "10.142.0.107:9100" 22:20:00 DEBUG "pod": "ds-cts-0" 22:20:00 DEBUG "pod": "ds-idrepo-0" 22:20:00 DEBUG { 22:20:00 DEBUG "value": [ 22:20:00 DEBUG "pod": "ds-idrepo-1" 22:20:00 DEBUG "pod": "ds-cts-0" 22:20:00 DEBUG }, 22:20:00 DEBUG "pod": "am-767f6b67b5-9zw22" 22:20:00 DEBUG { 22:20:00 DEBUG "result": [ 22:20:00 DEBUG "pod": "ds-idrepo-1" 22:20:00 DEBUG "pod": "am-767f6b67b5-9zw22" 22:20:00 DEBUG "value": [ 22:20:00 DEBUG }, 22:20:00 DEBUG "pod": "ds-cts-0" 22:20:00 DEBUG }, 22:20:00 DEBUG }, 22:20:00 DEBUG }, 22:20:00 DEBUG "metric": { 22:20:00 DEBUG 22:20:00 DEBUG }, 22:20:00 DEBUG }, 22:20:00 DEBUG "value": [ 22:20:00 DEBUG }, 22:20:00 DEBUG "metric": { 22:20:00 DEBUG { 22:20:00 DEBUG }, 22:20:00 DEBUG }, 22:20:00 DEBUG 22:20:00 DEBUG "value": [ 22:20:00 DEBUG }, 22:20:00 DEBUG "value": [ 22:20:00 DEBUG "value": [ 22:20:00 DEBUG "value": [ 22:20:00 DEBUG "pod": "ds-idrepo-0" 22:20:00 DEBUG 22:20:00 DEBUG "value": [ 22:20:00 DEBUG "value": [ 22:20:00 DEBUG 22:20:00 DEBUG "value": [ 22:20:00 DEBUG "pod": "am-767f6b67b5-9zw22" 22:20:00 DEBUG "metric": { 22:20:00 DEBUG "pod": "ds-idrepo-0" 22:20:00 DEBUG "value": [ 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG "value": [ 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG }, 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG }, 22:20:00 DEBUG "value": [ 22:20:00 DEBUG }, 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG "value": [ 22:20:00 DEBUG [...] 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG "value": [ 22:20:00 DEBUG 22:20:00 DEBUG "value": [ 22:20:00 DEBUG 22:20:00 DEBUG [...] 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG [...] 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG [...] 22:20:00 DEBUG 22:20:00 DEBUG [...] 22:20:00 DEBUG [...] 22:20:00 DEBUG [...] 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG [...] 22:20:00 DEBUG [...] 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG [...] 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG [...] 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG [...] 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG [...] 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG "pod": "overseer-0-967d957f7-zk66v" 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG "instance": "10.142.0.43:9100" 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG [...] 22:20:00 DEBUG 22:20:00 DEBUG }, 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG "pod": "ds-idrepo-2" 22:20:00 DEBUG [...] 22:20:00 DEBUG }, 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG }, 22:20:00 DEBUG [...] 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG "value": [ 22:20:00 DEBUG 22:20:00 DEBUG "instance": "10.142.0.43:9100" 22:20:00 DEBUG "pod": "ds-cts-2" 22:20:00 DEBUG 22:20:00 DEBUG }, 22:20:00 DEBUG 22:20:00 DEBUG "value": [ 22:20:00 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:20:00 DEBUG "pod": "ds-cts-2" 22:20:00 DEBUG "value": [ 22:20:00 DEBUG 22:20:00 DEBUG "pod": "am-767f6b67b5-vbltz" 22:20:00 DEBUG 22:20:00 DEBUG 22:20:00 DEBUG "pod": "am-767f6b67b5-vbltz" 22:20:00 DEBUG 1713648000, 22:20:00 DEBUG "pod": "ds-cts-2" 22:20:00 DEBUG }, 22:20:00 DEBUG }, 22:20:00 DEBUG "instance": "10.142.0.43:9100" 22:20:00 DEBUG "value": [ 22:20:00 DEBUG 22:20:00 DEBUG 1713648000, 22:20:00 DEBUG }, 22:20:00 DEBUG }, 22:20:00 DEBUG 1713648000, 22:20:00 DEBUG 22:20:00 DEBUG }, 22:20:00 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:20:00 DEBUG 22:20:00 DEBUG }, 22:20:00 DEBUG "17277538304" 22:20:00 DEBUG }, 22:20:00 DEBUG "value": [ 22:20:00 DEBUG "value": [ 22:20:00 DEBUG }, 22:20:00 DEBUG 1713648000, 22:20:00 DEBUG 22:20:00 DEBUG "0" 22:20:00 DEBUG "value": [ 22:20:00 DEBUG "value": [ 22:20:00 DEBUG "4125444705280" 22:20:00 DEBUG 22:20:00 DEBUG "value": [ 22:20:00 DEBUG }, 22:20:00 DEBUG "pod": "ds-idrepo-2" 22:20:00 DEBUG "value": [ 22:20:00 DEBUG ] 22:20:00 DEBUG "value": [ 22:20:00 DEBUG 1713648000, 22:20:00 DEBUG 1713648000, 22:20:00 DEBUG "value": [ 22:20:00 DEBUG "0" 22:20:00 DEBUG "pod": "ds-idrepo-2" 22:20:00 DEBUG ] 22:20:00 DEBUG 1713648000, 22:20:00 DEBUG 1713648000, 22:20:00 DEBUG ] 22:20:00 DEBUG "pod": "am-767f6b67b5-vbltz" 22:20:00 DEBUG 1713648000, 22:20:00 DEBUG "value": [ 22:20:00 DEBUG }, 22:20:00 DEBUG 1713648000, 22:20:00 DEBUG } 22:20:00 DEBUG 1713648000, 22:20:00 DEBUG "16138018.833333334" 22:20:00 DEBUG "29.566666666666666" 22:20:00 DEBUG ] 22:20:00 DEBUG ] 22:20:00 DEBUG }, 22:20:00 DEBUG } 22:20:00 DEBUG "0" 22:20:00 DEBUG "0" 22:20:00 DEBUG } 22:20:00 DEBUG }, 22:20:00 DEBUG "269704.5" 22:20:00 DEBUG 1713648000, 22:20:00 DEBUG "value": [ 22:20:00 DEBUG "0" 22:20:00 DEBUG ] 22:20:00 DEBUG "25.884966666666667" 22:20:00 DEBUG ] 22:20:00 DEBUG 1713648000, 22:20:00 DEBUG } 22:20:00 DEBUG } 22:20:00 DEBUG "value": [ 22:20:00 DEBUG ] 22:20:00 DEBUG ] 22:20:00 DEBUG ] 22:20:00 DEBUG ] 22:20:00 DEBUG "value": [ 22:20:00 DEBUG ] 22:20:00 DEBUG "0" 22:20:00 DEBUG 1713648000, 22:20:00 DEBUG ] 22:20:00 DEBUG } 22:20:00 DEBUG ] 22:20:00 DEBUG } 22:20:00 DEBUG "0.48936666666672957" 22:20:00 DEBUG ] 22:20:00 DEBUG ] 22:20:00 DEBUG 1713648000, 22:20:00 DEBUG } 22:20:00 DEBUG } 22:20:00 DEBUG } 22:20:00 DEBUG } 22:20:00 DEBUG 1713648000, 22:20:00 DEBUG } 22:20:00 DEBUG ] 22:20:00 DEBUG } 22:20:00 DEBUG } 22:20:00 DEBUG } 22:20:00 DEBUG } 22:20:00 DEBUG ] 22:20:00 DEBUG ] 22:20:00 DEBUG } 22:20:00 DEBUG } 22:20:00 DEBUG } 22:20:00 DEBUG } 22:20:00 DEBUG ] 22:20:00 DEBUG ] 22:20:00 DEBUG } 22:20:00 DEBUG "0.06666666666666667" 22:20:00 DEBUG ] 22:20:00 DEBUG "0" 22:20:00 DEBUG ] 22:20:00 DEBUG ] 22:20:00 DEBUG ] 22:20:00 DEBUG } 22:20:00 DEBUG } 22:20:00 DEBUG } 22:20:00 DEBUG "0" 22:20:00 DEBUG } 22:20:00 DEBUG } 22:20:00 DEBUG ] 22:20:00 DEBUG } 22:20:00 DEBUG ] 22:20:00 DEBUG } 22:20:00 DEBUG } 22:20:00 DEBUG } 22:20:00 DEBUG } 22:20:00 DEBUG ] 22:20:00 DEBUG ] 22:20:00 DEBUG } 22:20:00 DEBUG } 22:20:00 DEBUG } 22:20:00 DEBUG } 22:20:00 DEBUG } 22:20:00 DEBUG } 22:20:00 DEBUG } 22:20:00 DEBUG } 22:20:00 DEBUG } 22:20:00 DEBUG } 22:20:00 DEBUG ] 22:20:00 DEBUG ] 22:20:00 DEBUG } 22:20:00 DEBUG ] 22:20:00 DEBUG } 22:20:00 DEBUG } 22:20:00 DEBUG } 22:20:00 DEBUG } 22:20:00 DEBUG } 22:20:00 DEBUG } 22:20:44 WARNING File /tmp/lodemon_data-20240420_213952/node_disk_read_bytes_total.json does not exist. Skipping average value computation... 127.0.0.1 - - [20/Apr/2024 22:20:44] "GET /monitoring/average?start_time=20240420_211408&stop_time=20240420_211844 HTTP/1.1" 200 - 22:20:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713648055" 22:20:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648055" 22:20:55 ERROR [http_cmd]: ERROR 22:20:55 ERROR --- status code --- 22:20:55 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:20:55 ERROR --- http response --- 22:20:55 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:20:55 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:20:55 INFO [http_cmd]: http status code OK 22:20:55 DEBUG --- status code --- 22:20:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:55 DEBUG --- http response --- 22:20:55 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:20:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713648055" 22:20:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648055" 22:20:55 INFO [http_cmd]: http status code OK 22:20:55 DEBUG --- status code --- 22:20:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:55 DEBUG --- http response --- 22:20:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:20:55 DEBUG ----- output ----- 22:20:55 DEBUG { 22:20:55 DEBUG "status": "success", 22:20:55 DEBUG "data": { 22:20:55 DEBUG "resultType": "vector", 22:20:55 DEBUG "result": [ 22:20:55 DEBUG { 22:20:55 DEBUG "metric": { 22:20:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:20:55 DEBUG }, 22:20:55 DEBUG "value": [ 22:20:55 DEBUG 22:20:55 DEBUG 22:20:55 DEBUG 22:20:55 DEBUG [...] 22:20:55 DEBUG 22:20:55 DEBUG 22:20:55 DEBUG 22:20:55 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 22:20:55 DEBUG }, 22:20:55 DEBUG "value": [ 22:20:55 DEBUG 1713648055, 22:20:55 DEBUG "0" 22:20:55 DEBUG ] 22:20:55 DEBUG } 22:20:55 DEBUG ] 22:20:55 DEBUG } 22:20:55 DEBUG } 22:20:55 INFO [http_cmd]: http status code OK 22:20:55 DEBUG --- status code --- 22:20:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:55 DEBUG --- http response --- 22:20:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:20:55 DEBUG ----- output ----- 22:20:55 DEBUG { 22:20:55 DEBUG "status": "success", 22:20:55 DEBUG "data": { 22:20:55 DEBUG "resultType": "vector", 22:20:55 DEBUG "result": [ 22:20:55 DEBUG { 22:20:55 DEBUG "metric": { 22:20:55 DEBUG "pod": "ds-idrepo-0" 22:20:55 DEBUG }, 22:20:55 DEBUG "value": [ 22:20:55 DEBUG 22:20:55 DEBUG 22:20:55 DEBUG 22:20:55 DEBUG [...] 22:20:55 DEBUG 22:20:55 DEBUG 22:20:55 DEBUG 22:20:55 DEBUG "pod": "ds-cts-2" 22:20:55 DEBUG }, 22:20:55 DEBUG "value": [ 22:20:55 DEBUG 1713648055, 22:20:55 DEBUG "0" 22:20:55 DEBUG ] 22:20:55 DEBUG } 22:20:55 DEBUG ] 22:20:55 DEBUG } 22:20:55 DEBUG } 22:20:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648057" 22:20:57 INFO [http_cmd]: http status code OK 22:20:57 DEBUG --- status code --- 22:20:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:57 DEBUG --- http response --- 22:20:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:20:57 DEBUG ----- output ----- 22:20:57 DEBUG { 22:20:57 DEBUG "status": "success", 22:20:57 DEBUG "data": { 22:20:57 DEBUG "resultType": "vector", 22:20:57 DEBUG "result": [ 22:20:57 DEBUG { 22:20:57 DEBUG "metric": { 22:20:57 DEBUG "pod": "am-767f6b67b5-9zw22" 22:20:57 DEBUG }, 22:20:57 DEBUG "value": [ 22:20:57 DEBUG 22:20:57 DEBUG 22:20:57 DEBUG 22:20:57 DEBUG [...] 22:20:57 DEBUG 22:20:57 DEBUG 22:20:57 DEBUG 22:20:57 DEBUG "pod": "am-767f6b67b5-vbltz" 22:20:57 DEBUG }, 22:20:57 DEBUG "value": [ 22:20:57 DEBUG 1713648057, 22:20:57 DEBUG "0" 22:20:57 DEBUG ] 22:20:57 DEBUG } 22:20:57 DEBUG ] 22:20:57 DEBUG } 22:20:57 DEBUG } 22:20:58 INFO 22:20:58 INFO [loop_until]: kubectl --namespace=xlou top pods 22:20:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:20:58 INFO [loop_until]: OK (rc = 0) 22:20:58 DEBUG --- stdout --- 22:20:58 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 2720m 8727Mi am-767f6b67b5-qcqz2 2656m 8841Mi am-767f6b67b5-vbltz 2568m 8859Mi ds-cts-0 18408m 12299Mi ds-cts-1 12537m 12270Mi ds-cts-2 18921m 12452Mi ds-idrepo-0 614m 3927Mi ds-idrepo-1 622m 3966Mi ds-idrepo-2 600m 3879Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-7dd48676f6-tdxl9 1m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-967d957f7-zk66v 3070m 16Mi 22:20:58 DEBUG --- stderr --- 22:20:58 DEBUG 22:20:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713648059" 22:20:59 INFO [http_cmd]: http status code OK 22:20:59 DEBUG --- status code --- 22:20:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:59 DEBUG --- http response --- 22:20:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:20:59 DEBUG ----- output ----- 22:20:59 DEBUG { 22:20:59 DEBUG "status": "success", 22:20:59 DEBUG "data": { 22:20:59 DEBUG "resultType": "vector", 22:20:59 DEBUG "result": [ 22:20:59 DEBUG { 22:20:59 DEBUG "metric": { 22:20:59 DEBUG "instance": "10.142.0.107:9100" 22:20:59 DEBUG }, 22:20:59 DEBUG "value": [ 22:20:59 DEBUG 22:20:59 DEBUG 22:20:59 DEBUG 22:20:59 DEBUG [...] 22:20:59 DEBUG 22:20:59 DEBUG 22:20:59 DEBUG 22:20:59 DEBUG "instance": "10.142.0.43:9100" 22:20:59 DEBUG }, 22:20:59 DEBUG "value": [ 22:20:59 DEBUG 1713648059, 22:20:59 DEBUG "17808175.7" 22:20:59 DEBUG ] 22:20:59 DEBUG } 22:20:59 DEBUG ] 22:20:59 DEBUG } 22:20:59 DEBUG } 22:21:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648060" 22:21:00 INFO [http_cmd]: http status code OK 22:21:00 DEBUG --- status code --- 22:21:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:00 DEBUG --- http response --- 22:21:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:21:00 DEBUG ----- output ----- 22:21:00 DEBUG { 22:21:00 DEBUG "status": "success", 22:21:00 DEBUG "data": { 22:21:00 DEBUG "resultType": "vector", 22:21:00 DEBUG "result": [ 22:21:00 DEBUG { 22:21:00 DEBUG "metric": { 22:21:00 DEBUG "pod": "am-767f6b67b5-9zw22" 22:21:00 DEBUG }, 22:21:00 DEBUG "value": [ 22:21:00 DEBUG 22:21:00 DEBUG 22:21:00 DEBUG 22:21:00 DEBUG [...] 22:21:00 DEBUG 22:21:00 DEBUG 22:21:00 DEBUG 22:21:00 DEBUG "pod": "am-767f6b67b5-vbltz" 22:21:00 DEBUG }, 22:21:00 DEBUG "value": [ 22:21:00 DEBUG 1713648060, 22:21:00 DEBUG "0" 22:21:00 DEBUG ] 22:21:00 DEBUG } 22:21:00 DEBUG ] 22:21:00 DEBUG } 22:21:00 DEBUG } 22:21:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713648060" 22:21:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713648060" 22:21:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648060" 22:21:00 INFO [http_cmd]: http status code OK 22:21:00 DEBUG --- status code --- 22:21:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:00 DEBUG --- http response --- 22:21:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:21:00 DEBUG ----- output ----- 22:21:00 DEBUG { 22:21:00 DEBUG "status": "success", 22:21:00 DEBUG "data": { 22:21:00 DEBUG "resultType": "vector", 22:21:00 DEBUG "result": [ 22:21:00 DEBUG { 22:21:00 DEBUG "metric": { 22:21:00 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:21:00 DEBUG }, 22:21:00 DEBUG "value": [ 22:21:00 DEBUG 22:21:00 DEBUG 22:21:00 DEBUG 22:21:00 DEBUG [...] 22:21:00 DEBUG 22:21:00 DEBUG 22:21:00 DEBUG 22:21:00 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:21:00 DEBUG }, 22:21:00 DEBUG "value": [ 22:21:00 DEBUG 1713648060, 22:21:00 DEBUG "5.465781692469747" 22:21:00 DEBUG ] 22:21:00 DEBUG } 22:21:00 DEBUG ] 22:21:00 DEBUG } 22:21:00 DEBUG } 22:21:00 INFO [http_cmd]: http status code OK 22:21:00 DEBUG --- status code --- 22:21:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:00 DEBUG --- http response --- 22:21:00 INFO [http_cmd]: http status code OK 22:21:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:21:00 DEBUG --- status code --- 22:21:00 DEBUG ----- output ----- 22:21:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:00 DEBUG { 22:21:00 DEBUG --- http response --- 22:21:00 DEBUG "status": "success", 22:21:00 DEBUG "data": { 22:21:00 DEBUG "resultType": "vector", 22:21:00 DEBUG "result": [ 22:21:00 DEBUG { 22:21:00 DEBUG "metric": { 22:21:00 DEBUG "pod": "am-767f6b67b5-9zw22" 22:21:00 DEBUG }, 22:21:00 DEBUG "value": [ 22:21:00 DEBUG 22:21:00 DEBUG 22:21:00 DEBUG 22:21:00 DEBUG [...] 22:21:00 DEBUG 22:21:00 DEBUG 22:21:00 DEBUG 22:21:00 DEBUG "pod": "am-767f6b67b5-vbltz" 22:21:00 DEBUG }, 22:21:00 DEBUG "value": [ 22:21:00 DEBUG 1713648060, 22:21:00 DEBUG "101.751" 22:21:00 DEBUG ] 22:21:00 DEBUG } 22:21:00 DEBUG ] 22:21:00 DEBUG } 22:21:00 DEBUG } 22:21:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:21:00 DEBUG ----- output ----- 22:21:00 DEBUG { 22:21:00 DEBUG "status": "success", 22:21:00 DEBUG "data": { 22:21:00 DEBUG "resultType": "vector", 22:21:00 DEBUG "result": [ 22:21:00 DEBUG { 22:21:00 DEBUG "metric": { 22:21:00 DEBUG "pod": "am-767f6b67b5-9zw22" 22:21:00 DEBUG }, 22:21:00 DEBUG "value": [ 22:21:00 DEBUG 22:21:00 DEBUG 22:21:00 DEBUG 22:21:00 DEBUG [...] 22:21:00 DEBUG 22:21:00 DEBUG 22:21:00 DEBUG 22:21:00 DEBUG "pod": "am-767f6b67b5-vbltz" 22:21:00 DEBUG }, 22:21:00 DEBUG "value": [ 22:21:00 DEBUG 1713648060, 22:21:00 DEBUG "2342.633333333333" 22:21:00 DEBUG ] 22:21:00 DEBUG } 22:21:00 DEBUG ] 22:21:00 DEBUG } 22:21:00 DEBUG } 22:21:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713648060" 22:21:00 INFO [http_cmd]: http status code OK 22:21:00 DEBUG --- status code --- 22:21:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:00 DEBUG --- http response --- 22:21:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:21:00 DEBUG ----- output ----- 22:21:00 DEBUG { 22:21:00 DEBUG "status": "success", 22:21:00 DEBUG "data": { 22:21:00 DEBUG "resultType": "vector", 22:21:00 DEBUG "result": [ 22:21:00 DEBUG { 22:21:00 DEBUG "metric": { 22:21:00 DEBUG "instance": "10.142.0.107:9100" 22:21:00 DEBUG }, 22:21:00 DEBUG "value": [ 22:21:00 DEBUG 22:21:00 DEBUG 22:21:00 DEBUG 22:21:00 DEBUG [...] 22:21:00 DEBUG 22:21:00 DEBUG 22:21:00 DEBUG 22:21:00 DEBUG "instance": "10.142.0.43:9100" 22:21:00 DEBUG }, 22:21:00 DEBUG "value": [ 22:21:00 DEBUG 1713648060, 22:21:00 DEBUG "0.18066666666665954" 22:21:00 DEBUG ] 22:21:00 DEBUG } 22:21:00 DEBUG ] 22:21:00 DEBUG } 22:21:00 DEBUG } 22:21:00 INFO 22:21:00 INFO [loop_until]: kubectl --namespace=xlou top node 22:21:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:21:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648060" 22:21:00 INFO [http_cmd]: http status code OK 22:21:00 DEBUG --- status code --- 22:21:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:00 DEBUG --- http response --- 22:21:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:21:00 DEBUG ----- output ----- 22:21:00 DEBUG { 22:21:00 DEBUG "status": "success", 22:21:00 DEBUG "data": { 22:21:00 DEBUG "resultType": "vector", 22:21:00 DEBUG "result": [ 22:21:00 DEBUG { 22:21:00 DEBUG "metric": { 22:21:00 DEBUG "pod": "ds-cts-0" 22:21:00 DEBUG }, 22:21:00 DEBUG "value": [ 22:21:00 DEBUG 22:21:00 DEBUG 22:21:00 DEBUG 22:21:00 DEBUG [...] 22:21:00 DEBUG 22:21:00 DEBUG 22:21:00 DEBUG 22:21:00 DEBUG "pod": "ds-cts-2" 22:21:00 DEBUG }, 22:21:00 DEBUG "value": [ 22:21:00 DEBUG 1713648060, 22:21:00 DEBUG "30" 22:21:00 DEBUG ] 22:21:00 DEBUG } 22:21:00 DEBUG ] 22:21:00 DEBUG } 22:21:00 DEBUG } 22:21:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648060" 22:21:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648060" 22:21:00 INFO [http_cmd]: http status code OK 22:21:00 DEBUG --- status code --- 22:21:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:00 DEBUG --- http response --- 22:21:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:21:00 DEBUG ----- output ----- 22:21:00 DEBUG { 22:21:00 DEBUG "status": "success", 22:21:00 DEBUG "data": { 22:21:00 DEBUG "resultType": "vector", 22:21:00 DEBUG "result": [ 22:21:00 DEBUG { 22:21:00 DEBUG "metric": { 22:21:00 DEBUG "pod": "am-767f6b67b5-9zw22" 22:21:00 DEBUG }, 22:21:00 DEBUG "value": [ 22:21:00 DEBUG 22:21:00 DEBUG 22:21:00 DEBUG 22:21:00 DEBUG [...] 22:21:00 DEBUG 22:21:00 DEBUG 22:21:00 DEBUG 22:21:00 DEBUG "pod": "am-767f6b67b5-vbltz" 22:21:00 INFO [http_cmd]: http status code OK 22:21:00 DEBUG }, 22:21:00 DEBUG --- status code --- 22:21:00 DEBUG "value": [ 22:21:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:00 DEBUG 1713648060, 22:21:00 DEBUG --- http response --- 22:21:00 DEBUG "0" 22:21:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 22:21:00 DEBUG ] 22:21:00 DEBUG ----- output ----- 22:21:00 INFO [loop_until]: OK (rc = 0) 22:21:00 DEBUG } 22:21:00 DEBUG { 22:21:00 DEBUG --- stdout --- 22:21:00 DEBUG ] 22:21:00 DEBUG "status": "success", 22:21:00 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6231Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 690m 2% 7702Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 2698m 9% 11490Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 88m 0% 3380Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 18531m 62% 15544Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 153m 0% 5182Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 2671m 8% 11782Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 74m 0% 3304Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 2930m 9% 11138Mi 9% gke-xlou-cdm-ds-0f601751-j488 671m 2% 7283Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 11534m 38% 17386Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 700m 2% 7947Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 19151m 64% 15152Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 121m 0% 8176Mi 13% 22:21:00 DEBUG } 22:21:00 DEBUG "data": { 22:21:00 DEBUG --- stderr --- 22:21:00 DEBUG } 22:21:00 DEBUG "resultType": "vector", 22:21:00 DEBUG 22:21:00 DEBUG "result": [ 22:21:00 DEBUG { 22:21:00 DEBUG "metric": { 22:21:00 DEBUG "pod": "ds-idrepo-1" 22:21:00 DEBUG }, 22:21:00 DEBUG "value": [ 22:21:00 DEBUG 22:21:00 DEBUG 22:21:00 DEBUG 22:21:00 DEBUG [...] 22:21:00 DEBUG 22:21:00 DEBUG 22:21:00 DEBUG 22:21:00 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 22:21:00 DEBUG }, 22:21:00 DEBUG "value": [ 22:21:00 DEBUG 1713648060, 22:21:00 DEBUG "0" 22:21:00 DEBUG ] 22:21:00 DEBUG } 22:21:00 DEBUG ] 22:21:00 DEBUG } 22:21:00 DEBUG } 22:21:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713648060" 22:21:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648061" 22:21:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648061" 22:21:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648061" 22:21:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648061" 22:21:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713648061" 22:21:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713648061" 22:21:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648061" 22:21:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648061" 22:21:01 INFO [http_cmd]: http status code OK 22:21:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648061" 22:21:01 INFO [http_cmd]: http status code OK 22:21:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713648061" 22:21:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648061" 22:21:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648061" 22:21:01 INFO [http_cmd]: http status code OK 22:21:01 DEBUG --- status code --- 22:21:01 INFO [http_cmd]: http status code OK 22:21:01 DEBUG --- status code --- 22:21:01 INFO [http_cmd]: http status code OK 22:21:01 INFO [http_cmd]: http status code OK 22:21:01 INFO [http_cmd]: http status code OK 22:21:01 DEBUG --- status code --- 22:21:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:01 DEBUG --- status code --- 22:21:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:01 DEBUG --- status code --- 22:21:01 DEBUG --- status code --- 22:21:01 DEBUG --- status code --- 22:21:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:01 DEBUG --- http response --- 22:21:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:01 DEBUG --- http response --- 22:21:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:01 DEBUG --- http response --- 22:21:01 INFO [http_cmd]: http status code OK 22:21:01 INFO [http_cmd]: http status code OK 22:21:01 DEBUG --- http response --- 22:21:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:21:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:21:01 DEBUG --- http response --- 22:21:01 DEBUG --- http response --- 22:21:01 DEBUG --- http response --- 22:21:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:21:01 DEBUG --- status code --- 22:21:01 DEBUG --- status code --- 22:21:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:21:01 INFO [http_cmd]: http status code OK 22:21:01 DEBUG ----- output ----- 22:21:01 DEBUG ----- output ----- 22:21:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:21:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:21:01 INFO [http_cmd]: http status code OK 22:21:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:21:01 INFO [http_cmd]: http status code OK 22:21:01 INFO [http_cmd]: http status code OK 22:21:01 DEBUG ----- output ----- 22:21:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:01 DEBUG ----- output ----- 22:21:01 DEBUG --- status code --- 22:21:01 DEBUG { 22:21:01 DEBUG { 22:21:01 DEBUG ----- output ----- 22:21:01 DEBUG ----- output ----- 22:21:01 DEBUG --- status code --- 22:21:01 DEBUG ----- output ----- 22:21:01 DEBUG --- status code --- 22:21:01 DEBUG --- status code --- 22:21:01 DEBUG { 22:21:01 DEBUG --- http response --- 22:21:01 DEBUG --- http response --- 22:21:01 DEBUG { 22:21:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:01 DEBUG "status": "success", 22:21:01 DEBUG "status": "success", 22:21:01 DEBUG { 22:21:01 DEBUG { 22:21:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:01 DEBUG { 22:21:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:01 DEBUG "status": "success", 22:21:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:21:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:21:01 DEBUG "status": "success", 22:21:01 DEBUG --- http response --- 22:21:01 DEBUG "data": { 22:21:01 DEBUG "data": { 22:21:01 DEBUG "status": "success", 22:21:01 DEBUG "status": "success", 22:21:01 DEBUG --- http response --- 22:21:01 DEBUG "status": "success", 22:21:01 DEBUG "data": { 22:21:01 DEBUG --- http response --- 22:21:01 DEBUG "data": { 22:21:01 DEBUG ----- output ----- 22:21:01 DEBUG ----- output ----- 22:21:01 DEBUG "data": { 22:21:01 DEBUG "resultType": "vector", 22:21:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:21:01 DEBUG "resultType": "vector", 22:21:01 DEBUG "data": { 22:21:01 DEBUG "data": { 22:21:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:21:01 DEBUG --- http response --- 22:21:01 DEBUG "resultType": "vector", 22:21:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:21:01 DEBUG "resultType": "vector", 22:21:01 DEBUG { 22:21:01 DEBUG { 22:21:01 DEBUG "resultType": "vector", 22:21:01 DEBUG "result": [ 22:21:01 DEBUG ----- output ----- 22:21:01 DEBUG "result": [ 22:21:01 DEBUG "resultType": "vector", 22:21:01 DEBUG "resultType": "vector", 22:21:01 DEBUG ----- output ----- 22:21:01 DEBUG "result": [ 22:21:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:21:01 DEBUG ----- output ----- 22:21:01 DEBUG "result": [ 22:21:01 DEBUG "status": "success", 22:21:01 DEBUG "status": "success", 22:21:01 DEBUG "result": [ 22:21:01 DEBUG { 22:21:01 DEBUG { 22:21:01 DEBUG { 22:21:01 DEBUG "result": [ 22:21:01 DEBUG "result": [ 22:21:01 DEBUG { 22:21:01 DEBUG { 22:21:01 DEBUG ----- output ----- 22:21:01 DEBUG { 22:21:01 DEBUG { 22:21:01 DEBUG "data": { 22:21:01 DEBUG "data": { 22:21:01 DEBUG { 22:21:01 DEBUG "metric": { 22:21:01 DEBUG "status": "success", 22:21:01 DEBUG "metric": { 22:21:01 DEBUG { 22:21:01 DEBUG { 22:21:01 DEBUG "status": "success", 22:21:01 DEBUG "metric": { 22:21:01 DEBUG { 22:21:01 DEBUG "status": "success", 22:21:01 DEBUG "metric": { 22:21:01 DEBUG "resultType": "vector", 22:21:01 DEBUG "resultType": "vector", 22:21:01 DEBUG "metric": { 22:21:01 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:21:01 DEBUG "data": { 22:21:01 DEBUG "pod": "ds-idrepo-1" 22:21:01 DEBUG "metric": { 22:21:01 DEBUG "metric": { 22:21:01 DEBUG "data": { 22:21:01 DEBUG "instance": "10.142.0.107:9100" 22:21:01 DEBUG "status": "success", 22:21:01 DEBUG "data": { 22:21:01 DEBUG "pod": "am-767f6b67b5-9zw22" 22:21:01 DEBUG "result": [ 22:21:01 DEBUG "result": [ 22:21:01 DEBUG "pod": "ds-cts-0" 22:21:01 DEBUG }, 22:21:01 DEBUG "resultType": "vector", 22:21:01 DEBUG }, 22:21:01 DEBUG "pod": "ds-idrepo-0" 22:21:01 DEBUG "instance": "10.142.0.107:9100" 22:21:01 DEBUG "resultType": "vector", 22:21:01 DEBUG }, 22:21:01 DEBUG "data": { 22:21:01 DEBUG "resultType": "vector", 22:21:01 DEBUG }, 22:21:01 DEBUG { 22:21:01 DEBUG { 22:21:01 DEBUG }, 22:21:01 DEBUG "value": [ 22:21:01 DEBUG "result": [ 22:21:01 DEBUG "value": [ 22:21:01 DEBUG }, 22:21:01 DEBUG }, 22:21:01 DEBUG "result": [ 22:21:01 DEBUG "value": [ 22:21:01 DEBUG "resultType": "vector", 22:21:01 DEBUG "result": [ 22:21:01 DEBUG "value": [ 22:21:01 DEBUG "metric": { 22:21:01 DEBUG "metric": { 22:21:01 DEBUG "value": [ 22:21:01 DEBUG 22:21:01 DEBUG { 22:21:01 DEBUG 22:21:01 DEBUG 22:21:01 DEBUG "value": [ 22:21:01 DEBUG { 22:21:01 DEBUG 22:21:01 DEBUG "result": [ 22:21:01 DEBUG { 22:21:01 DEBUG 22:21:01 DEBUG "pod": "ds-idrepo-1" 22:21:01 DEBUG "pod": "ds-idrepo-0" 22:21:01 DEBUG }, 22:21:01 DEBUG 22:21:01 DEBUG "metric": { 22:21:01 DEBUG "value": [ 22:21:01 DEBUG 22:21:01 DEBUG 22:21:01 DEBUG "metric": { 22:21:01 DEBUG 22:21:01 DEBUG { 22:21:01 DEBUG "metric": { 22:21:01 DEBUG 22:21:01 DEBUG }, 22:21:01 DEBUG 22:21:01 DEBUG 22:21:01 DEBUG "value": [ 22:21:01 DEBUG "pod": "ds-cts-0" 22:21:01 DEBUG 22:21:01 DEBUG [...] 22:21:01 DEBUG 22:21:01 DEBUG 22:21:01 DEBUG 22:21:01 DEBUG "metric": { 22:21:01 DEBUG "pod": "am-767f6b67b5-9zw22" 22:21:01 DEBUG 22:21:01 DEBUG "value": [ 22:21:01 DEBUG 22:21:01 DEBUG 22:21:01 DEBUG 22:21:01 DEBUG }, 22:21:01 DEBUG 22:21:01 DEBUG 22:21:01 DEBUG [...] 22:21:01 DEBUG "pod": "ds-idrepo-0" 22:21:01 DEBUG [...] 22:21:01 DEBUG "instance": "10.142.0.107:9100" 22:21:01 DEBUG }, 22:21:01 DEBUG [...] 22:21:01 DEBUG 22:21:01 DEBUG [...] 22:21:01 DEBUG [...] 22:21:01 DEBUG 22:21:01 DEBUG "value": [ 22:21:01 DEBUG 22:21:01 DEBUG 22:21:01 DEBUG 22:21:01 DEBUG }, 22:21:01 DEBUG 22:21:01 DEBUG }, 22:21:01 DEBUG "value": [ 22:21:01 DEBUG 22:21:01 DEBUG 22:21:01 DEBUG 22:21:01 DEBUG 22:21:01 DEBUG 22:21:01 DEBUG 22:21:01 DEBUG [...] 22:21:01 DEBUG 22:21:01 DEBUG 22:21:01 DEBUG "value": [ 22:21:01 DEBUG 22:21:01 DEBUG "value": [ 22:21:01 DEBUG 22:21:01 DEBUG 22:21:01 DEBUG 22:21:01 DEBUG 22:21:01 DEBUG 22:21:01 DEBUG [...] 22:21:01 DEBUG 22:21:01 DEBUG 22:21:01 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 22:21:01 DEBUG 22:21:01 DEBUG 22:21:01 DEBUG 22:21:01 DEBUG 22:21:01 DEBUG 22:21:01 DEBUG 22:21:01 DEBUG "pod": "am-767f6b67b5-vbltz" 22:21:01 DEBUG 22:21:01 DEBUG 22:21:01 DEBUG 22:21:01 DEBUG 22:21:01 DEBUG [...] 22:21:01 DEBUG }, 22:21:01 DEBUG "pod": "ds-idrepo-2" 22:21:01 DEBUG 22:21:01 DEBUG "instance": "10.142.0.43:9100" 22:21:01 DEBUG 22:21:01 DEBUG 22:21:01 DEBUG [...] 22:21:01 DEBUG }, 22:21:01 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:21:01 DEBUG "pod": "ds-cts-2" 22:21:01 DEBUG 22:21:01 DEBUG 22:21:01 DEBUG 22:21:01 DEBUG "value": [ 22:21:01 DEBUG }, 22:21:01 DEBUG 22:21:01 DEBUG }, 22:21:01 DEBUG 22:21:01 DEBUG [...] 22:21:01 DEBUG 22:21:01 DEBUG "value": [ 22:21:01 DEBUG }, 22:21:01 DEBUG }, 22:21:01 DEBUG 22:21:01 DEBUG "instance": "10.142.0.43:9100" 22:21:01 DEBUG 22:21:01 DEBUG 1713648061, 22:21:01 DEBUG "value": [ 22:21:01 DEBUG [...] 22:21:01 DEBUG "value": [ 22:21:01 DEBUG 22:21:01 DEBUG [...] 22:21:01 DEBUG 22:21:01 DEBUG 1713648061, 22:21:01 DEBUG "value": [ 22:21:01 DEBUG "value": [ 22:21:01 DEBUG "pod": "ds-idrepo-2" 22:21:01 DEBUG }, 22:21:01 DEBUG 22:21:01 DEBUG "0.00001631701631698519" 22:21:01 DEBUG 1713648061, 22:21:01 DEBUG 22:21:01 DEBUG 1713648061, 22:21:01 DEBUG 22:21:01 DEBUG 22:21:01 DEBUG 22:21:01 DEBUG "139545" 22:21:01 DEBUG 1713648060, 22:21:01 DEBUG 1713648061, 22:21:01 DEBUG }, 22:21:01 DEBUG "value": [ 22:21:01 DEBUG 22:21:01 DEBUG ] 22:21:01 DEBUG "0" 22:21:01 DEBUG 22:21:01 DEBUG "4129408209920" 22:21:01 DEBUG ] 22:21:01 DEBUG 22:21:01 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 22:21:01 DEBUG ] 22:21:01 DEBUG "114896896" 22:21:01 DEBUG "23.355033333333335" 22:21:01 DEBUG "value": [ 22:21:01 DEBUG 1713648061, 22:21:01 DEBUG "pod": "ds-cts-2" 22:21:01 DEBUG } 22:21:01 DEBUG ] 22:21:01 DEBUG 22:21:01 DEBUG } 22:21:01 DEBUG 22:21:01 DEBUG 22:21:01 DEBUG }, 22:21:01 DEBUG } 22:21:01 DEBUG ] 22:21:01 DEBUG ] 22:21:01 DEBUG 1713648061, 22:21:01 DEBUG "17815202.866666667" 22:21:01 DEBUG }, 22:21:01 DEBUG } 22:21:01 DEBUG ] 22:21:01 DEBUG "pod": "ds-idrepo-2" 22:21:01 DEBUG ] 22:21:01 DEBUG "instance": "10.142.0.43:9100" 22:21:01 DEBUG "pod": "am-767f6b67b5-vbltz" 22:21:01 DEBUG "value": [ 22:21:01 DEBUG ] 22:21:01 DEBUG } 22:21:01 DEBUG } 22:21:01 DEBUG "0" 22:21:01 DEBUG ] 22:21:01 DEBUG "value": [ 22:21:01 DEBUG } 22:21:01 DEBUG } 22:21:01 DEBUG }, 22:21:01 DEBUG } 22:21:01 DEBUG }, 22:21:01 DEBUG }, 22:21:01 DEBUG 1713648061, 22:21:01 DEBUG } 22:21:01 DEBUG ] 22:21:01 DEBUG ] 22:21:01 DEBUG ] 22:21:01 DEBUG } 22:21:01 DEBUG 1713648061, 22:21:01 DEBUG ] 22:21:01 DEBUG "value": [ 22:21:01 DEBUG } 22:21:01 DEBUG "value": [ 22:21:01 DEBUG "value": [ 22:21:01 DEBUG "0" 22:21:01 DEBUG } 22:21:01 DEBUG } 22:21:01 DEBUG } 22:21:01 DEBUG } 22:21:01 DEBUG ] 22:21:01 DEBUG "0" 22:21:01 DEBUG } 22:21:01 DEBUG 1713648061, 22:21:01 DEBUG 1713648061, 22:21:01 DEBUG 1713648061, 22:21:01 DEBUG ] 22:21:01 DEBUG } 22:21:01 DEBUG } 22:21:01 DEBUG ] 22:21:01 DEBUG } 22:21:01 DEBUG ] 22:21:01 DEBUG } 22:21:01 DEBUG "0" 22:21:01 DEBUG "0.5283000000000915" 22:21:01 DEBUG "0" 22:21:01 DEBUG } 22:21:01 DEBUG } 22:21:01 DEBUG } 22:21:01 DEBUG } 22:21:01 DEBUG ] 22:21:01 DEBUG ] 22:21:01 DEBUG ] 22:21:01 DEBUG } 22:21:01 DEBUG } 22:21:01 DEBUG ] 22:21:01 DEBUG } 22:21:01 DEBUG } 22:21:01 DEBUG ] 22:21:01 DEBUG ] 22:21:01 DEBUG } 22:21:01 DEBUG ] 22:21:01 DEBUG ] 22:21:01 DEBUG } 22:21:01 DEBUG } 22:21:01 DEBUG } 22:21:01 DEBUG } 22:21:01 DEBUG } 22:21:01 DEBUG } 22:21:01 DEBUG } 22:21:01 DEBUG } 22:21:01 DEBUG } 22:21:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648115" 22:21:55 INFO [http_cmd]: http status code OK 22:21:55 DEBUG --- status code --- 22:21:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:55 DEBUG --- http response --- 22:21:55 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:21:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713648115" 22:21:55 ERROR [http_cmd]: ERROR 22:21:55 ERROR --- status code --- 22:21:55 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:21:55 ERROR --- http response --- 22:21:55 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:21:55 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:21:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648115" 22:21:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713648115" 22:21:55 INFO [http_cmd]: http status code OK 22:21:55 DEBUG --- status code --- 22:21:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:55 DEBUG --- http response --- 22:21:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:21:55 DEBUG ----- output ----- 22:21:55 DEBUG { 22:21:55 DEBUG "status": "success", 22:21:55 DEBUG "data": { 22:21:55 DEBUG "resultType": "vector", 22:21:55 DEBUG "result": [ 22:21:55 DEBUG { 22:21:55 DEBUG "metric": { 22:21:55 DEBUG "pod": "ds-idrepo-0" 22:21:55 DEBUG }, 22:21:55 DEBUG "value": [ 22:21:55 DEBUG 22:21:55 DEBUG 22:21:55 DEBUG 22:21:55 DEBUG [...] 22:21:55 DEBUG 22:21:55 DEBUG 22:21:55 DEBUG 22:21:55 DEBUG "pod": "ds-cts-2" 22:21:55 DEBUG }, 22:21:55 DEBUG "value": [ 22:21:55 DEBUG 1713648115, 22:21:55 DEBUG "0" 22:21:55 DEBUG ] 22:21:55 DEBUG } 22:21:55 DEBUG ] 22:21:55 DEBUG } 22:21:55 DEBUG } 22:21:55 INFO [http_cmd]: http status code OK 22:21:55 DEBUG --- status code --- 22:21:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:55 DEBUG --- http response --- 22:21:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:21:55 DEBUG ----- output ----- 22:21:55 DEBUG { 22:21:55 DEBUG "status": "success", 22:21:55 DEBUG "data": { 22:21:55 DEBUG "resultType": "vector", 22:21:55 DEBUG "result": [ 22:21:55 DEBUG { 22:21:55 DEBUG "metric": { 22:21:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:21:55 DEBUG }, 22:21:55 DEBUG "value": [ 22:21:55 DEBUG 22:21:55 DEBUG 22:21:55 DEBUG 22:21:55 DEBUG [...] 22:21:55 DEBUG 22:21:55 DEBUG 22:21:55 DEBUG 22:21:55 DEBUG "pod": "ds-cts-0" 22:21:55 DEBUG }, 22:21:55 DEBUG "value": [ 22:21:55 DEBUG 1713648115, 22:21:55 DEBUG "338675800.6380556" 22:21:55 DEBUG ] 22:21:55 DEBUG } 22:21:55 DEBUG ] 22:21:55 DEBUG } 22:21:55 DEBUG } 22:21:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648117" 22:21:57 INFO [http_cmd]: http status code OK 22:21:57 DEBUG --- status code --- 22:21:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:57 DEBUG --- http response --- 22:21:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:21:57 DEBUG ----- output ----- 22:21:57 DEBUG { 22:21:57 DEBUG "status": "success", 22:21:57 DEBUG "data": { 22:21:57 DEBUG "resultType": "vector", 22:21:57 DEBUG "result": [ 22:21:57 DEBUG { 22:21:57 DEBUG "metric": { 22:21:57 DEBUG "pod": "am-767f6b67b5-9zw22" 22:21:57 DEBUG }, 22:21:57 DEBUG "value": [ 22:21:57 DEBUG 22:21:57 DEBUG 22:21:57 DEBUG 22:21:57 DEBUG [...] 22:21:57 DEBUG 22:21:57 DEBUG 22:21:57 DEBUG 22:21:57 DEBUG "pod": "am-767f6b67b5-vbltz" 22:21:57 DEBUG }, 22:21:57 DEBUG "value": [ 22:21:57 DEBUG 1713648117, 22:21:57 DEBUG "0.03333333333333333" 22:21:57 DEBUG ] 22:21:57 DEBUG } 22:21:57 DEBUG ] 22:21:57 DEBUG } 22:21:57 DEBUG } 22:21:58 INFO 22:21:58 INFO [loop_until]: kubectl --namespace=xlou top pods 22:21:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:21:58 INFO [loop_until]: OK (rc = 0) 22:21:58 DEBUG --- stdout --- 22:21:58 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 1718m 8720Mi am-767f6b67b5-qcqz2 1700m 8838Mi am-767f6b67b5-vbltz 1816m 8855Mi ds-cts-0 18553m 12300Mi ds-cts-1 3986m 12257Mi ds-cts-2 19057m 12448Mi ds-idrepo-0 1074m 3942Mi ds-idrepo-1 886m 3983Mi ds-idrepo-2 803m 3878Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 6m 1179Mi idm-b85f9d978-wnhxb 5m 1239Mi lodemon-7dd48676f6-tdxl9 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-967d957f7-zk66v 1190m 381Mi 22:21:58 DEBUG --- stderr --- 22:21:58 DEBUG 22:21:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713648119" 22:21:59 INFO [http_cmd]: http status code OK 22:21:59 DEBUG --- status code --- 22:21:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:59 DEBUG --- http response --- 22:21:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:21:59 DEBUG ----- output ----- 22:21:59 DEBUG { 22:21:59 DEBUG "status": "success", 22:21:59 DEBUG "data": { 22:21:59 DEBUG "resultType": "vector", 22:21:59 DEBUG "result": [ 22:21:59 DEBUG { 22:21:59 DEBUG "metric": { 22:21:59 DEBUG "instance": "10.142.0.107:9100" 22:21:59 DEBUG }, 22:21:59 DEBUG "value": [ 22:21:59 DEBUG 22:21:59 DEBUG 22:21:59 DEBUG 22:21:59 DEBUG [...] 22:21:59 DEBUG 22:21:59 DEBUG 22:21:59 DEBUG 22:21:59 DEBUG "instance": "10.142.0.43:9100" 22:21:59 DEBUG }, 22:21:59 DEBUG "value": [ 22:21:59 DEBUG 1713648119, 22:21:59 DEBUG "14432595.666666666" 22:21:59 DEBUG ] 22:21:59 DEBUG } 22:21:59 DEBUG ] 22:21:59 DEBUG } 22:21:59 DEBUG } 22:22:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648120" 22:22:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713648120" 22:22:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713648120" 22:22:00 INFO [http_cmd]: http status code OK 22:22:00 DEBUG --- status code --- 22:22:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:00 DEBUG --- http response --- 22:22:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:22:00 DEBUG ----- output ----- 22:22:00 DEBUG { 22:22:00 DEBUG "status": "success", 22:22:00 DEBUG "data": { 22:22:00 DEBUG "resultType": "vector", 22:22:00 DEBUG "result": [ 22:22:00 DEBUG { 22:22:00 DEBUG "metric": { 22:22:00 DEBUG "pod": "am-767f6b67b5-9zw22" 22:22:00 DEBUG }, 22:22:00 DEBUG "value": [ 22:22:00 DEBUG 22:22:00 DEBUG 22:22:00 DEBUG 22:22:00 DEBUG [...] 22:22:00 DEBUG 22:22:00 DEBUG 22:22:00 DEBUG 22:22:00 DEBUG "pod": "am-767f6b67b5-vbltz" 22:22:00 DEBUG }, 22:22:00 DEBUG "value": [ 22:22:00 DEBUG 1713648120, 22:22:00 DEBUG "0.16666666666666666" 22:22:00 DEBUG ] 22:22:00 DEBUG } 22:22:00 DEBUG ] 22:22:00 DEBUG } 22:22:00 DEBUG } 22:22:00 INFO [http_cmd]: http status code OK 22:22:00 DEBUG --- status code --- 22:22:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:00 DEBUG --- http response --- 22:22:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:22:00 DEBUG ----- output ----- 22:22:00 DEBUG { 22:22:00 DEBUG "status": "success", 22:22:00 DEBUG "data": { 22:22:00 DEBUG "resultType": "vector", 22:22:00 DEBUG "result": [ 22:22:00 DEBUG { 22:22:00 DEBUG "metric": { 22:22:00 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:22:00 DEBUG }, 22:22:00 DEBUG "value": [ 22:22:00 DEBUG 22:22:00 DEBUG 22:22:00 DEBUG 22:22:00 DEBUG [...] 22:22:00 DEBUG 22:22:00 DEBUG 22:22:00 DEBUG 22:22:00 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:22:00 DEBUG }, 22:22:00 DEBUG "value": [ 22:22:00 DEBUG 1713648120, 22:22:00 DEBUG "0.0270855370640005" 22:22:00 DEBUG ] 22:22:00 DEBUG } 22:22:00 DEBUG ] 22:22:00 DEBUG } 22:22:00 DEBUG } 22:22:00 INFO [http_cmd]: http status code OK 22:22:00 DEBUG --- status code --- 22:22:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:00 DEBUG --- http response --- 22:22:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:22:00 DEBUG ----- output ----- 22:22:00 DEBUG { 22:22:00 DEBUG "status": "success", 22:22:00 DEBUG "data": { 22:22:00 DEBUG "resultType": "vector", 22:22:00 DEBUG "result": [ 22:22:00 DEBUG { 22:22:00 DEBUG "metric": { 22:22:00 DEBUG "pod": "am-767f6b67b5-9zw22" 22:22:00 DEBUG }, 22:22:00 DEBUG "value": [ 22:22:00 DEBUG 22:22:00 DEBUG 22:22:00 DEBUG 22:22:00 DEBUG [...] 22:22:00 DEBUG 22:22:00 DEBUG 22:22:00 DEBUG 22:22:00 DEBUG "pod": "am-767f6b67b5-vbltz" 22:22:00 DEBUG }, 22:22:00 DEBUG "value": [ 22:22:00 DEBUG 1713648120, 22:22:00 DEBUG "101.858" 22:22:00 DEBUG ] 22:22:00 DEBUG } 22:22:00 DEBUG ] 22:22:00 DEBUG } 22:22:00 DEBUG } 22:22:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648120" 22:22:00 INFO [http_cmd]: http status code OK 22:22:00 DEBUG --- status code --- 22:22:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:00 DEBUG --- http response --- 22:22:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:22:00 DEBUG ----- output ----- 22:22:00 DEBUG { 22:22:00 DEBUG "status": "success", 22:22:00 DEBUG "data": { 22:22:00 DEBUG "resultType": "vector", 22:22:00 DEBUG "result": [ 22:22:00 DEBUG { 22:22:00 DEBUG "metric": { 22:22:00 DEBUG "pod": "am-767f6b67b5-9zw22" 22:22:00 DEBUG }, 22:22:00 DEBUG "value": [ 22:22:00 DEBUG 22:22:00 DEBUG 22:22:00 DEBUG 22:22:00 DEBUG [...] 22:22:00 DEBUG 22:22:00 DEBUG 22:22:00 DEBUG 22:22:00 DEBUG "pod": "am-767f6b67b5-vbltz" 22:22:00 DEBUG }, 22:22:00 DEBUG "value": [ 22:22:00 DEBUG 1713648120, 22:22:00 DEBUG "1408.0666666666666" 22:22:00 DEBUG ] 22:22:00 DEBUG } 22:22:00 DEBUG ] 22:22:00 DEBUG } 22:22:00 DEBUG } 22:22:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713648120" 22:22:00 INFO [http_cmd]: http status code OK 22:22:00 DEBUG --- status code --- 22:22:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:00 DEBUG --- http response --- 22:22:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:22:00 DEBUG ----- output ----- 22:22:00 DEBUG { 22:22:00 DEBUG "status": "success", 22:22:00 DEBUG "data": { 22:22:00 DEBUG "resultType": "vector", 22:22:00 DEBUG "result": [ 22:22:00 DEBUG { 22:22:00 DEBUG "metric": { 22:22:00 DEBUG "instance": "10.142.0.107:9100" 22:22:00 DEBUG }, 22:22:00 DEBUG "value": [ 22:22:00 DEBUG 22:22:00 DEBUG 22:22:00 DEBUG 22:22:00 DEBUG [...] 22:22:00 DEBUG 22:22:00 DEBUG 22:22:00 DEBUG 22:22:00 DEBUG "instance": "10.142.0.43:9100" 22:22:00 DEBUG }, 22:22:00 DEBUG "value": [ 22:22:00 DEBUG 1713648120, 22:22:00 DEBUG "0.011666666666667424" 22:22:00 DEBUG ] 22:22:00 DEBUG } 22:22:00 DEBUG ] 22:22:00 DEBUG } 22:22:00 DEBUG } 22:22:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648121" 22:22:01 INFO 22:22:01 INFO [loop_until]: kubectl --namespace=xlou top node 22:22:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:22:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648121" 22:22:01 INFO [http_cmd]: http status code OK 22:22:01 DEBUG --- status code --- 22:22:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:01 DEBUG --- http response --- 22:22:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:22:01 DEBUG ----- output ----- 22:22:01 DEBUG { 22:22:01 DEBUG "status": "success", 22:22:01 DEBUG "data": { 22:22:01 DEBUG "resultType": "vector", 22:22:01 DEBUG "result": [ 22:22:01 DEBUG { 22:22:01 DEBUG "metric": { 22:22:01 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:22:01 DEBUG }, 22:22:01 DEBUG "value": [ 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG [...] 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG "pod": "idm-b85f9d978-txpwv" 22:22:01 DEBUG }, 22:22:01 DEBUG "value": [ 22:22:01 DEBUG 1713648121, 22:22:01 DEBUG "0" 22:22:01 DEBUG ] 22:22:01 DEBUG } 22:22:01 DEBUG ] 22:22:01 DEBUG } 22:22:01 DEBUG } 22:22:01 INFO [http_cmd]: http status code OK 22:22:01 DEBUG --- status code --- 22:22:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:01 DEBUG --- http response --- 22:22:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:22:01 DEBUG ----- output ----- 22:22:01 DEBUG { 22:22:01 DEBUG "status": "success", 22:22:01 DEBUG "data": { 22:22:01 DEBUG "resultType": "vector", 22:22:01 DEBUG "result": [ 22:22:01 DEBUG { 22:22:01 DEBUG "metric": { 22:22:01 DEBUG "pod": "ds-cts-0" 22:22:01 DEBUG }, 22:22:01 DEBUG "value": [ 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG [...] 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG "pod": "ds-cts-2" 22:22:01 DEBUG }, 22:22:01 DEBUG "value": [ 22:22:01 DEBUG 1713648121, 22:22:01 DEBUG "30" 22:22:01 DEBUG ] 22:22:01 DEBUG } 22:22:01 DEBUG ] 22:22:01 DEBUG } 22:22:01 DEBUG } 22:22:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648121" 22:22:01 INFO [http_cmd]: http status code OK 22:22:01 DEBUG --- status code --- 22:22:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:01 DEBUG --- http response --- 22:22:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:22:01 DEBUG ----- output ----- 22:22:01 DEBUG { 22:22:01 DEBUG "status": "success", 22:22:01 DEBUG "data": { 22:22:01 DEBUG "resultType": "vector", 22:22:01 DEBUG "result": [ 22:22:01 DEBUG { 22:22:01 DEBUG "metric": { 22:22:01 DEBUG "pod": "am-767f6b67b5-9zw22" 22:22:01 DEBUG }, 22:22:01 DEBUG "value": [ 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG [...] 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG "pod": "am-767f6b67b5-vbltz" 22:22:01 DEBUG }, 22:22:01 DEBUG "value": [ 22:22:01 DEBUG 1713648121, 22:22:01 DEBUG "0" 22:22:01 DEBUG ] 22:22:01 DEBUG } 22:22:01 DEBUG ] 22:22:01 DEBUG } 22:22:01 DEBUG } 22:22:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648121" 22:22:01 INFO [http_cmd]: http status code OK 22:22:01 DEBUG --- status code --- 22:22:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:01 DEBUG --- http response --- 22:22:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:22:01 DEBUG ----- output ----- 22:22:01 DEBUG { 22:22:01 DEBUG "status": "success", 22:22:01 DEBUG "data": { 22:22:01 DEBUG "resultType": "vector", 22:22:01 DEBUG "result": [ 22:22:01 DEBUG { 22:22:01 DEBUG "metric": { 22:22:01 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:22:01 DEBUG }, 22:22:01 DEBUG "value": [ 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG [...] 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 22:22:01 DEBUG }, 22:22:01 DEBUG "value": [ 22:22:01 DEBUG 1713648121, 22:22:01 DEBUG "0" 22:22:01 DEBUG ] 22:22:01 DEBUG } 22:22:01 DEBUG ] 22:22:01 DEBUG } 22:22:01 DEBUG } 22:22:01 INFO [loop_until]: OK (rc = 0) 22:22:01 DEBUG --- stdout --- 22:22:01 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 66m 0% 6220Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 1190m 3% 7759Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 1858m 6% 11479Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 91m 0% 3406Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 18182m 60% 15492Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 142m 0% 5175Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 1535m 5% 11751Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 77m 0% 3270Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 1984m 6% 11141Mi 9% gke-xlou-cdm-ds-0f601751-j488 948m 3% 7241Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 4383m 14% 17373Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 993m 3% 7935Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 18790m 62% 15152Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 1405m 8% 8336Mi 14% 22:22:01 DEBUG --- stderr --- 22:22:01 DEBUG 22:22:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713648121" 22:22:01 INFO [http_cmd]: http status code OK 22:22:01 DEBUG --- status code --- 22:22:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:01 DEBUG --- http response --- 22:22:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:22:01 DEBUG ----- output ----- 22:22:01 DEBUG { 22:22:01 DEBUG "status": "success", 22:22:01 DEBUG "data": { 22:22:01 DEBUG "resultType": "vector", 22:22:01 DEBUG "result": [ 22:22:01 DEBUG { 22:22:01 DEBUG "metric": { 22:22:01 DEBUG "instance": "10.142.0.107:9100" 22:22:01 DEBUG }, 22:22:01 DEBUG "value": [ 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG [...] 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG "instance": "10.142.0.43:9100" 22:22:01 DEBUG }, 22:22:01 DEBUG "value": [ 22:22:01 DEBUG 1713648121, 22:22:01 DEBUG "4131064902656" 22:22:01 DEBUG ] 22:22:01 DEBUG } 22:22:01 DEBUG ] 22:22:01 DEBUG } 22:22:01 DEBUG } 22:22:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648121" 22:22:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648121" 22:22:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648121" 22:22:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648121" 22:22:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713648121" 22:22:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713648121" 22:22:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713648121" 22:22:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648121" 22:22:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648121" 22:22:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648121" 22:22:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648121" 22:22:01 INFO [http_cmd]: http status code OK 22:22:01 INFO [http_cmd]: http status code OK 22:22:01 INFO [http_cmd]: http status code OK 22:22:01 DEBUG --- status code --- 22:22:01 DEBUG --- status code --- 22:22:01 INFO [http_cmd]: http status code OK 22:22:01 INFO [http_cmd]: http status code OK 22:22:01 INFO [http_cmd]: http status code OK 22:22:01 INFO [http_cmd]: http status code OK 22:22:01 DEBUG --- status code --- 22:22:01 INFO [http_cmd]: http status code OK 22:22:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:01 DEBUG --- status code --- 22:22:01 DEBUG --- status code --- 22:22:01 DEBUG --- status code --- 22:22:01 DEBUG --- status code --- 22:22:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:01 DEBUG --- status code --- 22:22:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:01 DEBUG --- http response --- 22:22:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:01 INFO [http_cmd]: http status code OK 22:22:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:01 DEBUG --- http response --- 22:22:01 DEBUG --- http response --- 22:22:01 DEBUG --- http response --- 22:22:01 DEBUG --- http response --- 22:22:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:22:01 DEBUG --- http response --- 22:22:01 DEBUG --- status code --- 22:22:01 DEBUG --- http response --- 22:22:01 DEBUG --- http response --- 22:22:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:22:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:22:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:22:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:22:01 DEBUG ----- output ----- 22:22:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:22:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:22:01 DEBUG ----- output ----- 22:22:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:22:01 DEBUG ----- output ----- 22:22:01 DEBUG ----- output ----- 22:22:01 DEBUG ----- output ----- 22:22:01 DEBUG { 22:22:01 DEBUG --- http response --- 22:22:01 DEBUG ----- output ----- 22:22:01 DEBUG ----- output ----- 22:22:01 DEBUG { 22:22:01 INFO [http_cmd]: http status code OK 22:22:01 DEBUG ----- output ----- 22:22:01 DEBUG { 22:22:01 DEBUG { 22:22:01 DEBUG { 22:22:01 DEBUG "status": "success", 22:22:01 INFO [http_cmd]: http status code OK 22:22:01 DEBUG { 22:22:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:22:01 DEBUG { 22:22:01 DEBUG "status": "success", 22:22:01 DEBUG --- status code --- 22:22:01 DEBUG { 22:22:01 DEBUG "status": "success", 22:22:01 DEBUG "status": "success", 22:22:01 DEBUG "status": "success", 22:22:01 DEBUG "data": { 22:22:01 DEBUG --- status code --- 22:22:01 DEBUG "status": "success", 22:22:01 DEBUG ----- output ----- 22:22:01 DEBUG "status": "success", 22:22:01 DEBUG "data": { 22:22:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:01 DEBUG "status": "success", 22:22:01 DEBUG "data": { 22:22:01 DEBUG "data": { 22:22:01 DEBUG "data": { 22:22:01 DEBUG "resultType": "vector", 22:22:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:01 DEBUG "data": { 22:22:01 DEBUG { 22:22:01 DEBUG "data": { 22:22:01 DEBUG "resultType": "vector", 22:22:01 DEBUG --- http response --- 22:22:01 DEBUG "data": { 22:22:01 DEBUG "resultType": "vector", 22:22:01 DEBUG "resultType": "vector", 22:22:01 DEBUG "resultType": "vector", 22:22:01 DEBUG "result": [ 22:22:01 DEBUG --- http response --- 22:22:01 DEBUG "resultType": "vector", 22:22:01 DEBUG "status": "success", 22:22:01 DEBUG "resultType": "vector", 22:22:01 DEBUG "result": [ 22:22:01 DEBUG "resultType": "vector", 22:22:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:22:01 DEBUG "result": [ 22:22:01 DEBUG "result": [ 22:22:01 DEBUG "result": [ 22:22:01 DEBUG { 22:22:01 DEBUG "result": [ 22:22:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:22:01 DEBUG "data": { 22:22:01 DEBUG "result": [ 22:22:01 DEBUG { 22:22:01 DEBUG "result": [ 22:22:01 DEBUG ----- output ----- 22:22:01 DEBUG { 22:22:01 DEBUG { 22:22:01 DEBUG { 22:22:01 DEBUG "metric": { 22:22:01 DEBUG { 22:22:01 DEBUG ----- output ----- 22:22:01 DEBUG "resultType": "vector", 22:22:01 DEBUG { 22:22:01 DEBUG "metric": { 22:22:01 DEBUG { 22:22:01 DEBUG { 22:22:01 DEBUG "metric": { 22:22:01 DEBUG "metric": { 22:22:01 DEBUG "metric": { 22:22:01 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:22:01 DEBUG "metric": { 22:22:01 DEBUG { 22:22:01 DEBUG "result": [ 22:22:01 DEBUG "metric": { 22:22:01 DEBUG "pod": "am-767f6b67b5-9zw22" 22:22:01 DEBUG "metric": { 22:22:01 DEBUG "status": "success", 22:22:01 DEBUG "instance": "10.142.0.107:9100" 22:22:01 DEBUG "instance": "10.142.0.107:9100" 22:22:01 DEBUG "pod": "ds-idrepo-0" 22:22:01 DEBUG }, 22:22:01 DEBUG "pod": "ds-cts-0" 22:22:01 DEBUG "status": "success", 22:22:01 DEBUG { 22:22:01 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:22:01 DEBUG }, 22:22:01 DEBUG "pod": "ds-idrepo-0" 22:22:01 DEBUG "data": { 22:22:01 DEBUG }, 22:22:01 DEBUG }, 22:22:01 DEBUG }, 22:22:01 DEBUG "value": [ 22:22:01 DEBUG }, 22:22:01 DEBUG "data": { 22:22:01 DEBUG "metric": { 22:22:01 DEBUG }, 22:22:01 DEBUG "value": [ 22:22:01 DEBUG }, 22:22:01 DEBUG "resultType": "vector", 22:22:01 DEBUG "value": [ 22:22:01 DEBUG "value": [ 22:22:01 DEBUG "value": [ 22:22:01 DEBUG 22:22:01 DEBUG "value": [ 22:22:01 DEBUG "resultType": "vector", 22:22:01 DEBUG "pod": "ds-cts-0" 22:22:01 DEBUG "value": [ 22:22:01 DEBUG 22:22:01 DEBUG "value": [ 22:22:01 DEBUG "result": [ 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG "result": [ 22:22:01 DEBUG }, 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG { 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG { 22:22:01 DEBUG "value": [ 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG "metric": { 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG [...] 22:22:01 DEBUG 22:22:01 DEBUG "metric": { 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG [...] 22:22:01 DEBUG 22:22:01 DEBUG "pod": "ds-idrepo-0" 22:22:01 DEBUG [...] 22:22:01 DEBUG [...] 22:22:01 DEBUG [...] 22:22:01 DEBUG 22:22:01 DEBUG [...] 22:22:01 DEBUG "pod": "am-767f6b67b5-9zw22" 22:22:01 DEBUG 22:22:01 DEBUG [...] 22:22:01 DEBUG 22:22:01 DEBUG [...] 22:22:01 DEBUG }, 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG }, 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG "value": [ 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG "value": [ 22:22:01 DEBUG [...] 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG "pod": "idm-b85f9d978-txpwv" 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG "pod": "am-767f6b67b5-vbltz" 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG "instance": "10.142.0.43:9100" 22:22:01 DEBUG "instance": "10.142.0.43:9100" 22:22:01 DEBUG "pod": "ds-idrepo-2" 22:22:01 DEBUG }, 22:22:01 DEBUG "pod": "ds-cts-2" 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:22:01 DEBUG }, 22:22:01 DEBUG "pod": "ds-idrepo-2" 22:22:01 DEBUG 22:22:01 DEBUG }, 22:22:01 DEBUG }, 22:22:01 DEBUG }, 22:22:01 DEBUG "value": [ 22:22:01 DEBUG }, 22:22:01 DEBUG 22:22:01 DEBUG 22:22:01 DEBUG }, 22:22:01 DEBUG "value": [ 22:22:01 DEBUG }, 22:22:01 DEBUG [...] 22:22:01 DEBUG "value": [ 22:22:01 DEBUG "value": [ 22:22:01 DEBUG "value": [ 22:22:01 DEBUG 1713648121, 22:22:01 DEBUG "value": [ 22:22:01 DEBUG [...] 22:22:01 DEBUG "pod": "ds-cts-2" 22:22:01 DEBUG "value": [ 22:22:01 DEBUG 1713648121, 22:22:01 DEBUG "value": [ 22:22:01 DEBUG 22:22:01 DEBUG 1713648121, 22:22:01 DEBUG 1713648121, 22:22:01 DEBUG 1713648121, 22:22:01 DEBUG "0" 22:22:01 DEBUG 1713648121, 22:22:01 DEBUG 22:22:01 DEBUG }, 22:22:01 DEBUG 1713648121, 22:22:01 DEBUG "21292" 22:22:01 DEBUG 1713648121, 22:22:01 DEBUG 22:22:01 DEBUG "0.016300000000038988" 22:22:01 DEBUG "14439741.433333334" 22:22:01 DEBUG "0.06666666666666667" 22:22:01 DEBUG ] 22:22:01 DEBUG "19.035866666666667" 22:22:01 DEBUG 22:22:01 DEBUG "value": [ 22:22:01 DEBUG "116457472" 22:22:01 DEBUG ] 22:22:01 DEBUG "0.06666666666666667" 22:22:01 DEBUG 22:22:01 DEBUG ] 22:22:01 DEBUG ] 22:22:01 DEBUG ] 22:22:01 DEBUG } 22:22:01 DEBUG ] 22:22:01 DEBUG 22:22:01 DEBUG 1713648121, 22:22:01 DEBUG ] 22:22:01 DEBUG } 22:22:01 DEBUG ] 22:22:01 DEBUG "pod": "ds-idrepo-2" 22:22:01 DEBUG } 22:22:01 DEBUG } 22:22:01 DEBUG } 22:22:01 DEBUG ] 22:22:01 DEBUG } 22:22:01 DEBUG "pod": "am-767f6b67b5-vbltz" 22:22:01 DEBUG "0" 22:22:01 DEBUG } 22:22:01 DEBUG ] 22:22:01 DEBUG } 22:22:01 DEBUG }, 22:22:01 DEBUG ] 22:22:01 DEBUG ] 22:22:01 DEBUG ] 22:22:01 DEBUG } 22:22:01 DEBUG ] 22:22:01 DEBUG }, 22:22:01 DEBUG ] 22:22:01 DEBUG ] 22:22:01 DEBUG } 22:22:01 DEBUG ] 22:22:01 DEBUG "value": [ 22:22:01 DEBUG } 22:22:01 DEBUG } 22:22:01 DEBUG } 22:22:01 DEBUG } 22:22:01 DEBUG } 22:22:01 DEBUG "value": [ 22:22:01 DEBUG } 22:22:01 DEBUG } 22:22:01 DEBUG } 22:22:01 DEBUG } 22:22:01 DEBUG 1713648121, 22:22:01 DEBUG } 22:22:01 DEBUG } 22:22:01 DEBUG } 22:22:01 DEBUG } 22:22:01 DEBUG 1713648121, 22:22:01 DEBUG ] 22:22:01 DEBUG } 22:22:01 DEBUG } 22:22:01 DEBUG "0.06666666666666667" 22:22:01 DEBUG "0.03333333333333333" 22:22:01 DEBUG } 22:22:01 DEBUG ] 22:22:01 DEBUG ] 22:22:01 DEBUG } 22:22:01 DEBUG } 22:22:01 DEBUG } 22:22:01 DEBUG ] 22:22:01 DEBUG ] 22:22:01 DEBUG } 22:22:01 DEBUG } 22:22:01 DEBUG } 22:22:01 DEBUG } 22:22:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648175" 22:22:55 INFO [http_cmd]: http status code OK 22:22:55 DEBUG --- status code --- 22:22:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:55 DEBUG --- http response --- 22:22:55 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:22:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713648175" 22:22:55 ERROR [http_cmd]: ERROR 22:22:55 ERROR --- status code --- 22:22:55 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:22:55 ERROR --- http response --- 22:22:55 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:22:55 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:22:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713648175" 22:22:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648175" 22:22:56 INFO [http_cmd]: http status code OK 22:22:56 DEBUG --- status code --- 22:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:56 DEBUG --- http response --- 22:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:22:56 DEBUG ----- output ----- 22:22:56 DEBUG { 22:22:56 DEBUG "status": "success", 22:22:56 DEBUG "data": { 22:22:56 DEBUG "resultType": "vector", 22:22:56 DEBUG "result": [ 22:22:56 DEBUG { 22:22:56 DEBUG "metric": { 22:22:56 DEBUG "pod": "ds-idrepo-0" 22:22:56 DEBUG }, 22:22:56 DEBUG "value": [ 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG [...] 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG "pod": "ds-cts-2" 22:22:56 DEBUG }, 22:22:56 DEBUG "value": [ 22:22:56 DEBUG 1713648175, 22:22:56 DEBUG "0" 22:22:56 DEBUG ] 22:22:56 DEBUG } 22:22:56 DEBUG ] 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 INFO [http_cmd]: http status code OK 22:22:56 DEBUG --- status code --- 22:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:56 DEBUG --- http response --- 22:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 22:22:56 DEBUG ----- output ----- 22:22:56 DEBUG { 22:22:56 DEBUG "status": "success", 22:22:56 DEBUG "data": { 22:22:56 DEBUG "resultType": "vector", 22:22:56 DEBUG "result": [ 22:22:56 DEBUG { 22:22:56 DEBUG "metric": { 22:22:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:22:56 DEBUG }, 22:22:56 DEBUG "value": [ 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG [...] 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG "pod": "overseer-0-967d957f7-zk66v" 22:22:56 DEBUG }, 22:22:56 DEBUG "value": [ 22:22:56 DEBUG 1713648175, 22:22:56 DEBUG "0" 22:22:56 DEBUG ] 22:22:56 DEBUG } 22:22:56 DEBUG ] 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648177" 22:22:57 INFO [http_cmd]: http status code OK 22:22:57 DEBUG --- status code --- 22:22:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:57 DEBUG --- http response --- 22:22:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:22:57 DEBUG ----- output ----- 22:22:57 DEBUG { 22:22:57 DEBUG "status": "success", 22:22:57 DEBUG "data": { 22:22:57 DEBUG "resultType": "vector", 22:22:57 DEBUG "result": [ 22:22:57 DEBUG { 22:22:57 DEBUG "metric": { 22:22:57 DEBUG "pod": "am-767f6b67b5-9zw22" 22:22:57 DEBUG }, 22:22:57 DEBUG "value": [ 22:22:57 DEBUG 22:22:57 DEBUG 22:22:57 DEBUG 22:22:57 DEBUG [...] 22:22:57 DEBUG 22:22:57 DEBUG 22:22:57 DEBUG 22:22:57 DEBUG "pod": "am-767f6b67b5-vbltz" 22:22:57 DEBUG }, 22:22:57 DEBUG "value": [ 22:22:57 DEBUG 1713648177, 22:22:57 DEBUG "0" 22:22:57 DEBUG ] 22:22:57 DEBUG } 22:22:57 DEBUG ] 22:22:57 DEBUG } 22:22:57 DEBUG } 22:22:58 INFO 22:22:58 INFO [loop_until]: kubectl --namespace=xlou top pods 22:22:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:22:58 INFO [loop_until]: OK (rc = 0) 22:22:58 DEBUG --- stdout --- 22:22:58 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 642m 8707Mi am-767f6b67b5-qcqz2 616m 8824Mi am-767f6b67b5-vbltz 499m 8846Mi ds-cts-0 17991m 10509Mi ds-cts-1 17m 12257Mi ds-cts-2 17730m 12437Mi ds-idrepo-0 107m 3958Mi ds-idrepo-1 13m 3990Mi ds-idrepo-2 35m 3895Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1179Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-7dd48676f6-tdxl9 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-967d957f7-zk66v 1006m 1265Mi 22:22:58 DEBUG --- stderr --- 22:22:58 DEBUG 22:23:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713648180" 22:23:00 INFO [http_cmd]: http status code OK 22:23:00 DEBUG --- status code --- 22:23:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:00 DEBUG --- http response --- 22:23:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:23:00 DEBUG ----- output ----- 22:23:00 DEBUG { 22:23:00 DEBUG "status": "success", 22:23:00 DEBUG "data": { 22:23:00 DEBUG "resultType": "vector", 22:23:00 DEBUG "result": [ 22:23:00 DEBUG { 22:23:00 DEBUG "metric": { 22:23:00 DEBUG "instance": "10.142.0.107:9100" 22:23:00 DEBUG }, 22:23:00 DEBUG "value": [ 22:23:00 DEBUG 22:23:00 DEBUG 22:23:00 DEBUG 22:23:00 DEBUG [...] 22:23:00 DEBUG 22:23:00 DEBUG 22:23:00 DEBUG 22:23:00 DEBUG "instance": "10.142.0.43:9100" 22:23:00 DEBUG }, 22:23:00 DEBUG "value": [ 22:23:00 DEBUG 1713648180, 22:23:00 DEBUG "29618.13333333333" 22:23:00 DEBUG ] 22:23:00 DEBUG } 22:23:00 DEBUG ] 22:23:00 DEBUG } 22:23:00 DEBUG } 22:23:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648180" 22:23:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648180" 22:23:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713648180" 22:23:00 INFO [http_cmd]: http status code OK 22:23:00 DEBUG --- status code --- 22:23:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:00 DEBUG --- http response --- 22:23:00 INFO [http_cmd]: http status code OK 22:23:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713648180" 22:23:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:23:00 DEBUG --- status code --- 22:23:00 DEBUG ----- output ----- 22:23:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:00 DEBUG { 22:23:00 DEBUG --- http response --- 22:23:00 DEBUG "status": "success", 22:23:00 DEBUG "data": { 22:23:00 DEBUG "resultType": "vector", 22:23:00 DEBUG "result": [ 22:23:00 DEBUG { 22:23:00 DEBUG "metric": { 22:23:00 DEBUG "pod": "am-767f6b67b5-9zw22" 22:23:00 DEBUG }, 22:23:00 DEBUG "value": [ 22:23:00 DEBUG 22:23:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:23:00 DEBUG 22:23:00 DEBUG ----- output ----- 22:23:00 DEBUG 22:23:00 DEBUG { 22:23:00 DEBUG [...] 22:23:00 DEBUG "status": "success", 22:23:00 DEBUG 22:23:00 DEBUG "data": { 22:23:00 DEBUG 22:23:00 DEBUG "resultType": "vector", 22:23:00 DEBUG 22:23:00 DEBUG "result": [ 22:23:00 DEBUG "pod": "am-767f6b67b5-vbltz" 22:23:00 DEBUG { 22:23:00 DEBUG }, 22:23:00 DEBUG "metric": { 22:23:00 DEBUG "value": [ 22:23:00 DEBUG "pod": "am-767f6b67b5-9zw22" 22:23:00 DEBUG 1713648180, 22:23:00 DEBUG }, 22:23:00 DEBUG "0.06666666666666667" 22:23:00 DEBUG "value": [ 22:23:00 DEBUG ] 22:23:00 DEBUG 22:23:00 DEBUG } 22:23:00 DEBUG 22:23:00 DEBUG ] 22:23:00 DEBUG 22:23:00 DEBUG } 22:23:00 DEBUG [...] 22:23:00 DEBUG } 22:23:00 DEBUG 22:23:00 DEBUG 22:23:00 DEBUG 22:23:00 DEBUG "pod": "am-767f6b67b5-vbltz" 22:23:00 DEBUG }, 22:23:00 DEBUG "value": [ 22:23:00 DEBUG 1713648180, 22:23:00 DEBUG "0" 22:23:00 DEBUG ] 22:23:00 DEBUG } 22:23:00 DEBUG ] 22:23:00 DEBUG } 22:23:00 DEBUG } 22:23:00 INFO [http_cmd]: http status code OK 22:23:00 DEBUG --- status code --- 22:23:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:00 DEBUG --- http response --- 22:23:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:23:00 DEBUG ----- output ----- 22:23:00 DEBUG { 22:23:00 DEBUG "status": "success", 22:23:00 DEBUG "data": { 22:23:00 DEBUG "resultType": "vector", 22:23:00 DEBUG "result": [ 22:23:00 DEBUG { 22:23:00 DEBUG "metric": { 22:23:00 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:23:00 DEBUG }, 22:23:00 DEBUG "value": [ 22:23:00 DEBUG 22:23:00 DEBUG 22:23:00 DEBUG 22:23:00 DEBUG [...] 22:23:00 DEBUG 22:23:00 DEBUG 22:23:00 DEBUG 22:23:00 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:23:00 DEBUG }, 22:23:00 DEBUG "value": [ 22:23:00 DEBUG 1713648180, 22:23:00 DEBUG "1.0427543736813423" 22:23:00 DEBUG ] 22:23:00 DEBUG } 22:23:00 DEBUG ] 22:23:00 DEBUG } 22:23:00 DEBUG } 22:23:00 INFO [http_cmd]: http status code OK 22:23:00 DEBUG --- status code --- 22:23:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:00 DEBUG --- http response --- 22:23:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:23:00 DEBUG ----- output ----- 22:23:00 DEBUG { 22:23:00 DEBUG "status": "success", 22:23:00 DEBUG "data": { 22:23:00 DEBUG "resultType": "vector", 22:23:00 DEBUG "result": [ 22:23:00 DEBUG { 22:23:00 DEBUG "metric": { 22:23:00 DEBUG "pod": "am-767f6b67b5-9zw22" 22:23:00 DEBUG }, 22:23:00 DEBUG "value": [ 22:23:00 DEBUG 22:23:00 DEBUG 22:23:00 DEBUG 22:23:00 DEBUG [...] 22:23:00 DEBUG 22:23:00 DEBUG 22:23:00 DEBUG 22:23:00 DEBUG "pod": "am-767f6b67b5-vbltz" 22:23:00 DEBUG }, 22:23:00 DEBUG "value": [ 22:23:00 DEBUG 1713648180, 22:23:00 DEBUG "101.872" 22:23:00 DEBUG ] 22:23:00 DEBUG } 22:23:00 DEBUG ] 22:23:00 DEBUG } 22:23:00 DEBUG } 22:23:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713648180" 22:23:00 INFO [http_cmd]: http status code OK 22:23:00 DEBUG --- status code --- 22:23:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:00 DEBUG --- http response --- 22:23:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:23:00 DEBUG ----- output ----- 22:23:00 DEBUG { 22:23:00 DEBUG "status": "success", 22:23:00 DEBUG "data": { 22:23:00 DEBUG "resultType": "vector", 22:23:00 DEBUG "result": [ 22:23:00 DEBUG { 22:23:00 DEBUG "metric": { 22:23:00 DEBUG "instance": "10.142.0.107:9100" 22:23:00 DEBUG }, 22:23:00 DEBUG "value": [ 22:23:00 DEBUG 22:23:00 DEBUG 22:23:00 DEBUG 22:23:00 DEBUG [...] 22:23:00 DEBUG 22:23:00 DEBUG 22:23:00 DEBUG 22:23:00 DEBUG "instance": "10.142.0.43:9100" 22:23:00 DEBUG }, 22:23:00 DEBUG "value": [ 22:23:00 DEBUG 1713648180, 22:23:00 DEBUG "0.001999999999998181" 22:23:00 DEBUG ] 22:23:00 DEBUG } 22:23:00 DEBUG ] 22:23:00 DEBUG } 22:23:00 DEBUG } 22:23:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648181" 22:23:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648181" 22:23:01 INFO [http_cmd]: http status code OK 22:23:01 DEBUG --- status code --- 22:23:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:01 DEBUG --- http response --- 22:23:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:23:01 DEBUG ----- output ----- 22:23:01 DEBUG { 22:23:01 DEBUG "status": "success", 22:23:01 DEBUG "data": { 22:23:01 DEBUG "resultType": "vector", 22:23:01 DEBUG "result": [ 22:23:01 DEBUG { 22:23:01 DEBUG "metric": { 22:23:01 DEBUG "pod": "ds-cts-0" 22:23:01 DEBUG }, 22:23:01 DEBUG "value": [ 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG [...] 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG "pod": "ds-cts-2" 22:23:01 DEBUG }, 22:23:01 DEBUG "value": [ 22:23:01 DEBUG 1713648181, 22:23:01 DEBUG "29.866666666666667" 22:23:01 DEBUG ] 22:23:01 DEBUG } 22:23:01 DEBUG ] 22:23:01 DEBUG } 22:23:01 DEBUG } 22:23:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648181" 22:23:01 INFO [http_cmd]: http status code OK 22:23:01 DEBUG --- status code --- 22:23:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:01 DEBUG --- http response --- 22:23:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:23:01 DEBUG ----- output ----- 22:23:01 DEBUG { 22:23:01 DEBUG "status": "success", 22:23:01 DEBUG "data": { 22:23:01 DEBUG "resultType": "vector", 22:23:01 DEBUG "result": [ 22:23:01 DEBUG { 22:23:01 DEBUG "metric": { 22:23:01 DEBUG "pod": "ds-idrepo-1" 22:23:01 DEBUG }, 22:23:01 DEBUG "value": [ 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG [...] 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG "pod": "idm-b85f9d978-txpwv" 22:23:01 DEBUG }, 22:23:01 DEBUG "value": [ 22:23:01 DEBUG 1713648181, 22:23:01 DEBUG "0.003321286157349426" 22:23:01 DEBUG ] 22:23:01 DEBUG } 22:23:01 DEBUG ] 22:23:01 DEBUG } 22:23:01 DEBUG } 22:23:01 INFO [http_cmd]: http status code OK 22:23:01 DEBUG --- status code --- 22:23:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:01 DEBUG --- http response --- 22:23:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:23:01 DEBUG ----- output ----- 22:23:01 DEBUG { 22:23:01 DEBUG "status": "success", 22:23:01 DEBUG "data": { 22:23:01 DEBUG "resultType": "vector", 22:23:01 DEBUG "result": [ 22:23:01 DEBUG { 22:23:01 DEBUG "metric": { 22:23:01 DEBUG "pod": "ds-idrepo-1" 22:23:01 DEBUG }, 22:23:01 DEBUG "value": [ 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG [...] 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG "pod": "idm-b85f9d978-txpwv" 22:23:01 DEBUG }, 22:23:01 DEBUG "value": [ 22:23:01 DEBUG 1713648181, 22:23:01 DEBUG "388.8853724919299" 22:23:01 DEBUG ] 22:23:01 DEBUG } 22:23:01 DEBUG ] 22:23:01 DEBUG } 22:23:01 DEBUG } 22:23:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648181" 22:23:01 INFO [http_cmd]: http status code OK 22:23:01 DEBUG --- status code --- 22:23:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:01 DEBUG --- http response --- 22:23:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:23:01 DEBUG ----- output ----- 22:23:01 DEBUG { 22:23:01 DEBUG "status": "success", 22:23:01 DEBUG "data": { 22:23:01 DEBUG "resultType": "vector", 22:23:01 DEBUG "result": [ 22:23:01 DEBUG { 22:23:01 DEBUG "metric": { 22:23:01 DEBUG "pod": "am-767f6b67b5-9zw22" 22:23:01 DEBUG }, 22:23:01 DEBUG "value": [ 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG [...] 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG "pod": "am-767f6b67b5-vbltz" 22:23:01 DEBUG }, 22:23:01 DEBUG "value": [ 22:23:01 DEBUG 1713648181, 22:23:01 DEBUG "0" 22:23:01 DEBUG ] 22:23:01 DEBUG } 22:23:01 DEBUG ] 22:23:01 DEBUG } 22:23:01 DEBUG } 22:23:01 INFO 22:23:01 INFO [loop_until]: kubectl --namespace=xlou top node 22:23:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:23:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713648181" 22:23:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713648181" 22:23:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648181" 22:23:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648181" 22:23:01 INFO [http_cmd]: http status code OK 22:23:01 DEBUG --- status code --- 22:23:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:01 DEBUG --- http response --- 22:23:01 INFO [http_cmd]: http status code OK 22:23:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:23:01 DEBUG --- status code --- 22:23:01 DEBUG ----- output ----- 22:23:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:01 DEBUG { 22:23:01 DEBUG --- http response --- 22:23:01 DEBUG "status": "success", 22:23:01 DEBUG "data": { 22:23:01 DEBUG "resultType": "vector", 22:23:01 DEBUG "result": [ 22:23:01 DEBUG { 22:23:01 DEBUG "metric": { 22:23:01 DEBUG "instance": "10.142.0.107:9100" 22:23:01 DEBUG }, 22:23:01 DEBUG "value": [ 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG [...] 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG "instance": "10.142.0.43:9100" 22:23:01 DEBUG }, 22:23:01 DEBUG "value": [ 22:23:01 DEBUG 1713648181, 22:23:01 DEBUG "4131171834880" 22:23:01 DEBUG ] 22:23:01 DEBUG } 22:23:01 DEBUG ] 22:23:01 DEBUG } 22:23:01 DEBUG } 22:23:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:23:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648181" 22:23:01 DEBUG ----- output ----- 22:23:01 DEBUG { 22:23:01 DEBUG "status": "success", 22:23:01 DEBUG "data": { 22:23:01 DEBUG "resultType": "vector", 22:23:01 DEBUG "result": [ 22:23:01 DEBUG { 22:23:01 DEBUG "metric": { 22:23:01 DEBUG "instance": "10.142.0.107:9100" 22:23:01 DEBUG }, 22:23:01 DEBUG "value": [ 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG [...] 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG "instance": "10.142.0.43:9100" 22:23:01 DEBUG }, 22:23:01 DEBUG "value": [ 22:23:01 DEBUG 1713648181, 22:23:01 DEBUG "36696.433333333334" 22:23:01 DEBUG ] 22:23:01 DEBUG } 22:23:01 DEBUG ] 22:23:01 DEBUG } 22:23:01 DEBUG } 22:23:01 INFO [http_cmd]: http status code OK 22:23:01 DEBUG --- status code --- 22:23:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:01 DEBUG --- http response --- 22:23:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:23:01 DEBUG ----- output ----- 22:23:01 DEBUG { 22:23:01 DEBUG "status": "success", 22:23:01 DEBUG "data": { 22:23:01 DEBUG "resultType": "vector", 22:23:01 DEBUG "result": [ 22:23:01 DEBUG { 22:23:01 DEBUG "metric": { 22:23:01 DEBUG "pod": "ds-idrepo-0" 22:23:01 DEBUG }, 22:23:01 DEBUG "value": [ 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG [...] 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG "pod": "ds-idrepo-2" 22:23:01 DEBUG }, 22:23:01 DEBUG "value": [ 22:23:01 DEBUG 1713648181, 22:23:01 DEBUG "0" 22:23:01 DEBUG ] 22:23:01 DEBUG } 22:23:01 DEBUG ] 22:23:01 DEBUG } 22:23:01 DEBUG } 22:23:01 INFO [http_cmd]: http status code OK 22:23:01 DEBUG --- status code --- 22:23:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:01 DEBUG --- http response --- 22:23:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:23:01 DEBUG ----- output ----- 22:23:01 DEBUG { 22:23:01 DEBUG "status": "success", 22:23:01 DEBUG "data": { 22:23:01 DEBUG "resultType": "vector", 22:23:01 DEBUG "result": [ 22:23:01 DEBUG { 22:23:01 DEBUG "metric": { 22:23:01 DEBUG "pod": "ds-idrepo-1" 22:23:01 DEBUG }, 22:23:01 DEBUG "value": [ 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG [...] 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG "pod": "idm-b85f9d978-txpwv" 22:23:01 DEBUG }, 22:23:01 DEBUG "value": [ 22:23:01 DEBUG 1713648181, 22:23:01 DEBUG "0" 22:23:01 DEBUG ] 22:23:01 DEBUG } 22:23:01 DEBUG ] 22:23:01 DEBUG } 22:23:01 DEBUG } 22:23:01 INFO [http_cmd]: http status code OK 22:23:01 DEBUG --- status code --- 22:23:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:01 DEBUG --- http response --- 22:23:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:23:01 DEBUG ----- output ----- 22:23:01 DEBUG { 22:23:01 DEBUG "status": "success", 22:23:01 DEBUG "data": { 22:23:01 DEBUG "resultType": "vector", 22:23:01 DEBUG "result": [ 22:23:01 DEBUG { 22:23:01 DEBUG "metric": { 22:23:01 DEBUG "pod": "ds-idrepo-0" 22:23:01 DEBUG }, 22:23:01 DEBUG "value": [ 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG [...] 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG "pod": "ds-idrepo-2" 22:23:01 DEBUG }, 22:23:01 DEBUG "value": [ 22:23:01 DEBUG 1713648181, 22:23:01 DEBUG "0" 22:23:01 DEBUG ] 22:23:01 DEBUG } 22:23:01 DEBUG ] 22:23:01 DEBUG } 22:23:01 DEBUG } 22:23:01 INFO [loop_until]: OK (rc = 0) 22:23:01 DEBUG --- stdout --- 22:23:01 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 6219Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 70m 0% 7678Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 750m 2% 11461Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 82m 0% 3382Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 17992m 60% 15077Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 157m 0% 5178Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 753m 2% 11735Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 78m 0% 3243Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 681m 2% 11123Mi 9% gke-xlou-cdm-ds-0f601751-j488 68m 0% 7284Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 78m 0% 17350Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 89m 0% 7934Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 18163m 60% 14419Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 1079m 6% 9213Mi 15% 22:23:01 DEBUG --- stderr --- 22:23:01 DEBUG 22:23:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713648181" 22:23:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648181" 22:23:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713648181" 22:23:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648181" 22:23:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648181" 22:23:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648181" 22:23:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648181" 22:23:01 INFO [http_cmd]: http status code OK 22:23:01 DEBUG --- status code --- 22:23:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:01 DEBUG --- http response --- 22:23:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:23:01 DEBUG ----- output ----- 22:23:01 DEBUG { 22:23:01 DEBUG "status": "success", 22:23:01 DEBUG "data": { 22:23:01 DEBUG "resultType": "vector", 22:23:01 DEBUG "result": [ 22:23:01 DEBUG { 22:23:01 DEBUG "metric": { 22:23:01 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:23:01 DEBUG }, 22:23:01 DEBUG "value": [ 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG [...] 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:23:01 DEBUG }, 22:23:01 DEBUG "value": [ 22:23:01 DEBUG 1713648181, 22:23:01 DEBUG "983973888" 22:23:01 DEBUG ] 22:23:01 DEBUG } 22:23:01 DEBUG ] 22:23:01 DEBUG } 22:23:01 DEBUG } 22:23:01 INFO [http_cmd]: http status code OK 22:23:01 INFO [http_cmd]: http status code OK 22:23:01 INFO [http_cmd]: http status code OK 22:23:01 INFO [http_cmd]: http status code OK 22:23:01 INFO [http_cmd]: http status code OK 22:23:01 DEBUG --- status code --- 22:23:01 INFO [http_cmd]: http status code OK 22:23:01 DEBUG --- status code --- 22:23:01 DEBUG --- status code --- 22:23:01 DEBUG --- status code --- 22:23:01 DEBUG --- status code --- 22:23:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:01 DEBUG --- status code --- 22:23:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:01 DEBUG --- http response --- 22:23:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:01 DEBUG --- http response --- 22:23:01 DEBUG --- http response --- 22:23:01 DEBUG --- http response --- 22:23:01 DEBUG --- http response --- 22:23:01 DEBUG --- http response --- 22:23:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:23:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:23:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:23:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:23:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:23:01 DEBUG ----- output ----- 22:23:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:23:01 DEBUG ----- output ----- 22:23:01 DEBUG ----- output ----- 22:23:01 DEBUG ----- output ----- 22:23:01 DEBUG ----- output ----- 22:23:01 DEBUG { 22:23:01 DEBUG ----- output ----- 22:23:01 DEBUG { 22:23:01 DEBUG { 22:23:01 DEBUG { 22:23:01 DEBUG { 22:23:01 DEBUG "status": "success", 22:23:01 DEBUG { 22:23:01 DEBUG "status": "success", 22:23:01 DEBUG "status": "success", 22:23:01 DEBUG "status": "success", 22:23:01 DEBUG "status": "success", 22:23:01 DEBUG "data": { 22:23:01 DEBUG "status": "success", 22:23:01 DEBUG "data": { 22:23:01 DEBUG "data": { 22:23:01 DEBUG "data": { 22:23:01 DEBUG "data": { 22:23:01 DEBUG "resultType": "vector", 22:23:01 DEBUG "data": { 22:23:01 DEBUG "resultType": "vector", 22:23:01 DEBUG "resultType": "vector", 22:23:01 DEBUG "resultType": "vector", 22:23:01 DEBUG "resultType": "vector", 22:23:01 DEBUG "result": [ 22:23:01 DEBUG "resultType": "vector", 22:23:01 DEBUG "result": [ 22:23:01 DEBUG "result": [ 22:23:01 DEBUG "result": [ 22:23:01 DEBUG "result": [ 22:23:01 DEBUG { 22:23:01 DEBUG "result": [ 22:23:01 DEBUG { 22:23:01 DEBUG { 22:23:01 DEBUG { 22:23:01 DEBUG { 22:23:01 DEBUG "metric": { 22:23:01 DEBUG { 22:23:01 DEBUG "metric": { 22:23:01 DEBUG "metric": { 22:23:01 DEBUG "metric": { 22:23:01 DEBUG "metric": { 22:23:01 DEBUG "pod": "ds-cts-0" 22:23:01 DEBUG "metric": { 22:23:01 DEBUG "pod": "ds-cts-0" 22:23:01 DEBUG "pod": "ds-idrepo-0" 22:23:01 DEBUG "pod": "am-767f6b67b5-9zw22" 22:23:01 DEBUG "pod": "am-767f6b67b5-9zw22" 22:23:01 DEBUG }, 22:23:01 DEBUG "instance": "10.142.0.107:9100" 22:23:01 DEBUG }, 22:23:01 DEBUG }, 22:23:01 DEBUG }, 22:23:01 DEBUG }, 22:23:01 DEBUG "value": [ 22:23:01 DEBUG }, 22:23:01 DEBUG "value": [ 22:23:01 DEBUG "value": [ 22:23:01 DEBUG "value": [ 22:23:01 DEBUG "value": [ 22:23:01 DEBUG 22:23:01 DEBUG "value": [ 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG [...] 22:23:01 DEBUG 22:23:01 DEBUG [...] 22:23:01 DEBUG [...] 22:23:01 DEBUG [...] 22:23:01 DEBUG [...] 22:23:01 DEBUG 22:23:01 DEBUG [...] 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG 22:23:01 DEBUG "pod": "ds-cts-2" 22:23:01 DEBUG 22:23:01 DEBUG "pod": "ds-cts-2" 22:23:01 DEBUG "pod": "ds-idrepo-2" 22:23:01 DEBUG "pod": "am-767f6b67b5-vbltz" 22:23:01 DEBUG "pod": "am-767f6b67b5-vbltz" 22:23:01 DEBUG }, 22:23:01 DEBUG "instance": "10.142.0.43:9100" 22:23:01 DEBUG }, 22:23:01 DEBUG }, 22:23:01 DEBUG }, 22:23:01 DEBUG }, 22:23:01 DEBUG "value": [ 22:23:01 DEBUG }, 22:23:01 DEBUG "value": [ 22:23:01 DEBUG "value": [ 22:23:01 DEBUG "value": [ 22:23:01 DEBUG "value": [ 22:23:01 DEBUG 1713648181, 22:23:01 DEBUG "value": [ 22:23:01 DEBUG 1713648181, 22:23:01 DEBUG 1713648181, 22:23:01 DEBUG 1713648181, 22:23:01 DEBUG 1713648181, 22:23:01 DEBUG "0" 22:23:01 DEBUG 1713648181, 22:23:01 DEBUG "8.185366666666667" 22:23:01 DEBUG "0" 22:23:01 DEBUG "0" 22:23:01 DEBUG "3" 22:23:01 DEBUG ] 22:23:01 DEBUG "0.00260000000005031" 22:23:01 DEBUG ] 22:23:01 DEBUG ] 22:23:01 DEBUG ] 22:23:01 DEBUG ] 22:23:01 DEBUG } 22:23:01 DEBUG ] 22:23:01 DEBUG } 22:23:01 DEBUG } 22:23:01 DEBUG } 22:23:01 DEBUG } 22:23:01 DEBUG ] 22:23:01 DEBUG } 22:23:01 DEBUG ] 22:23:01 DEBUG ] 22:23:01 DEBUG ] 22:23:01 DEBUG ] 22:23:01 DEBUG } 22:23:01 DEBUG ] 22:23:01 DEBUG } 22:23:01 DEBUG } 22:23:01 DEBUG } 22:23:01 DEBUG } 22:23:01 DEBUG } 22:23:01 DEBUG } 22:23:01 DEBUG } 22:23:01 DEBUG } 22:23:01 DEBUG } 22:23:01 DEBUG } 22:23:01 DEBUG } 22:23:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713648235" 22:23:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648235" 22:23:55 INFO [http_cmd]: http status code OK 22:23:55 DEBUG --- status code --- 22:23:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:55 DEBUG --- http response --- 22:23:55 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:23:55 ERROR [http_cmd]: ERROR 22:23:55 ERROR --- status code --- 22:23:55 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:23:55 ERROR --- http response --- 22:23:55 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:23:55 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713648236" 22:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648236" 22:23:56 INFO [http_cmd]: http status code OK 22:23:56 DEBUG --- status code --- 22:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:56 DEBUG --- http response --- 22:23:56 INFO [http_cmd]: http status code OK 22:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:23:56 DEBUG --- status code --- 22:23:56 DEBUG ----- output ----- 22:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:56 DEBUG { 22:23:56 DEBUG --- http response --- 22:23:56 DEBUG "status": "success", 22:23:56 DEBUG "data": { 22:23:56 DEBUG "resultType": "vector", 22:23:56 DEBUG "result": [ 22:23:56 DEBUG { 22:23:56 DEBUG "metric": { 22:23:56 DEBUG "pod": "ds-idrepo-0" 22:23:56 DEBUG }, 22:23:56 DEBUG "value": [ 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG [...] 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG "pod": "ds-cts-2" 22:23:56 DEBUG }, 22:23:56 DEBUG "value": [ 22:23:56 DEBUG 1713648236, 22:23:56 DEBUG "0" 22:23:56 DEBUG ] 22:23:56 DEBUG } 22:23:56 DEBUG ] 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:23:56 DEBUG ----- output ----- 22:23:56 DEBUG { 22:23:56 DEBUG "status": "success", 22:23:56 DEBUG "data": { 22:23:56 DEBUG "resultType": "vector", 22:23:56 DEBUG "result": [ 22:23:56 DEBUG { 22:23:56 DEBUG "metric": { 22:23:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:23:56 DEBUG }, 22:23:56 DEBUG "value": [ 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG [...] 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG "pod": "overseer-0-967d957f7-zk66v" 22:23:56 DEBUG }, 22:23:56 DEBUG "value": [ 22:23:56 DEBUG 1713648236, 22:23:56 DEBUG "11102943.090814903" 22:23:56 DEBUG ] 22:23:56 DEBUG } 22:23:56 DEBUG ] 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648237" 22:23:57 INFO [http_cmd]: http status code OK 22:23:57 DEBUG --- status code --- 22:23:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:57 DEBUG --- http response --- 22:23:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:23:57 DEBUG ----- output ----- 22:23:57 DEBUG { 22:23:57 DEBUG "status": "success", 22:23:57 DEBUG "data": { 22:23:57 DEBUG "resultType": "vector", 22:23:57 DEBUG "result": [ 22:23:57 DEBUG { 22:23:57 DEBUG "metric": { 22:23:57 DEBUG "pod": "am-767f6b67b5-9zw22" 22:23:57 DEBUG }, 22:23:57 DEBUG "value": [ 22:23:57 DEBUG 22:23:57 DEBUG 22:23:57 DEBUG 22:23:57 DEBUG [...] 22:23:57 DEBUG 22:23:57 DEBUG 22:23:57 DEBUG 22:23:57 DEBUG "pod": "am-767f6b67b5-vbltz" 22:23:57 DEBUG }, 22:23:57 DEBUG "value": [ 22:23:57 DEBUG 1713648237, 22:23:57 DEBUG "0" 22:23:57 DEBUG ] 22:23:57 DEBUG } 22:23:57 DEBUG ] 22:23:57 DEBUG } 22:23:57 DEBUG } 22:23:58 INFO 22:23:58 INFO [loop_until]: kubectl --namespace=xlou top pods 22:23:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:23:58 INFO [loop_until]: OK (rc = 0) 22:23:58 DEBUG --- stdout --- 22:23:58 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 6m 8707Mi am-767f6b67b5-qcqz2 7m 8822Mi am-767f6b67b5-vbltz 6m 8842Mi ds-cts-0 19m 5661Mi ds-cts-1 18m 12257Mi ds-cts-2 17m 11758Mi ds-idrepo-0 14m 3957Mi ds-idrepo-1 12m 3989Mi ds-idrepo-2 13m 3895Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1179Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-7dd48676f6-tdxl9 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-967d957f7-zk66v 1015m 2302Mi 22:23:58 DEBUG --- stderr --- 22:23:58 DEBUG 22:24:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713648240" 22:24:00 INFO [http_cmd]: http status code OK 22:24:00 DEBUG --- status code --- 22:24:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:00 DEBUG --- http response --- 22:24:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:24:00 DEBUG ----- output ----- 22:24:00 DEBUG { 22:24:00 DEBUG "status": "success", 22:24:00 DEBUG "data": { 22:24:00 DEBUG "resultType": "vector", 22:24:00 DEBUG "result": [ 22:24:00 DEBUG { 22:24:00 DEBUG "metric": { 22:24:00 DEBUG "instance": "10.142.0.107:9100" 22:24:00 DEBUG }, 22:24:00 DEBUG "value": [ 22:24:00 DEBUG 22:24:00 DEBUG 22:24:00 DEBUG 22:24:00 DEBUG [...] 22:24:00 DEBUG 22:24:00 DEBUG 22:24:00 DEBUG 22:24:00 DEBUG "instance": "10.142.0.43:9100" 22:24:00 DEBUG }, 22:24:00 DEBUG "value": [ 22:24:00 DEBUG 1713648240, 22:24:00 DEBUG "27360.1" 22:24:00 DEBUG ] 22:24:00 DEBUG } 22:24:00 DEBUG ] 22:24:00 DEBUG } 22:24:00 DEBUG } 22:24:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648240" 22:24:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648240" 22:24:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713648240" 22:24:00 INFO [http_cmd]: http status code OK 22:24:00 DEBUG --- status code --- 22:24:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:00 DEBUG --- http response --- 22:24:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:24:00 DEBUG ----- output ----- 22:24:00 DEBUG { 22:24:00 DEBUG "status": "success", 22:24:00 DEBUG "data": { 22:24:00 DEBUG "resultType": "vector", 22:24:00 DEBUG "result": [ 22:24:00 DEBUG { 22:24:00 DEBUG "metric": { 22:24:00 DEBUG "pod": "am-767f6b67b5-9zw22" 22:24:00 DEBUG }, 22:24:00 DEBUG "value": [ 22:24:00 DEBUG 22:24:00 DEBUG 22:24:00 DEBUG 22:24:00 DEBUG [...] 22:24:00 DEBUG 22:24:00 DEBUG 22:24:00 DEBUG 22:24:00 DEBUG "pod": "am-767f6b67b5-vbltz" 22:24:00 DEBUG }, 22:24:00 DEBUG "value": [ 22:24:00 DEBUG 1713648240, 22:24:00 DEBUG "0" 22:24:00 DEBUG ] 22:24:00 DEBUG } 22:24:00 DEBUG ] 22:24:00 DEBUG } 22:24:00 DEBUG } 22:24:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713648240" 22:24:00 INFO [http_cmd]: http status code OK 22:24:00 DEBUG --- status code --- 22:24:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:00 DEBUG --- http response --- 22:24:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:24:00 DEBUG ----- output ----- 22:24:00 DEBUG { 22:24:00 DEBUG "status": "success", 22:24:00 DEBUG "data": { 22:24:00 DEBUG "resultType": "vector", 22:24:00 DEBUG "result": [ 22:24:00 DEBUG { 22:24:00 DEBUG "metric": { 22:24:00 DEBUG "pod": "am-767f6b67b5-9zw22" 22:24:00 DEBUG }, 22:24:00 DEBUG "value": [ 22:24:00 DEBUG 22:24:00 DEBUG 22:24:00 DEBUG 22:24:00 DEBUG [...] 22:24:00 DEBUG 22:24:00 DEBUG 22:24:00 DEBUG 22:24:00 DEBUG "pod": "am-767f6b67b5-vbltz" 22:24:00 DEBUG }, 22:24:00 DEBUG "value": [ 22:24:00 DEBUG 1713648240, 22:24:00 DEBUG "0" 22:24:00 DEBUG ] 22:24:00 DEBUG } 22:24:00 DEBUG ] 22:24:00 DEBUG } 22:24:00 DEBUG } 22:24:00 INFO [http_cmd]: http status code OK 22:24:00 DEBUG --- status code --- 22:24:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:00 DEBUG --- http response --- 22:24:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:24:00 DEBUG ----- output ----- 22:24:00 DEBUG { 22:24:00 DEBUG "status": "success", 22:24:00 DEBUG "data": { 22:24:00 DEBUG "resultType": "vector", 22:24:00 DEBUG "result": [ 22:24:00 DEBUG { 22:24:00 DEBUG "metric": { 22:24:00 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:24:00 DEBUG }, 22:24:00 DEBUG "value": [ 22:24:00 DEBUG 22:24:00 DEBUG 22:24:00 DEBUG 22:24:00 DEBUG [...] 22:24:00 DEBUG 22:24:00 DEBUG 22:24:00 DEBUG 22:24:00 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:24:00 DEBUG }, 22:24:00 DEBUG "value": [ 22:24:00 DEBUG 1713648240, 22:24:00 DEBUG "1.0316509548562902" 22:24:00 DEBUG ] 22:24:00 DEBUG } 22:24:00 DEBUG ] 22:24:00 DEBUG } 22:24:00 DEBUG } 22:24:00 INFO [http_cmd]: http status code OK 22:24:00 DEBUG --- status code --- 22:24:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:00 DEBUG --- http response --- 22:24:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:24:00 DEBUG ----- output ----- 22:24:00 DEBUG { 22:24:00 DEBUG "status": "success", 22:24:00 DEBUG "data": { 22:24:00 DEBUG "resultType": "vector", 22:24:00 DEBUG "result": [ 22:24:00 DEBUG { 22:24:00 DEBUG "metric": { 22:24:00 DEBUG "pod": "am-767f6b67b5-9zw22" 22:24:00 DEBUG }, 22:24:00 DEBUG "value": [ 22:24:00 DEBUG 22:24:00 DEBUG 22:24:00 DEBUG 22:24:00 DEBUG [...] 22:24:00 DEBUG 22:24:00 DEBUG 22:24:00 DEBUG 22:24:00 DEBUG "pod": "am-767f6b67b5-vbltz" 22:24:00 DEBUG }, 22:24:00 DEBUG "value": [ 22:24:00 DEBUG 1713648240, 22:24:00 DEBUG "101.872" 22:24:00 DEBUG ] 22:24:00 DEBUG } 22:24:00 DEBUG ] 22:24:00 DEBUG } 22:24:00 DEBUG } 22:24:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713648240" 22:24:00 INFO [http_cmd]: http status code OK 22:24:00 DEBUG --- status code --- 22:24:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:00 DEBUG --- http response --- 22:24:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:24:00 DEBUG ----- output ----- 22:24:00 DEBUG { 22:24:00 DEBUG "status": "success", 22:24:00 DEBUG "data": { 22:24:00 DEBUG "resultType": "vector", 22:24:00 DEBUG "result": [ 22:24:00 DEBUG { 22:24:00 DEBUG "metric": { 22:24:00 DEBUG "instance": "10.142.0.107:9100" 22:24:00 DEBUG }, 22:24:00 DEBUG "value": [ 22:24:00 DEBUG 22:24:00 DEBUG 22:24:00 DEBUG 22:24:00 DEBUG [...] 22:24:00 DEBUG 22:24:00 DEBUG 22:24:00 DEBUG 22:24:00 DEBUG "instance": "10.142.0.43:9100" 22:24:00 DEBUG }, 22:24:00 DEBUG "value": [ 22:24:00 DEBUG 1713648240, 22:24:00 DEBUG "0.0006666666666698499" 22:24:00 DEBUG ] 22:24:00 DEBUG } 22:24:00 DEBUG ] 22:24:00 DEBUG } 22:24:00 DEBUG } 22:24:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648241" 22:24:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648241" 22:24:01 INFO [http_cmd]: http status code OK 22:24:01 INFO [http_cmd]: http status code OK 22:24:01 DEBUG --- status code --- 22:24:01 DEBUG --- status code --- 22:24:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:01 DEBUG --- http response --- 22:24:01 DEBUG --- http response --- 22:24:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:24:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:24:01 DEBUG ----- output ----- 22:24:01 DEBUG ----- output ----- 22:24:01 DEBUG { 22:24:01 DEBUG { 22:24:01 DEBUG "status": "success", 22:24:01 DEBUG "status": "success", 22:24:01 DEBUG "data": { 22:24:01 DEBUG "data": { 22:24:01 DEBUG "resultType": "vector", 22:24:01 DEBUG "resultType": "vector", 22:24:01 DEBUG "result": [ 22:24:01 DEBUG "result": [ 22:24:01 DEBUG { 22:24:01 DEBUG { 22:24:01 DEBUG "metric": { 22:24:01 DEBUG "metric": { 22:24:01 DEBUG "pod": "ds-cts-0" 22:24:01 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:24:01 DEBUG }, 22:24:01 DEBUG }, 22:24:01 DEBUG "value": [ 22:24:01 DEBUG "value": [ 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG [...] 22:24:01 DEBUG [...] 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG "pod": "ds-cts-2" 22:24:01 DEBUG "pod": "ds-idrepo-2" 22:24:01 DEBUG }, 22:24:01 DEBUG }, 22:24:01 DEBUG "value": [ 22:24:01 DEBUG "value": [ 22:24:01 DEBUG 1713648241, 22:24:01 DEBUG 1713648241, 22:24:01 DEBUG "27.933333333333334" 22:24:01 DEBUG "0" 22:24:01 DEBUG ] 22:24:01 DEBUG ] 22:24:01 DEBUG } 22:24:01 DEBUG } 22:24:01 DEBUG ] 22:24:01 DEBUG ] 22:24:01 DEBUG } 22:24:01 DEBUG } 22:24:01 DEBUG } 22:24:01 DEBUG } 22:24:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648241" 22:24:01 INFO [http_cmd]: http status code OK 22:24:01 DEBUG --- status code --- 22:24:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:01 DEBUG --- http response --- 22:24:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:24:01 DEBUG ----- output ----- 22:24:01 DEBUG { 22:24:01 DEBUG "status": "success", 22:24:01 DEBUG "data": { 22:24:01 DEBUG "resultType": "vector", 22:24:01 DEBUG "result": [ 22:24:01 DEBUG { 22:24:01 DEBUG "metric": { 22:24:01 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:24:01 DEBUG }, 22:24:01 DEBUG "value": [ 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG [...] 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG "pod": "am-767f6b67b5-qcqz2" 22:24:01 DEBUG }, 22:24:01 DEBUG "value": [ 22:24:01 DEBUG 1713648241, 22:24:01 DEBUG "529.267347202481" 22:24:01 DEBUG ] 22:24:01 DEBUG } 22:24:01 DEBUG ] 22:24:01 DEBUG } 22:24:01 DEBUG } 22:24:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648241" 22:24:01 INFO [http_cmd]: http status code OK 22:24:01 DEBUG --- status code --- 22:24:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:01 DEBUG --- http response --- 22:24:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:24:01 DEBUG ----- output ----- 22:24:01 DEBUG { 22:24:01 DEBUG "status": "success", 22:24:01 DEBUG "data": { 22:24:01 DEBUG "resultType": "vector", 22:24:01 DEBUG "result": [ 22:24:01 DEBUG { 22:24:01 DEBUG "metric": { 22:24:01 DEBUG "pod": "am-767f6b67b5-9zw22" 22:24:01 DEBUG }, 22:24:01 DEBUG "value": [ 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG [...] 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG "pod": "am-767f6b67b5-vbltz" 22:24:01 DEBUG }, 22:24:01 DEBUG "value": [ 22:24:01 DEBUG 1713648241, 22:24:01 DEBUG "0" 22:24:01 DEBUG ] 22:24:01 DEBUG } 22:24:01 DEBUG ] 22:24:01 DEBUG } 22:24:01 DEBUG } 22:24:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713648241" 22:24:01 INFO [http_cmd]: http status code OK 22:24:01 DEBUG --- status code --- 22:24:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:01 DEBUG --- http response --- 22:24:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:24:01 DEBUG ----- output ----- 22:24:01 DEBUG { 22:24:01 DEBUG "status": "success", 22:24:01 DEBUG "data": { 22:24:01 DEBUG "resultType": "vector", 22:24:01 DEBUG "result": [ 22:24:01 DEBUG { 22:24:01 DEBUG "metric": { 22:24:01 DEBUG "instance": "10.142.0.107:9100" 22:24:01 DEBUG }, 22:24:01 DEBUG "value": [ 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG [...] 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG "instance": "10.142.0.43:9100" 22:24:01 DEBUG }, 22:24:01 DEBUG "value": [ 22:24:01 DEBUG 1713648241, 22:24:01 DEBUG "34044.166666666664" 22:24:01 DEBUG ] 22:24:01 DEBUG } 22:24:01 DEBUG ] 22:24:01 DEBUG } 22:24:01 DEBUG } 22:24:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713648241" 22:24:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648241" 22:24:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648241" 22:24:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648241" 22:24:01 INFO [http_cmd]: http status code OK 22:24:01 DEBUG --- status code --- 22:24:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:01 DEBUG --- http response --- 22:24:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:24:01 DEBUG ----- output ----- 22:24:01 DEBUG { 22:24:01 DEBUG "status": "success", 22:24:01 DEBUG "data": { 22:24:01 DEBUG "resultType": "vector", 22:24:01 DEBUG "result": [ 22:24:01 DEBUG { 22:24:01 DEBUG "metric": { 22:24:01 DEBUG "instance": "10.142.0.107:9100" 22:24:01 DEBUG }, 22:24:01 DEBUG "value": [ 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG [...] 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG "instance": "10.142.0.43:9100" 22:24:01 DEBUG }, 22:24:01 DEBUG "value": [ 22:24:01 DEBUG 1713648241, 22:24:01 DEBUG "4131173942272" 22:24:01 DEBUG ] 22:24:01 DEBUG } 22:24:01 DEBUG ] 22:24:01 DEBUG } 22:24:01 DEBUG } 22:24:01 INFO [http_cmd]: http status code OK 22:24:01 DEBUG --- status code --- 22:24:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:01 DEBUG --- http response --- 22:24:01 INFO [http_cmd]: http status code OK 22:24:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:24:01 INFO [http_cmd]: http status code OK 22:24:01 DEBUG --- status code --- 22:24:01 DEBUG ----- output ----- 22:24:01 DEBUG --- status code --- 22:24:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:01 DEBUG { 22:24:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:01 DEBUG --- http response --- 22:24:01 DEBUG "status": "success", 22:24:01 DEBUG --- http response --- 22:24:01 DEBUG "data": { 22:24:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:24:01 DEBUG "resultType": "vector", 22:24:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:24:01 DEBUG ----- output ----- 22:24:01 DEBUG "result": [ 22:24:01 DEBUG ----- output ----- 22:24:01 DEBUG { 22:24:01 DEBUG { 22:24:01 DEBUG { 22:24:01 DEBUG "status": "success", 22:24:01 DEBUG "metric": { 22:24:01 DEBUG "status": "success", 22:24:01 DEBUG "data": { 22:24:01 DEBUG "pod": "ds-idrepo-0" 22:24:01 DEBUG "data": { 22:24:01 DEBUG "resultType": "vector", 22:24:01 DEBUG }, 22:24:01 DEBUG "resultType": "vector", 22:24:01 DEBUG "result": [ 22:24:01 DEBUG "value": [ 22:24:01 DEBUG "result": [ 22:24:01 DEBUG { 22:24:01 DEBUG 22:24:01 DEBUG { 22:24:01 DEBUG "metric": { 22:24:01 DEBUG 22:24:01 DEBUG "metric": { 22:24:01 DEBUG "pod": "ds-idrepo-0" 22:24:01 DEBUG 22:24:01 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:24:01 DEBUG }, 22:24:01 DEBUG [...] 22:24:01 DEBUG }, 22:24:01 DEBUG "value": [ 22:24:01 DEBUG 22:24:01 DEBUG "value": [ 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG "pod": "ds-idrepo-2" 22:24:01 DEBUG 22:24:01 DEBUG [...] 22:24:01 DEBUG }, 22:24:01 DEBUG [...] 22:24:01 DEBUG 22:24:01 DEBUG "value": [ 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG 1713648241, 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG "0" 22:24:01 DEBUG 22:24:01 DEBUG "pod": "ds-idrepo-2" 22:24:01 DEBUG ] 22:24:01 DEBUG "pod": "am-767f6b67b5-qcqz2" 22:24:01 DEBUG }, 22:24:01 DEBUG } 22:24:01 DEBUG }, 22:24:01 DEBUG "value": [ 22:24:01 DEBUG ] 22:24:01 DEBUG "value": [ 22:24:01 DEBUG 1713648241, 22:24:01 DEBUG } 22:24:01 DEBUG 1713648241, 22:24:01 DEBUG "0" 22:24:01 DEBUG } 22:24:01 DEBUG "0" 22:24:01 DEBUG ] 22:24:01 DEBUG ] 22:24:01 DEBUG } 22:24:01 DEBUG } 22:24:01 DEBUG ] 22:24:01 DEBUG ] 22:24:01 DEBUG } 22:24:01 DEBUG } 22:24:01 DEBUG } 22:24:01 DEBUG } 22:24:01 INFO 22:24:01 INFO [loop_until]: kubectl --namespace=xlou top node 22:24:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:24:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648241" 22:24:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648241" 22:24:01 INFO [http_cmd]: http status code OK 22:24:01 DEBUG --- status code --- 22:24:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:01 DEBUG --- http response --- 22:24:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:24:01 DEBUG ----- output ----- 22:24:01 DEBUG { 22:24:01 DEBUG "status": "success", 22:24:01 DEBUG "data": { 22:24:01 DEBUG "resultType": "vector", 22:24:01 DEBUG "result": [ 22:24:01 DEBUG { 22:24:01 DEBUG "metric": { 22:24:01 DEBUG "pod": "am-767f6b67b5-9zw22" 22:24:01 DEBUG }, 22:24:01 DEBUG "value": [ 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG [...] 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG "pod": "am-767f6b67b5-vbltz" 22:24:01 DEBUG }, 22:24:01 DEBUG "value": [ 22:24:01 DEBUG 1713648241, 22:24:01 DEBUG "0" 22:24:01 DEBUG ] 22:24:01 DEBUG } 22:24:01 DEBUG ] 22:24:01 DEBUG } 22:24:01 DEBUG } 22:24:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713648241" 22:24:01 INFO [http_cmd]: http status code OK 22:24:01 DEBUG --- status code --- 22:24:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:01 DEBUG --- http response --- 22:24:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:24:01 DEBUG ----- output ----- 22:24:01 DEBUG { 22:24:01 DEBUG "status": "success", 22:24:01 DEBUG "data": { 22:24:01 DEBUG "resultType": "vector", 22:24:01 DEBUG "result": [ 22:24:01 DEBUG { 22:24:01 DEBUG "metric": { 22:24:01 DEBUG "pod": "ds-cts-0" 22:24:01 DEBUG }, 22:24:01 DEBUG "value": [ 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG [...] 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG "pod": "ds-cts-2" 22:24:01 DEBUG }, 22:24:01 DEBUG "value": [ 22:24:01 DEBUG 1713648241, 22:24:01 DEBUG "0" 22:24:01 DEBUG ] 22:24:01 DEBUG } 22:24:01 DEBUG ] 22:24:01 DEBUG } 22:24:01 DEBUG } 22:24:01 INFO [http_cmd]: http status code OK 22:24:01 DEBUG --- status code --- 22:24:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:01 DEBUG --- http response --- 22:24:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:24:01 DEBUG ----- output ----- 22:24:01 DEBUG { 22:24:01 DEBUG "status": "success", 22:24:01 DEBUG "data": { 22:24:01 DEBUG "resultType": "vector", 22:24:01 DEBUG "result": [ 22:24:01 DEBUG { 22:24:01 DEBUG "metric": { 22:24:01 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:24:01 DEBUG }, 22:24:01 DEBUG "value": [ 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG [...] 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:24:01 DEBUG }, 22:24:01 DEBUG "value": [ 22:24:01 DEBUG 1713648241, 22:24:01 DEBUG "1989308416" 22:24:01 DEBUG ] 22:24:01 DEBUG } 22:24:01 DEBUG ] 22:24:01 DEBUG } 22:24:01 DEBUG } 22:24:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648241" 22:24:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648241" 22:24:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648241" 22:24:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713648241" 22:24:01 INFO [http_cmd]: http status code OK 22:24:01 INFO [http_cmd]: http status code OK 22:24:01 DEBUG --- status code --- 22:24:01 DEBUG --- status code --- 22:24:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:01 DEBUG --- http response --- 22:24:01 DEBUG --- http response --- 22:24:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:24:01 DEBUG ----- output ----- 22:24:01 DEBUG { 22:24:01 DEBUG "status": "success", 22:24:01 DEBUG "data": { 22:24:01 DEBUG "resultType": "vector", 22:24:01 DEBUG "result": [ 22:24:01 DEBUG { 22:24:01 DEBUG "metric": { 22:24:01 DEBUG "pod": "ds-idrepo-0" 22:24:01 DEBUG }, 22:24:01 DEBUG "value": [ 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG [...] 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG "pod": "ds-idrepo-2" 22:24:01 DEBUG }, 22:24:01 DEBUG "value": [ 22:24:01 DEBUG 1713648241, 22:24:01 DEBUG "0" 22:24:01 DEBUG ] 22:24:01 DEBUG } 22:24:01 DEBUG ] 22:24:01 DEBUG } 22:24:01 DEBUG } 22:24:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:24:01 DEBUG ----- output ----- 22:24:01 DEBUG { 22:24:01 DEBUG "status": "success", 22:24:01 DEBUG "data": { 22:24:01 INFO [http_cmd]: http status code OK 22:24:01 DEBUG "resultType": "vector", 22:24:01 INFO [http_cmd]: http status code OK 22:24:01 DEBUG --- status code --- 22:24:01 DEBUG "result": [ 22:24:01 DEBUG --- status code --- 22:24:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:01 DEBUG { 22:24:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:01 DEBUG --- http response --- 22:24:01 DEBUG "metric": { 22:24:01 DEBUG --- http response --- 22:24:01 DEBUG "pod": "am-767f6b67b5-9zw22" 22:24:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:24:01 DEBUG }, 22:24:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:24:01 DEBUG ----- output ----- 22:24:01 DEBUG "value": [ 22:24:01 DEBUG ----- output ----- 22:24:01 DEBUG { 22:24:01 DEBUG 22:24:01 DEBUG { 22:24:01 DEBUG "status": "success", 22:24:01 DEBUG 22:24:01 DEBUG "status": "success", 22:24:01 DEBUG "data": { 22:24:01 DEBUG 22:24:01 DEBUG "data": { 22:24:01 DEBUG "resultType": "vector", 22:24:01 DEBUG [...] 22:24:01 DEBUG "resultType": "vector", 22:24:01 DEBUG "result": [ 22:24:01 DEBUG 22:24:01 DEBUG "result": [ 22:24:01 DEBUG { 22:24:01 DEBUG 22:24:01 DEBUG { 22:24:01 DEBUG "metric": { 22:24:01 DEBUG 22:24:01 DEBUG "metric": { 22:24:01 DEBUG "pod": "ds-cts-0" 22:24:01 DEBUG "pod": "am-767f6b67b5-vbltz" 22:24:01 DEBUG "instance": "10.142.0.107:9100" 22:24:01 DEBUG }, 22:24:01 DEBUG }, 22:24:01 DEBUG }, 22:24:01 DEBUG "value": [ 22:24:01 DEBUG "value": [ 22:24:01 DEBUG "value": [ 22:24:01 DEBUG 22:24:01 DEBUG 1713648241, 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG "2" 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG ] 22:24:01 DEBUG 22:24:01 DEBUG [...] 22:24:01 DEBUG } 22:24:01 DEBUG [...] 22:24:01 DEBUG 22:24:01 DEBUG ] 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG } 22:24:01 DEBUG 22:24:01 DEBUG 22:24:01 DEBUG } 22:24:01 DEBUG 22:24:01 DEBUG "pod": "ds-cts-2" 22:24:01 DEBUG "instance": "10.142.0.43:9100" 22:24:01 DEBUG }, 22:24:01 DEBUG }, 22:24:01 DEBUG "value": [ 22:24:01 DEBUG "value": [ 22:24:01 DEBUG 1713648241, 22:24:01 DEBUG 1713648241, 22:24:01 DEBUG "0" 22:24:01 DEBUG "0.001033333333327846" 22:24:01 DEBUG ] 22:24:01 DEBUG ] 22:24:01 DEBUG } 22:24:01 DEBUG } 22:24:01 DEBUG ] 22:24:01 DEBUG ] 22:24:01 DEBUG } 22:24:01 DEBUG } 22:24:01 DEBUG } 22:24:01 DEBUG } 22:24:01 INFO [loop_until]: OK (rc = 0) 22:24:01 DEBUG --- stdout --- 22:24:01 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6211Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 72m 0% 7703Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 57m 0% 11459Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 85m 0% 3378Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 73m 0% 8861Mi 7% gke-xlou-cdm-default-pool-1b0769c8-wpwl 154m 0% 5192Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 59m 0% 11745Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 79m 0% 3244Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 56m 0% 11106Mi 9% gke-xlou-cdm-ds-0f601751-j488 65m 0% 7293Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 79m 0% 17349Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 90m 0% 7957Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 71m 0% 14444Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 1083m 6% 10183Mi 17% 22:24:01 DEBUG --- stderr --- 22:24:01 DEBUG 22:24:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648295" 22:24:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713648295" 22:24:55 ERROR [http_cmd]: ERROR 22:24:55 INFO [http_cmd]: http status code OK 22:24:55 ERROR --- status code --- 22:24:55 DEBUG --- status code --- 22:24:55 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:24:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:55 ERROR --- http response --- 22:24:55 DEBUG --- http response --- 22:24:55 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:24:55 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:24:55 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713648296" 22:24:56 INFO [http_cmd]: http status code OK 22:24:56 DEBUG --- status code --- 22:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:56 DEBUG --- http response --- 22:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:24:56 DEBUG ----- output ----- 22:24:56 DEBUG { 22:24:56 DEBUG "status": "success", 22:24:56 DEBUG "data": { 22:24:56 DEBUG "resultType": "vector", 22:24:56 DEBUG "result": [ 22:24:56 DEBUG { 22:24:56 DEBUG "metric": { 22:24:56 DEBUG "pod": "ds-idrepo-0" 22:24:56 DEBUG }, 22:24:56 DEBUG "value": [ 22:24:56 DEBUG 22:24:56 DEBUG 22:24:56 DEBUG 22:24:56 DEBUG [...] 22:24:56 DEBUG 22:24:56 DEBUG 22:24:56 DEBUG 22:24:56 DEBUG "pod": "ds-cts-2" 22:24:56 DEBUG }, 22:24:56 DEBUG "value": [ 22:24:56 DEBUG 1713648296, 22:24:56 DEBUG "0" 22:24:56 DEBUG ] 22:24:56 DEBUG } 22:24:56 DEBUG ] 22:24:56 DEBUG } 22:24:56 DEBUG } 22:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648296" 22:24:56 INFO [http_cmd]: http status code OK 22:24:56 DEBUG --- status code --- 22:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:56 DEBUG --- http response --- 22:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:24:56 DEBUG ----- output ----- 22:24:56 DEBUG { 22:24:56 DEBUG "status": "success", 22:24:56 DEBUG "data": { 22:24:56 DEBUG "resultType": "vector", 22:24:56 DEBUG "result": [ 22:24:56 DEBUG { 22:24:56 DEBUG "metric": { 22:24:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 22:24:56 DEBUG }, 22:24:56 DEBUG "value": [ 22:24:56 DEBUG 22:24:56 DEBUG 22:24:56 DEBUG 22:24:56 DEBUG [...] 22:24:56 DEBUG 22:24:56 DEBUG 22:24:56 DEBUG 22:24:56 DEBUG "pod": "overseer-0-967d957f7-zk66v" 22:24:56 DEBUG }, 22:24:56 DEBUG "value": [ 22:24:56 DEBUG 1713648296, 22:24:56 DEBUG "69707843.03969409" 22:24:56 DEBUG ] 22:24:56 DEBUG } 22:24:56 DEBUG ] 22:24:56 DEBUG } 22:24:56 DEBUG } 22:24:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648297" 22:24:57 INFO [http_cmd]: http status code OK 22:24:57 DEBUG --- status code --- 22:24:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:57 DEBUG --- http response --- 22:24:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:24:57 DEBUG ----- output ----- 22:24:57 DEBUG { 22:24:57 DEBUG "status": "success", 22:24:57 DEBUG "data": { 22:24:57 DEBUG "resultType": "vector", 22:24:57 DEBUG "result": [ 22:24:57 DEBUG { 22:24:57 DEBUG "metric": { 22:24:57 DEBUG "pod": "am-767f6b67b5-9zw22" 22:24:57 DEBUG }, 22:24:57 DEBUG "value": [ 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG [...] 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG "pod": "am-767f6b67b5-vbltz" 22:24:57 DEBUG }, 22:24:57 DEBUG "value": [ 22:24:57 DEBUG 1713648297, 22:24:57 DEBUG "0" 22:24:57 DEBUG ] 22:24:57 DEBUG } 22:24:57 DEBUG ] 22:24:57 DEBUG } 22:24:57 DEBUG } 22:24:58 INFO 22:24:58 INFO [loop_until]: kubectl --namespace=xlou top pods 22:24:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:24:58 INFO [loop_until]: OK (rc = 0) 22:24:58 DEBUG --- stdout --- 22:24:58 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 6m 8706Mi am-767f6b67b5-qcqz2 7m 8822Mi am-767f6b67b5-vbltz 7m 8842Mi ds-cts-0 17m 5661Mi ds-cts-1 17m 12259Mi ds-cts-2 17m 11758Mi ds-idrepo-0 16m 3957Mi ds-idrepo-1 16m 3989Mi ds-idrepo-2 13m 3896Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1179Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-7dd48676f6-tdxl9 1m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-967d957f7-zk66v 689m 2611Mi 22:24:58 DEBUG --- stderr --- 22:24:58 DEBUG 22:25:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713648300" 22:25:00 INFO [http_cmd]: http status code OK 22:25:00 DEBUG --- status code --- 22:25:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:00 DEBUG --- http response --- 22:25:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:25:00 DEBUG ----- output ----- 22:25:00 DEBUG { 22:25:00 DEBUG "status": "success", 22:25:00 DEBUG "data": { 22:25:00 DEBUG "resultType": "vector", 22:25:00 DEBUG "result": [ 22:25:00 DEBUG { 22:25:00 DEBUG "metric": { 22:25:00 DEBUG "instance": "10.142.0.107:9100" 22:25:00 DEBUG }, 22:25:00 DEBUG "value": [ 22:25:00 DEBUG 22:25:00 DEBUG 22:25:00 DEBUG 22:25:00 DEBUG [...] 22:25:00 DEBUG 22:25:00 DEBUG 22:25:00 DEBUG 22:25:00 DEBUG "instance": "10.142.0.43:9100" 22:25:00 DEBUG }, 22:25:00 DEBUG "value": [ 22:25:00 DEBUG 1713648300, 22:25:00 DEBUG "27466.699999999997" 22:25:00 DEBUG ] 22:25:00 DEBUG } 22:25:00 DEBUG ] 22:25:00 DEBUG } 22:25:00 DEBUG } 22:25:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648300" 22:25:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648300" 22:25:00 INFO [http_cmd]: http status code OK 22:25:00 DEBUG --- status code --- 22:25:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:00 DEBUG --- http response --- 22:25:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:25:00 DEBUG ----- output ----- 22:25:00 DEBUG { 22:25:00 DEBUG "status": "success", 22:25:00 DEBUG "data": { 22:25:00 DEBUG "resultType": "vector", 22:25:00 DEBUG "result": [ 22:25:00 DEBUG { 22:25:00 DEBUG "metric": { 22:25:00 DEBUG "pod": "am-767f6b67b5-9zw22" 22:25:00 DEBUG }, 22:25:00 DEBUG "value": [ 22:25:00 DEBUG 22:25:00 DEBUG 22:25:00 DEBUG 22:25:00 DEBUG [...] 22:25:00 DEBUG 22:25:00 DEBUG 22:25:00 DEBUG 22:25:00 DEBUG "pod": "am-767f6b67b5-vbltz" 22:25:00 DEBUG }, 22:25:00 DEBUG "value": [ 22:25:00 DEBUG 1713648300, 22:25:00 DEBUG "0.06666666666666667" 22:25:00 DEBUG ] 22:25:00 DEBUG } 22:25:00 DEBUG ] 22:25:00 DEBUG } 22:25:00 DEBUG } 22:25:00 INFO [http_cmd]: http status code OK 22:25:00 DEBUG --- status code --- 22:25:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:00 DEBUG --- http response --- 22:25:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:25:00 DEBUG ----- output ----- 22:25:00 DEBUG { 22:25:00 DEBUG "status": "success", 22:25:00 DEBUG "data": { 22:25:00 DEBUG "resultType": "vector", 22:25:00 DEBUG "result": [ 22:25:00 DEBUG { 22:25:00 DEBUG "metric": { 22:25:00 DEBUG "pod": "am-767f6b67b5-9zw22" 22:25:00 DEBUG }, 22:25:00 DEBUG "value": [ 22:25:00 DEBUG 22:25:00 DEBUG 22:25:00 DEBUG 22:25:00 DEBUG [...] 22:25:00 DEBUG 22:25:00 DEBUG 22:25:00 DEBUG 22:25:00 DEBUG "pod": "am-767f6b67b5-vbltz" 22:25:00 DEBUG }, 22:25:00 DEBUG "value": [ 22:25:00 DEBUG 1713648300, 22:25:00 DEBUG "0" 22:25:00 DEBUG ] 22:25:00 DEBUG } 22:25:00 DEBUG ] 22:25:00 DEBUG } 22:25:00 DEBUG } 22:25:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713648300" 22:25:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713648300" 22:25:00 INFO [http_cmd]: http status code OK 22:25:00 DEBUG --- status code --- 22:25:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:00 DEBUG --- http response --- 22:25:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:25:00 DEBUG ----- output ----- 22:25:00 DEBUG { 22:25:00 DEBUG "status": "success", 22:25:00 DEBUG "data": { 22:25:00 DEBUG "resultType": "vector", 22:25:00 DEBUG "result": [ 22:25:00 DEBUG { 22:25:00 DEBUG "metric": { 22:25:00 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:25:00 DEBUG }, 22:25:00 DEBUG "value": [ 22:25:00 DEBUG 22:25:00 DEBUG 22:25:00 DEBUG 22:25:00 DEBUG [...] 22:25:00 DEBUG 22:25:00 DEBUG 22:25:00 DEBUG 22:25:00 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:25:00 DEBUG }, 22:25:00 DEBUG "value": [ 22:25:00 DEBUG 1713648300, 22:25:00 DEBUG "0.472640520216552" 22:25:00 DEBUG ] 22:25:00 DEBUG } 22:25:00 DEBUG ] 22:25:00 DEBUG } 22:25:00 DEBUG } 22:25:00 INFO [http_cmd]: http status code OK 22:25:00 DEBUG --- status code --- 22:25:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:00 DEBUG --- http response --- 22:25:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:25:00 DEBUG ----- output ----- 22:25:00 DEBUG { 22:25:00 DEBUG "status": "success", 22:25:00 DEBUG "data": { 22:25:00 DEBUG "resultType": "vector", 22:25:00 DEBUG "result": [ 22:25:00 DEBUG { 22:25:00 DEBUG "metric": { 22:25:00 DEBUG "pod": "am-767f6b67b5-9zw22" 22:25:00 DEBUG }, 22:25:00 DEBUG "value": [ 22:25:00 DEBUG 22:25:00 DEBUG 22:25:00 DEBUG 22:25:00 DEBUG [...] 22:25:00 DEBUG 22:25:00 DEBUG 22:25:00 DEBUG 22:25:00 DEBUG "pod": "am-767f6b67b5-vbltz" 22:25:00 DEBUG }, 22:25:00 DEBUG "value": [ 22:25:00 DEBUG 1713648300, 22:25:00 DEBUG "101.8725" 22:25:00 DEBUG ] 22:25:00 DEBUG } 22:25:00 DEBUG ] 22:25:00 DEBUG } 22:25:00 DEBUG } 22:25:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713648300" 22:25:00 INFO [http_cmd]: http status code OK 22:25:00 DEBUG --- status code --- 22:25:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:00 DEBUG --- http response --- 22:25:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:25:00 DEBUG ----- output ----- 22:25:00 DEBUG { 22:25:00 DEBUG "status": "success", 22:25:00 DEBUG "data": { 22:25:00 DEBUG "resultType": "vector", 22:25:00 DEBUG "result": [ 22:25:00 DEBUG { 22:25:00 DEBUG "metric": { 22:25:00 DEBUG "instance": "10.142.0.107:9100" 22:25:00 DEBUG }, 22:25:00 DEBUG "value": [ 22:25:00 DEBUG 22:25:00 DEBUG 22:25:00 DEBUG 22:25:00 DEBUG [...] 22:25:00 DEBUG 22:25:00 DEBUG 22:25:00 DEBUG 22:25:00 DEBUG "instance": "10.142.0.43:9100" 22:25:00 DEBUG }, 22:25:00 DEBUG "value": [ 22:25:00 DEBUG 1713648300, 22:25:00 DEBUG "0.0003333333333330302" 22:25:00 DEBUG ] 22:25:00 DEBUG } 22:25:00 DEBUG ] 22:25:00 DEBUG } 22:25:00 DEBUG } 22:25:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648301" 22:25:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648301" 22:25:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648301" 22:25:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648301" 22:25:01 INFO [http_cmd]: http status code OK 22:25:01 DEBUG --- status code --- 22:25:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:01 DEBUG --- http response --- 22:25:01 INFO [http_cmd]: http status code OK 22:25:01 INFO [http_cmd]: http status code OK 22:25:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:25:01 DEBUG --- status code --- 22:25:01 DEBUG --- status code --- 22:25:01 DEBUG ----- output ----- 22:25:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:01 DEBUG { 22:25:01 DEBUG --- http response --- 22:25:01 DEBUG --- http response --- 22:25:01 DEBUG "status": "success", 22:25:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:25:01 DEBUG "data": { 22:25:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 22:25:01 DEBUG ----- output ----- 22:25:01 DEBUG "resultType": "vector", 22:25:01 DEBUG ----- output ----- 22:25:01 DEBUG { 22:25:01 DEBUG "result": [ 22:25:01 DEBUG { 22:25:01 DEBUG "status": "success", 22:25:01 DEBUG { 22:25:01 DEBUG "status": "success", 22:25:01 DEBUG "data": { 22:25:01 DEBUG "metric": { 22:25:01 DEBUG "data": { 22:25:01 DEBUG "resultType": "vector", 22:25:01 DEBUG "pod": "ds-cts-0" 22:25:01 DEBUG "resultType": "vector", 22:25:01 DEBUG "result": [ 22:25:01 DEBUG }, 22:25:01 DEBUG "result": [ 22:25:01 DEBUG { 22:25:01 DEBUG "value": [ 22:25:01 DEBUG { 22:25:01 DEBUG "metric": { 22:25:01 DEBUG 22:25:01 DEBUG "metric": { 22:25:01 DEBUG "pod": "am-767f6b67b5-9zw22" 22:25:01 DEBUG 22:25:01 DEBUG "pod": "ds-idrepo-0" 22:25:01 DEBUG }, 22:25:01 DEBUG 22:25:01 DEBUG }, 22:25:01 DEBUG "value": [ 22:25:01 DEBUG [...] 22:25:01 DEBUG "value": [ 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG [...] 22:25:01 DEBUG "pod": "ds-cts-2" 22:25:01 DEBUG [...] 22:25:01 DEBUG 22:25:01 DEBUG }, 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG "value": [ 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG 1713648301, 22:25:01 DEBUG 22:25:01 DEBUG "pod": "am-767f6b67b5-vbltz" 22:25:01 DEBUG "29.599999999999998" 22:25:01 DEBUG "pod": "overseer-0-967d957f7-zk66v" 22:25:01 DEBUG }, 22:25:01 DEBUG ] 22:25:01 DEBUG }, 22:25:01 DEBUG "value": [ 22:25:01 DEBUG } 22:25:01 DEBUG "value": [ 22:25:01 DEBUG 1713648301, 22:25:01 DEBUG ] 22:25:01 DEBUG 1713648301, 22:25:01 DEBUG "0" 22:25:01 DEBUG } 22:25:01 DEBUG "0.20767462106150894" 22:25:01 DEBUG ] 22:25:01 DEBUG } 22:25:01 DEBUG ] 22:25:01 DEBUG } 22:25:01 DEBUG } 22:25:01 DEBUG ] 22:25:01 DEBUG ] 22:25:01 DEBUG } 22:25:01 DEBUG } 22:25:01 DEBUG } 22:25:01 INFO [http_cmd]: http status code OK 22:25:01 DEBUG } 22:25:01 DEBUG --- status code --- 22:25:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:01 DEBUG --- http response --- 22:25:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:25:01 DEBUG ----- output ----- 22:25:01 DEBUG { 22:25:01 DEBUG "status": "success", 22:25:01 DEBUG "data": { 22:25:01 DEBUG "resultType": "vector", 22:25:01 DEBUG "result": [ 22:25:01 DEBUG { 22:25:01 DEBUG "metric": { 22:25:01 DEBUG "pod": "ds-idrepo-0" 22:25:01 DEBUG }, 22:25:01 DEBUG "value": [ 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG [...] 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG "pod": "overseer-0-967d957f7-zk66v" 22:25:01 DEBUG }, 22:25:01 DEBUG "value": [ 22:25:01 DEBUG 1713648301, 22:25:01 DEBUG "314.77441136301553" 22:25:01 DEBUG ] 22:25:01 DEBUG } 22:25:01 DEBUG ] 22:25:01 DEBUG } 22:25:01 DEBUG } 22:25:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648301" 22:25:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713648301" 22:25:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713648301" 22:25:01 INFO [http_cmd]: http status code OK 22:25:01 DEBUG --- status code --- 22:25:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:01 DEBUG --- http response --- 22:25:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:25:01 DEBUG ----- output ----- 22:25:01 DEBUG { 22:25:01 DEBUG "status": "success", 22:25:01 DEBUG "data": { 22:25:01 DEBUG "resultType": "vector", 22:25:01 DEBUG "result": [ 22:25:01 DEBUG { 22:25:01 DEBUG "metric": { 22:25:01 DEBUG "pod": "ds-idrepo-0" 22:25:01 DEBUG }, 22:25:01 DEBUG "value": [ 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG [...] 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG "pod": "overseer-0-967d957f7-zk66v" 22:25:01 DEBUG }, 22:25:01 DEBUG "value": [ 22:25:01 DEBUG 1713648301, 22:25:01 DEBUG "4406.841759082217" 22:25:01 DEBUG ] 22:25:01 DEBUG } 22:25:01 DEBUG ] 22:25:01 DEBUG } 22:25:01 DEBUG } 22:25:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648301" 22:25:01 INFO [http_cmd]: http status code OK 22:25:01 INFO [http_cmd]: http status code OK 22:25:01 DEBUG --- status code --- 22:25:01 DEBUG --- status code --- 22:25:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:01 DEBUG --- http response --- 22:25:01 DEBUG --- http response --- 22:25:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:25:01 DEBUG ----- output ----- 22:25:01 DEBUG { 22:25:01 DEBUG "status": "success", 22:25:01 DEBUG "data": { 22:25:01 DEBUG "resultType": "vector", 22:25:01 DEBUG "result": [ 22:25:01 DEBUG { 22:25:01 DEBUG "metric": { 22:25:01 DEBUG "instance": "10.142.0.107:9100" 22:25:01 DEBUG }, 22:25:01 DEBUG "value": [ 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:25:01 DEBUG [...] 22:25:01 DEBUG ----- output ----- 22:25:01 DEBUG 22:25:01 DEBUG { 22:25:01 DEBUG 22:25:01 DEBUG "status": "success", 22:25:01 DEBUG 22:25:01 DEBUG "data": { 22:25:01 DEBUG "instance": "10.142.0.43:9100" 22:25:01 DEBUG "resultType": "vector", 22:25:01 DEBUG }, 22:25:01 DEBUG "result": [ 22:25:01 DEBUG "value": [ 22:25:01 DEBUG { 22:25:01 DEBUG 1713648301, 22:25:01 DEBUG "metric": { 22:25:01 DEBUG "4131175767040" 22:25:01 DEBUG "instance": "10.142.0.107:9100" 22:25:01 DEBUG ] 22:25:01 DEBUG }, 22:25:01 DEBUG } 22:25:01 DEBUG "value": [ 22:25:01 DEBUG ] 22:25:01 DEBUG 22:25:01 DEBUG } 22:25:01 DEBUG 22:25:01 DEBUG } 22:25:01 DEBUG 22:25:01 DEBUG [...] 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG "instance": "10.142.0.43:9100" 22:25:01 DEBUG }, 22:25:01 DEBUG "value": [ 22:25:01 DEBUG 1713648301, 22:25:01 DEBUG "34577.666666666664" 22:25:01 DEBUG ] 22:25:01 DEBUG } 22:25:01 DEBUG ] 22:25:01 DEBUG } 22:25:01 DEBUG } 22:25:01 INFO [http_cmd]: http status code OK 22:25:01 DEBUG --- status code --- 22:25:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:01 DEBUG --- http response --- 22:25:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:25:01 DEBUG ----- output ----- 22:25:01 DEBUG { 22:25:01 DEBUG "status": "success", 22:25:01 DEBUG "data": { 22:25:01 DEBUG "resultType": "vector", 22:25:01 DEBUG "result": [ 22:25:01 DEBUG { 22:25:01 DEBUG "metric": { 22:25:01 DEBUG "pod": "am-767f6b67b5-9zw22" 22:25:01 DEBUG }, 22:25:01 DEBUG "value": [ 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG [...] 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG "pod": "am-767f6b67b5-vbltz" 22:25:01 DEBUG }, 22:25:01 DEBUG "value": [ 22:25:01 DEBUG 1713648301, 22:25:01 DEBUG "0" 22:25:01 DEBUG ] 22:25:01 DEBUG } 22:25:01 DEBUG ] 22:25:01 DEBUG } 22:25:01 DEBUG } 22:25:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648301" 22:25:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648301" 22:25:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713648301" 22:25:01 INFO [http_cmd]: http status code OK 22:25:01 INFO [http_cmd]: http status code OK 22:25:01 DEBUG --- status code --- 22:25:01 DEBUG --- status code --- 22:25:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:01 DEBUG --- http response --- 22:25:01 DEBUG --- http response --- 22:25:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:25:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:25:01 DEBUG ----- output ----- 22:25:01 DEBUG ----- output ----- 22:25:01 DEBUG { 22:25:01 DEBUG { 22:25:01 DEBUG "status": "success", 22:25:01 DEBUG "status": "success", 22:25:01 DEBUG "data": { 22:25:01 DEBUG "data": { 22:25:01 DEBUG "resultType": "vector", 22:25:01 DEBUG "resultType": "vector", 22:25:01 DEBUG "result": [ 22:25:01 DEBUG "result": [ 22:25:01 DEBUG { 22:25:01 DEBUG { 22:25:01 DEBUG "metric": { 22:25:01 DEBUG "metric": { 22:25:01 DEBUG "pod": "ds-idrepo-0" 22:25:01 DEBUG "pod": "ds-idrepo-0" 22:25:01 DEBUG }, 22:25:01 DEBUG }, 22:25:01 DEBUG "value": [ 22:25:01 DEBUG "value": [ 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG [...] 22:25:01 DEBUG [...] 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG "pod": "ds-idrepo-2" 22:25:01 DEBUG "pod": "ds-idrepo-2" 22:25:01 DEBUG }, 22:25:01 DEBUG }, 22:25:01 DEBUG "value": [ 22:25:01 DEBUG "value": [ 22:25:01 DEBUG 1713648301, 22:25:01 DEBUG 1713648301, 22:25:01 DEBUG "0" 22:25:01 DEBUG "0" 22:25:01 DEBUG ] 22:25:01 DEBUG ] 22:25:01 DEBUG } 22:25:01 DEBUG } 22:25:01 DEBUG ] 22:25:01 DEBUG ] 22:25:01 DEBUG } 22:25:01 DEBUG } 22:25:01 DEBUG } 22:25:01 DEBUG } 22:25:01 INFO [http_cmd]: http status code OK 22:25:01 DEBUG --- status code --- 22:25:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:01 DEBUG --- http response --- 22:25:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:25:01 DEBUG ----- output ----- 22:25:01 DEBUG { 22:25:01 DEBUG "status": "success", 22:25:01 DEBUG "data": { 22:25:01 DEBUG "resultType": "vector", 22:25:01 DEBUG "result": [ 22:25:01 DEBUG { 22:25:01 DEBUG "metric": { 22:25:01 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 22:25:01 DEBUG }, 22:25:01 DEBUG "value": [ 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG [...] 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:25:01 DEBUG }, 22:25:01 DEBUG "value": [ 22:25:01 DEBUG 1713648301, 22:25:01 DEBUG "2406060032" 22:25:01 DEBUG ] 22:25:01 DEBUG } 22:25:01 DEBUG ] 22:25:01 DEBUG } 22:25:01 DEBUG } 22:25:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648301" 22:25:01 INFO [http_cmd]: http status code OK 22:25:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713648301" 22:25:01 DEBUG --- status code --- 22:25:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:01 DEBUG --- http response --- 22:25:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:25:01 DEBUG ----- output ----- 22:25:01 DEBUG { 22:25:01 DEBUG "status": "success", 22:25:01 DEBUG "data": { 22:25:01 DEBUG "resultType": "vector", 22:25:01 DEBUG "result": [ 22:25:01 DEBUG { 22:25:01 DEBUG "metric": { 22:25:01 DEBUG "pod": "ds-cts-0" 22:25:01 DEBUG }, 22:25:01 DEBUG "value": [ 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG [...] 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG "pod": "ds-cts-2" 22:25:01 DEBUG }, 22:25:01 DEBUG "value": [ 22:25:01 DEBUG 1713648301, 22:25:01 DEBUG "0" 22:25:01 DEBUG ] 22:25:01 DEBUG } 22:25:01 DEBUG ] 22:25:01 DEBUG } 22:25:01 DEBUG } 22:25:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648301" 22:25:01 INFO [http_cmd]: http status code OK 22:25:01 DEBUG --- status code --- 22:25:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:01 DEBUG --- http response --- 22:25:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:25:01 DEBUG ----- output ----- 22:25:01 DEBUG { 22:25:01 DEBUG "status": "success", 22:25:01 DEBUG "data": { 22:25:01 DEBUG "resultType": "vector", 22:25:01 DEBUG "result": [ 22:25:01 DEBUG { 22:25:01 DEBUG "metric": { 22:25:01 DEBUG "instance": "10.142.0.107:9100" 22:25:01 DEBUG }, 22:25:01 DEBUG "value": [ 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG [...] 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG "instance": "10.142.0.43:9100" 22:25:01 DEBUG }, 22:25:01 DEBUG "value": [ 22:25:01 DEBUG 1713648301, 22:25:01 DEBUG "0.0011666666666873956" 22:25:01 DEBUG ] 22:25:01 DEBUG } 22:25:01 DEBUG ] 22:25:01 DEBUG } 22:25:01 DEBUG } 22:25:01 INFO [http_cmd]: http status code OK 22:25:01 DEBUG --- status code --- 22:25:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:01 DEBUG --- http response --- 22:25:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:25:01 DEBUG ----- output ----- 22:25:01 DEBUG { 22:25:01 DEBUG "status": "success", 22:25:01 DEBUG "data": { 22:25:01 DEBUG "resultType": "vector", 22:25:01 DEBUG "result": [ 22:25:01 DEBUG { 22:25:01 DEBUG "metric": { 22:25:01 DEBUG "pod": "ds-cts-0" 22:25:01 DEBUG }, 22:25:01 DEBUG "value": [ 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG [...] 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG "pod": "ds-cts-2" 22:25:01 DEBUG }, 22:25:01 DEBUG "value": [ 22:25:01 DEBUG 1713648301, 22:25:01 DEBUG "0" 22:25:01 DEBUG ] 22:25:01 DEBUG } 22:25:01 DEBUG ] 22:25:01 DEBUG } 22:25:01 DEBUG } 22:25:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648301" 22:25:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713648301" 22:25:01 INFO 22:25:01 INFO [loop_until]: kubectl --namespace=xlou top node 22:25:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:25:01 INFO [http_cmd]: http status code OK 22:25:01 DEBUG --- status code --- 22:25:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:01 DEBUG --- http response --- 22:25:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:25:01 DEBUG ----- output ----- 22:25:01 DEBUG { 22:25:01 DEBUG "status": "success", 22:25:01 DEBUG "data": { 22:25:01 DEBUG "resultType": "vector", 22:25:01 DEBUG "result": [ 22:25:01 DEBUG { 22:25:01 DEBUG "metric": { 22:25:01 DEBUG "pod": "ds-idrepo-0" 22:25:01 DEBUG }, 22:25:01 DEBUG "value": [ 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG [...] 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG "pod": "ds-idrepo-2" 22:25:01 DEBUG }, 22:25:01 DEBUG "value": [ 22:25:01 DEBUG 1713648301, 22:25:01 DEBUG "0" 22:25:01 DEBUG ] 22:25:01 DEBUG } 22:25:01 DEBUG ] 22:25:01 DEBUG } 22:25:01 DEBUG } 22:25:01 INFO [http_cmd]: http status code OK 22:25:01 DEBUG --- status code --- 22:25:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:01 DEBUG --- http response --- 22:25:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:25:01 DEBUG ----- output ----- 22:25:01 DEBUG { 22:25:01 DEBUG "status": "success", 22:25:01 DEBUG "data": { 22:25:01 DEBUG "resultType": "vector", 22:25:01 DEBUG "result": [ 22:25:01 DEBUG { 22:25:01 DEBUG "metric": { 22:25:01 DEBUG "pod": "am-767f6b67b5-9zw22" 22:25:01 DEBUG }, 22:25:01 DEBUG "value": [ 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG [...] 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG 22:25:01 DEBUG "pod": "am-767f6b67b5-vbltz" 22:25:01 DEBUG }, 22:25:01 DEBUG "value": [ 22:25:01 DEBUG 1713648301, 22:25:01 DEBUG "1" 22:25:01 DEBUG ] 22:25:01 DEBUG } 22:25:01 DEBUG ] 22:25:01 DEBUG } 22:25:01 DEBUG } 22:25:01 INFO [loop_until]: OK (rc = 0) 22:25:01 DEBUG --- stdout --- 22:25:01 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6212Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 74m 0% 7698Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 57m 0% 11441Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 81m 0% 3375Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 80m 0% 8868Mi 7% gke-xlou-cdm-default-pool-1b0769c8-wpwl 155m 0% 5187Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 62m 0% 11735Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 80m 0% 3217Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 60m 0% 11083Mi 9% gke-xlou-cdm-ds-0f601751-j488 66m 0% 7288Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 78m 0% 17328Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 96m 0% 7950Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 71m 0% 14445Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 1017m 6% 10965Mi 18% 22:25:01 DEBUG --- stderr --- 22:25:01 DEBUG 22:25:16 INFO Finished: True 22:25:16 INFO Waiting for threads to register finish flag 22:26:01 INFO Done. Have a nice day! :) 127.0.0.1 - - [20/Apr/2024 22:26:01] "GET /monitoring/stop HTTP/1.1" 200 - 22:26:04 WARNING File /tmp/lodemon_data-20240420_213952/node_disk_read_bytes_total.json does not exist. Skipping graph generation... WARNING:bokeh.core.validation.check:W-1000 (MISSING_RENDERERS): Plot has no renderers: figure(id='p6481', ...) 127.0.0.1 - - [20/Apr/2024 22:26:08] "GET /monitoring/process HTTP/1.1" 200 -