==================================================================================================== ========================================= Pod describe ========================================= ==================================================================================================== Name: lodemon-766dc9b5df-2jmd4 Namespace: xlou Priority: 0 Service Account: k8s-svc-acct-lodemon Node: gke-xlou-cdm-ds-0f601751-mc5d/10.142.0.43 Start Time: Sun, 14 Apr 2024 02:14:28 +0000 Labels: app=lodemon app.kubernetes.io/name=lodemon pod-template-hash=766dc9b5df skaffold.dev/run-id=76f51739-0c90-489f-9fca-a831e0340461 Annotations: Status: Running IP: 10.106.46.135 IPs: IP: 10.106.46.135 Controlled By: ReplicaSet/lodemon-766dc9b5df Containers: lodemon: Container ID: containerd://f810215b0a734b903e063077aebeac193355867433e49fda8787a87cfbabd264 Image: gcr.io/engineeringpit/lodestar-images/lodestarbox:master-stable Image ID: gcr.io/engineeringpit/lodestar-images/lodestarbox@sha256:06c53901427b11990b989f0a16f422c2c89c055d3393605a0d2d532652d23afc Port: 8080/TCP Host Port: 0/TCP Command: python3 Args: /lodestar/scripts/lodemon_run.py -W default State: Running Started: Sun, 14 Apr 2024 02:14:28 +0000 Ready: True Restart Count: 0 Limits: cpu: 2 memory: 2Gi Requests: cpu: 1 memory: 1Gi Liveness: exec [cat /tmp/lodemon_alive] delay=20s timeout=1s period=10s #success=1 #failure=3 Readiness: exec [cat /tmp/lodemon_alive] delay=20s timeout=1s period=10s #success=1 #failure=3 Environment: SKAFFOLD_PROFILE: medium Mounts: /lodestar/config/config.yaml from config (rw,path="config.yaml") /lodestar/config/deployments_to_monitor.yaml from deployments-config (rw,path="deployments_to_monitor.yaml") /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-gfn48 (ro) Conditions: Type Status Initialized True Ready True ContainersReady True PodScheduled True Volumes: config: Type: ConfigMap (a volume populated by a ConfigMap) Name: lodemon-config Optional: false deployments-config: Type: ConfigMap (a volume populated by a ConfigMap) Name: lodemon-deployments Optional: false kube-api-access-gfn48: Type: Projected (a volume that contains injected data from multiple sources) TokenExpirationSeconds: 3607 ConfigMapName: kube-root-ca.crt ConfigMapOptional: DownwardAPI: true QoS Class: Burstable Node-Selectors: Tolerations: node.kubernetes.io/not-ready:NoExecute op=Exists for 300s node.kubernetes.io/unreachable:NoExecute op=Exists for 300s Events: Type Reason Age From Message ---- ------ ---- ---- ------- Normal Scheduled 14m default-scheduler Successfully assigned xlou/lodemon-766dc9b5df-2jmd4 to gke-xlou-cdm-ds-0f601751-mc5d Normal Pulling 14m kubelet Pulling image "gcr.io/engineeringpit/lodestar-images/lodestarbox:master-stable" Normal Pulled 14m kubelet Successfully pulled image "gcr.io/engineeringpit/lodestar-images/lodestarbox:master-stable" in 274.897265ms (274.903265ms including waiting) Normal Created 14m kubelet Created container lodemon Normal Started 14m kubelet Started container lodemon ==================================================================================================== =========================================== Pod logs =========================================== ==================================================================================================== * Serving Flask app 'lodemon_run' * Debug mode: off WARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead. * Running on all addresses (0.0.0.0) * Running on http://127.0.0.1:8080 * Running on http://10.106.46.135:8080 Press CTRL+C to quit 03:15:25 INFO 03:15:25 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:15:25 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:15:25 INFO [loop_until]: OK (rc = 0) 03:15:25 DEBUG --- stdout --- 03:15:25 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:15:25 DEBUG --- stderr --- 03:15:25 DEBUG 03:15:25 INFO 03:15:25 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:15:25 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:15:25 INFO [loop_until]: OK (rc = 0) 03:15:25 DEBUG --- stdout --- 03:15:25 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:15:25 DEBUG --- stderr --- 03:15:25 DEBUG 03:15:25 INFO 03:15:25 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:15:25 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:15:25 INFO [loop_until]: OK (rc = 0) 03:15:25 DEBUG --- stdout --- 03:15:25 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:15:25 DEBUG --- stderr --- 03:15:25 DEBUG 03:15:25 INFO 03:15:25 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:15:25 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:15:25 INFO [loop_until]: OK (rc = 0) 03:15:25 DEBUG --- stdout --- 03:15:25 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:15:25 DEBUG --- stderr --- 03:15:25 DEBUG 03:15:25 INFO 03:15:25 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:15:25 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:15:25 INFO [loop_until]: OK (rc = 0) 03:15:25 DEBUG --- stdout --- 03:15:25 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:15:25 DEBUG --- stderr --- 03:15:25 DEBUG 03:15:25 INFO 03:15:25 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:15:25 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:15:25 INFO [loop_until]: OK (rc = 0) 03:15:25 DEBUG --- stdout --- 03:15:25 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:15:25 DEBUG --- stderr --- 03:15:25 DEBUG 03:15:25 INFO 03:15:25 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:15:25 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:15:26 INFO [loop_until]: OK (rc = 0) 03:15:26 DEBUG --- stdout --- 03:15:26 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:15:26 DEBUG --- stderr --- 03:15:26 DEBUG 03:15:26 INFO 03:15:26 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:15:26 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:15:26 INFO [loop_until]: OK (rc = 0) 03:15:26 DEBUG --- stdout --- 03:15:26 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:15:26 DEBUG --- stderr --- 03:15:26 DEBUG 03:15:26 INFO 03:15:26 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:15:26 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:15:26 INFO [loop_until]: OK (rc = 0) 03:15:26 DEBUG --- stdout --- 03:15:26 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:15:26 DEBUG --- stderr --- 03:15:26 DEBUG 03:15:26 INFO 03:15:26 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:15:26 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:15:26 INFO [loop_until]: OK (rc = 0) 03:15:26 DEBUG --- stdout --- 03:15:26 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:15:26 DEBUG --- stderr --- 03:15:26 DEBUG 03:15:26 INFO 03:15:26 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:15:26 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:15:26 INFO [loop_until]: OK (rc = 0) 03:15:26 DEBUG --- stdout --- 03:15:26 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:15:26 DEBUG --- stderr --- 03:15:26 DEBUG 03:15:26 INFO 03:15:26 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:15:26 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:15:26 INFO [loop_until]: OK (rc = 0) 03:15:26 DEBUG --- stdout --- 03:15:26 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:15:26 DEBUG --- stderr --- 03:15:26 DEBUG 03:15:26 INFO 03:15:26 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:15:26 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:15:26 INFO [loop_until]: OK (rc = 0) 03:15:26 DEBUG --- stdout --- 03:15:26 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:15:26 DEBUG --- stderr --- 03:15:26 DEBUG 03:15:26 INFO 03:15:26 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:15:26 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:15:26 INFO [loop_until]: OK (rc = 0) 03:15:26 DEBUG --- stdout --- 03:15:26 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:15:26 DEBUG --- stderr --- 03:15:26 DEBUG 03:15:27 INFO 03:15:27 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:15:27 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:15:27 INFO [loop_until]: OK (rc = 0) 03:15:27 DEBUG --- stdout --- 03:15:27 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:15:27 DEBUG --- stderr --- 03:15:27 DEBUG 03:15:27 INFO 03:15:27 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:15:27 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:15:27 INFO [loop_until]: OK (rc = 0) 03:15:27 DEBUG --- stdout --- 03:15:27 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:15:27 DEBUG --- stderr --- 03:15:27 DEBUG 03:15:27 INFO 03:15:27 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:15:27 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:15:27 INFO [loop_until]: OK (rc = 0) 03:15:27 DEBUG --- stdout --- 03:15:27 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:15:27 DEBUG --- stderr --- 03:15:27 DEBUG 03:15:27 INFO 03:15:27 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:15:27 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:15:27 INFO [loop_until]: OK (rc = 0) 03:15:27 DEBUG --- stdout --- 03:15:27 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:15:27 DEBUG --- stderr --- 03:15:27 DEBUG 03:15:27 INFO 03:15:27 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:15:27 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:15:27 INFO [loop_until]: OK (rc = 0) 03:15:27 DEBUG --- stdout --- 03:15:27 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:15:27 DEBUG --- stderr --- 03:15:27 DEBUG 03:15:27 INFO 03:15:27 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:15:27 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:15:27 INFO [loop_until]: OK (rc = 0) 03:15:27 DEBUG --- stdout --- 03:15:27 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:15:27 DEBUG --- stderr --- 03:15:27 DEBUG 03:15:27 INFO 03:15:27 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:15:27 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:15:27 INFO [loop_until]: OK (rc = 0) 03:15:27 DEBUG --- stdout --- 03:15:27 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:15:27 DEBUG --- stderr --- 03:15:27 DEBUG 03:15:27 INFO 03:15:27 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:15:27 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:15:28 INFO [loop_until]: OK (rc = 0) 03:15:28 DEBUG --- stdout --- 03:15:28 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:15:28 DEBUG --- stderr --- 03:15:28 DEBUG 03:15:28 INFO 03:15:28 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:15:28 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:15:28 INFO [loop_until]: OK (rc = 0) 03:15:28 DEBUG --- stdout --- 03:15:28 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:15:28 DEBUG --- stderr --- 03:15:28 DEBUG 03:15:28 INFO 03:15:28 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:15:28 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:15:28 INFO [loop_until]: OK (rc = 0) 03:15:28 DEBUG --- stdout --- 03:15:28 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:15:28 DEBUG --- stderr --- 03:15:28 DEBUG 03:15:28 INFO 03:15:28 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:15:28 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:15:28 INFO [loop_until]: OK (rc = 0) 03:15:28 DEBUG --- stdout --- 03:15:28 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:15:28 DEBUG --- stderr --- 03:15:28 DEBUG 03:15:28 INFO 03:15:28 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:15:28 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:15:28 INFO [loop_until]: OK (rc = 0) 03:15:28 DEBUG --- stdout --- 03:15:28 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:15:28 DEBUG --- stderr --- 03:15:28 DEBUG 03:15:28 INFO 03:15:28 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:15:28 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:15:28 INFO [loop_until]: OK (rc = 0) 03:15:28 DEBUG --- stdout --- 03:15:28 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:15:28 DEBUG --- stderr --- 03:15:28 DEBUG 03:15:28 INFO Initializing monitoring instance threads 03:15:28 DEBUG Monitoring instance thread list: [, , , , , , , , , , , , , , , , , , , , , , , , , , , , ] 03:15:28 INFO Starting instance threads 03:15:28 INFO 03:15:28 INFO Thread started 03:15:28 INFO [loop_until]: kubectl --namespace=xlou top node 03:15:28 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:15:28 INFO 03:15:28 INFO Thread started 03:15:28 INFO [loop_until]: kubectl --namespace=xlou top pods 03:15:28 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713060928" 03:15:28 INFO Thread started 03:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713060928" 03:15:28 INFO Thread started 03:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713060928" 03:15:28 INFO Thread started 03:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713060928" 03:15:28 INFO Thread started 03:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713060928" 03:15:28 INFO Thread started 03:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713060928" 03:15:28 INFO Thread started 03:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713060928" 03:15:28 INFO Thread started 03:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713060928" 03:15:28 INFO Thread started 03:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713060928" 03:15:28 INFO Thread started 03:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713060928" 03:15:28 INFO Thread started 03:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713060928" 03:15:28 INFO Thread started 03:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713060928" 03:15:28 INFO Thread started 03:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713060928" 03:15:28 INFO Thread started 03:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713060928" 03:15:28 INFO Thread started 03:15:28 INFO [http_cmd]: http status code OK 03:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713060928" 03:15:28 DEBUG --- status code --- 03:15:28 INFO [http_cmd]: http status code OK 03:15:28 INFO Thread started 03:15:28 INFO [http_cmd]: http status code OK 03:15:28 INFO [http_cmd]: http status code OK 03:15:28 INFO [http_cmd]: http status code OK 03:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:28 DEBUG --- status code --- 03:15:28 INFO [http_cmd]: http status code OK 03:15:28 INFO [http_cmd]: http status code OK 03:15:28 INFO [http_cmd]: http status code OK 03:15:28 INFO [http_cmd]: http status code OK 03:15:28 INFO [http_cmd]: http status code OK 03:15:28 INFO [http_cmd]: http status code OK 03:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713060928" 03:15:28 DEBUG --- status code --- 03:15:28 INFO Thread started 03:15:28 INFO [http_cmd]: http status code OK 03:15:28 DEBUG --- status code --- 03:15:28 DEBUG --- status code --- 03:15:28 DEBUG --- http response --- 03:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:28 INFO [http_cmd]: http status code OK 03:15:28 DEBUG --- status code --- 03:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:28 DEBUG --- status code --- 03:15:28 DEBUG --- status code --- 03:15:28 DEBUG --- status code --- 03:15:28 DEBUG --- status code --- 03:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713060928" 03:15:28 DEBUG --- status code --- 03:15:28 INFO Thread started 03:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:28 DEBUG --- http response --- 03:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:15:28 DEBUG --- status code --- 03:15:28 DEBUG --- http response --- 03:15:28 DEBUG --- status code --- 03:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:28 DEBUG --- http response --- 03:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:28 INFO [http_cmd]: http status code OK 03:15:28 DEBUG --- http response --- 03:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713060928" 03:15:28 DEBUG --- http response --- 03:15:28 INFO Thread started 03:15:28 DEBUG ----- output ----- 03:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:28 DEBUG --- http response --- 03:15:28 DEBUG --- http response --- 03:15:28 DEBUG --- http response --- 03:15:28 DEBUG --- http response --- 03:15:28 DEBUG --- http response --- 03:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:15:28 DEBUG --- status code --- 03:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (161 lines): 03:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:15:28 DEBUG { 03:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713060928" 03:15:28 DEBUG ----- output ----- 03:15:28 INFO Thread started 03:15:28 DEBUG --- http response --- 03:15:28 DEBUG ----- output ----- 03:15:28 DEBUG --- http response --- 03:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:15:28 INFO [http_cmd]: http status code OK 03:15:28 INFO [http_cmd]: http status code OK 03:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:15:28 INFO [http_cmd]: http status code OK 03:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:15:28 DEBUG ----- output ----- 03:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:28 DEBUG ----- output ----- 03:15:28 DEBUG ----- output ----- 03:15:28 DEBUG "status": "success", 03:15:28 DEBUG { 03:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713060928" 03:15:28 INFO Thread started 03:15:28 DEBUG { 03:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (161 lines): 03:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:15:28 DEBUG ----- output ----- 03:15:28 DEBUG ----- output ----- 03:15:28 DEBUG --- status code --- 03:15:28 DEBUG --- status code --- 03:15:28 DEBUG ----- output ----- 03:15:28 DEBUG --- status code --- 03:15:28 DEBUG ----- output ----- 03:15:28 DEBUG ----- output ----- 03:15:28 DEBUG { 03:15:28 DEBUG --- http response --- 03:15:28 DEBUG { 03:15:28 DEBUG { 03:15:28 DEBUG "data": { 03:15:28 INFO [http_cmd]: http status code OK 03:15:28 DEBUG "status": "success", 03:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713060928" 03:15:28 DEBUG "status": "success", 03:15:28 INFO Thread started 03:15:28 DEBUG ----- output ----- 03:15:28 DEBUG ----- output ----- 03:15:28 DEBUG { 03:15:28 DEBUG { 03:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:28 DEBUG { 03:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:28 DEBUG { 03:15:28 DEBUG { 03:15:28 DEBUG "status": "success", 03:15:28 DEBUG "status": "success", 03:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:15:28 DEBUG "status": "success", 03:15:28 DEBUG "resultType": "vector", 03:15:28 DEBUG --- status code --- 03:15:28 DEBUG "data": { 03:15:28 DEBUG "data": { 03:15:28 DEBUG { 03:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713060928" 03:15:28 INFO Thread started 03:15:28 DEBUG { 03:15:28 DEBUG "status": "success", 03:15:28 DEBUG "status": "success", 03:15:28 DEBUG --- http response --- 03:15:28 DEBUG --- http response --- 03:15:28 DEBUG "status": "success", 03:15:28 DEBUG --- http response --- 03:15:28 DEBUG "status": "success", 03:15:28 DEBUG "status": "success", 03:15:28 DEBUG "data": { 03:15:28 DEBUG "data": { 03:15:28 DEBUG ----- output ----- 03:15:28 DEBUG "data": { 03:15:28 DEBUG "result": [ 03:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:28 DEBUG "resultType": "vector", 03:15:28 DEBUG "resultType": "vector", 03:15:28 DEBUG "status": "success", 03:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713060928" 03:15:28 DEBUG "status": "success", 03:15:28 INFO Thread started 03:15:28 INFO [http_cmd]: http status code OK 03:15:28 DEBUG "data": { 03:15:28 DEBUG "data": { 03:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:15:28 DEBUG "data": { 03:15:28 DEBUG "data": { 03:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:15:28 DEBUG "data": { 03:15:28 DEBUG "resultType": "vector", 03:15:28 INFO [http_cmd]: http status code OK 03:15:28 DEBUG "resultType": "vector", 03:15:28 DEBUG { 03:15:28 DEBUG "resultType": "vector", 03:15:28 DEBUG { 03:15:28 DEBUG --- http response --- 03:15:28 DEBUG "result": [ 03:15:28 DEBUG "result": [ 03:15:28 DEBUG "data": { 03:15:28 DEBUG "data": { 03:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713060928" 03:15:28 DEBUG --- status code --- 03:15:28 INFO Thread started 03:15:28 DEBUG "resultType": "vector", 03:15:28 DEBUG "resultType": "vector", 03:15:28 DEBUG ----- output ----- 03:15:28 DEBUG ----- output ----- 03:15:28 DEBUG "resultType": "vector", 03:15:28 DEBUG "resultType": "vector", 03:15:28 DEBUG ----- output ----- 03:15:28 DEBUG "resultType": "vector", 03:15:28 DEBUG "result": [ 03:15:28 INFO [http_cmd]: http status code OK 03:15:28 DEBUG --- status code --- 03:15:28 DEBUG "result": [ 03:15:28 DEBUG "status": "success", 03:15:28 DEBUG "result": [ 03:15:28 DEBUG "metric": { 03:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:15:28 DEBUG ----- output ----- 03:15:28 DEBUG { 03:15:28 DEBUG "resultType": "vector", 03:15:28 DEBUG "resultType": "vector", 03:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713060928" 03:15:28 DEBUG "result": [ 03:15:28 DEBUG "result": [ 03:15:28 INFO Thread started 03:15:28 DEBUG { 03:15:28 DEBUG { 03:15:28 DEBUG "result": [ 03:15:28 DEBUG "result": [ 03:15:28 DEBUG { 03:15:28 DEBUG "status": "success", 03:15:28 DEBUG { 03:15:28 DEBUG --- status code --- 03:15:28 INFO [http_cmd]: http status code OK 03:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:28 DEBUG { 03:15:28 DEBUG "data": { 03:15:28 DEBUG { 03:15:28 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 03:15:28 DEBUG }, 03:15:28 DEBUG { 03:15:28 DEBUG "metric": { 03:15:28 DEBUG "result": [ 03:15:28 DEBUG "result": [ 03:15:28 DEBUG --- http response --- 03:15:28 DEBUG { 03:15:28 DEBUG "metric": { 03:15:28 DEBUG { 03:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713060928" 03:15:28 INFO Thread started 03:15:28 DEBUG "status": "success", 03:15:28 DEBUG { 03:15:28 DEBUG { 03:15:28 DEBUG "result": [ 03:15:28 DEBUG "data": { 03:15:28 DEBUG "metric": { 03:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:28 DEBUG --- status code --- 03:15:28 DEBUG --- http response --- 03:15:28 DEBUG "metric": { 03:15:28 DEBUG "resultType": "vector", 03:15:28 DEBUG "metric": { 03:15:28 DEBUG { 03:15:28 DEBUG "value": [ 03:15:28 DEBUG "status": "success", 03:15:28 DEBUG "pod": "ds-idrepo-1" 03:15:28 DEBUG { 03:15:28 DEBUG { 03:15:28 INFO [http_cmd]: http status code OK 03:15:28 DEBUG "status": "success", 03:15:28 DEBUG "pod": "ds-idrepo-1" 03:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:15:28 DEBUG "metric": { 03:15:28 ERROR [http_cmd]: ERROR 03:15:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713060928" 03:15:28 DEBUG "data": { 03:15:28 INFO Thread started 03:15:28 DEBUG "metric": { 03:15:28 DEBUG "metric": { 03:15:28 DEBUG { 03:15:28 DEBUG "resultType": "vector", 03:15:28 DEBUG "instance": "10.142.0.107:9100" 03:15:28 DEBUG --- http response --- 03:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:28 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 03:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 03:15:28 DEBUG "result": [ 03:15:28 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 03:15:28 DEBUG }, 03:15:28 DEBUG 03:15:28 DEBUG "data": { 03:15:28 DEBUG }, 03:15:28 DEBUG "metric": { 03:15:28 DEBUG "metric": { 03:15:28 DEBUG --- status code --- 03:15:28 DEBUG "data": { 03:15:28 DEBUG }, 03:15:28 DEBUG ----- output ----- 03:15:28 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 03:15:28 ERROR --- status code --- 03:15:28 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:15:28 INFO All threads has been started 127.0.0.1 - - [14/Apr/2024 03:15:28] "GET /monitoring/start HTTP/1.1" 200 - 03:15:28 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 03:15:28 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:15:28 DEBUG "metric": { 03:15:28 DEBUG "result": [ 03:15:28 DEBUG }, 03:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:15:28 DEBUG --- http response --- 03:15:28 DEBUG }, 03:15:28 DEBUG ----- output ----- 03:15:28 DEBUG { 03:15:28 DEBUG "value": [ 03:15:28 DEBUG "metric": { 03:15:28 DEBUG "instance": "10.142.0.107:9100" 03:15:28 DEBUG 03:15:28 DEBUG "value": [ 03:15:28 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 03:15:28 DEBUG "instance": "10.142.0.107:9100" 03:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:28 DEBUG "resultType": "vector", 03:15:28 DEBUG "result": [ 03:15:28 DEBUG "value": [ 03:15:28 INFO [http_cmd]: http status code OK 03:15:28 DEBUG }, 03:15:28 DEBUG "resultType": "vector", 03:15:28 ERROR --- http response --- 03:15:28 DEBUG }, 03:15:28 DEBUG }, 03:15:28 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:15:28 DEBUG { 03:15:28 DEBUG "value": [ 03:15:28 DEBUG ----- output ----- 03:15:28 DEBUG "value": [ 03:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:15:28 DEBUG { 03:15:28 DEBUG "metric": { 03:15:28 DEBUG 03:15:28 DEBUG "resultType": "vector", 03:15:28 DEBUG }, 03:15:28 DEBUG 03:15:28 DEBUG 03:15:28 DEBUG }, 03:15:28 DEBUG }, 03:15:28 DEBUG "value": [ 03:15:28 DEBUG { 03:15:28 DEBUG { 03:15:28 DEBUG 03:15:28 DEBUG --- status code --- 03:15:28 DEBUG "value": [ 03:15:28 DEBUG "result": [ 03:15:28 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:15:28 INFO [http_cmd]: http status code OK 03:15:28 DEBUG "value": [ 03:15:28 DEBUG "value": [ 03:15:28 DEBUG 03:15:28 DEBUG }, 03:15:28 DEBUG 03:15:28 DEBUG { 03:15:28 DEBUG 03:15:28 DEBUG ----- output ----- 03:15:28 DEBUG "status": "success", 03:15:28 DEBUG "pod": "ds-idrepo-1" 03:15:28 DEBUG 03:15:28 DEBUG "result": [ 03:15:28 DEBUG "value": [ 03:15:28 DEBUG [...] 03:15:28 DEBUG 03:15:28 DEBUG "value": [ 03:15:28 DEBUG 03:15:28 DEBUG --- http response --- 03:15:28 DEBUG "status": "success", 03:15:28 DEBUG "metric": { 03:15:28 DEBUG 03:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:28 DEBUG 03:15:28 DEBUG 03:15:28 DEBUG { 03:15:28 DEBUG --- status code --- 03:15:28 DEBUG 03:15:28 DEBUG "metric": { 03:15:28 DEBUG 03:15:28 DEBUG "value": [ 03:15:28 DEBUG 03:15:28 DEBUG "status": "success", 03:15:28 DEBUG 03:15:28 DEBUG { 03:15:28 DEBUG "data": { 03:15:28 DEBUG }, 03:15:28 DEBUG 03:15:28 DEBUG { 03:15:28 DEBUG 03:15:28 DEBUG 03:15:28 DEBUG 03:15:28 DEBUG [...] 03:15:28 DEBUG 03:15:28 DEBUG "data": { 03:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:15:28 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:15:28 DEBUG 03:15:28 DEBUG --- http response --- 03:15:28 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:15:28 DEBUG 03:15:28 DEBUG "metric": { 03:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:28 DEBUG 03:15:28 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:15:28 DEBUG 03:15:28 DEBUG [...] 03:15:28 INFO [http_cmd]: http status code OK 03:15:28 DEBUG 03:15:28 DEBUG "data": { 03:15:28 DEBUG 03:15:28 DEBUG "status": "success", 03:15:28 DEBUG "resultType": "vector", 03:15:28 DEBUG "value": [ 03:15:28 DEBUG [...] 03:15:28 DEBUG "metric": { 03:15:28 DEBUG 03:15:28 DEBUG 03:15:28 DEBUG 03:15:28 DEBUG 03:15:28 DEBUG 03:15:28 DEBUG "resultType": "vector", 03:15:28 DEBUG ----- output ----- 03:15:28 DEBUG }, 03:15:28 DEBUG [...] 03:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:15:28 DEBUG [...] 03:15:28 DEBUG "pod": "ds-idrepo-1" 03:15:28 DEBUG --- http response --- 03:15:28 DEBUG 03:15:28 DEBUG }, 03:15:28 DEBUG 03:15:28 DEBUG 03:15:28 DEBUG --- status code --- 03:15:28 DEBUG [...] 03:15:28 DEBUG "resultType": "vector", 03:15:28 DEBUG [...] 03:15:28 DEBUG "data": { 03:15:28 DEBUG "result": [ 03:15:28 DEBUG 03:15:28 DEBUG 03:15:28 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:15:28 DEBUG 03:15:28 DEBUG 03:15:28 DEBUG 03:15:28 DEBUG 03:15:28 DEBUG 03:15:28 DEBUG "result": [ 03:15:28 DEBUG { 03:15:28 DEBUG "value": [ 03:15:28 DEBUG 03:15:28 DEBUG ----- output ----- 03:15:28 DEBUG 03:15:28 DEBUG }, 03:15:28 DEBUG [...] 03:15:28 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:15:28 DEBUG "value": [ 03:15:28 DEBUG 03:15:28 DEBUG 03:15:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:28 DEBUG 03:15:28 DEBUG "result": [ 03:15:28 DEBUG 03:15:28 DEBUG "resultType": "vector", 03:15:28 DEBUG { 03:15:28 DEBUG 03:15:28 DEBUG 03:15:28 DEBUG }, 03:15:28 DEBUG [...] 03:15:28 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:15:28 DEBUG [...] 03:15:28 DEBUG 03:15:28 DEBUG [...] 03:15:28 DEBUG { 03:15:28 DEBUG "status": "success", 03:15:28 DEBUG 03:15:28 DEBUG 03:15:28 DEBUG { 03:15:28 DEBUG 03:15:28 DEBUG "value": [ 03:15:28 DEBUG 03:15:28 DEBUG 03:15:28 DEBUG 03:15:28 DEBUG [...] 03:15:28 DEBUG --- http response --- 03:15:28 DEBUG 03:15:28 DEBUG { 03:15:28 DEBUG 03:15:28 DEBUG "result": [ 03:15:28 DEBUG "metric": { 03:15:28 DEBUG 03:15:28 DEBUG 03:15:28 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:15:28 DEBUG "value": [ 03:15:28 DEBUG }, 03:15:28 DEBUG 03:15:28 DEBUG "pod": "ds-idrepo-0" 03:15:28 DEBUG 03:15:28 DEBUG "metric": { 03:15:28 DEBUG "data": { 03:15:28 DEBUG 03:15:28 DEBUG 03:15:28 DEBUG "status": "success", 03:15:28 DEBUG 03:15:28 DEBUG 03:15:28 DEBUG 03:15:28 DEBUG 03:15:28 DEBUG 03:15:28 DEBUG 03:15:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:15:28 DEBUG 03:15:28 DEBUG "metric": { 03:15:28 DEBUG 03:15:28 DEBUG { 03:15:28 DEBUG "pod": "ds-idrepo-1" 03:15:28 DEBUG [...] 03:15:28 DEBUG 03:15:28 DEBUG }, 03:15:28 DEBUG 03:15:28 DEBUG "value": [ 03:15:28 DEBUG 03:15:28 DEBUG }, 03:15:28 DEBUG 03:15:28 DEBUG "pod": "ds-idrepo-1" 03:15:28 DEBUG "resultType": "vector", 03:15:28 DEBUG 03:15:28 DEBUG "pod": "ds-idrepo-0" 03:15:28 DEBUG "data": { 03:15:28 DEBUG "resultType": "vector", 03:15:28 DEBUG 03:15:28 DEBUG 03:15:28 DEBUG 03:15:28 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:15:28 DEBUG 03:15:28 DEBUG ----- output ----- 03:15:28 DEBUG "instance": "10.142.0.43:9100" 03:15:28 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:15:28 DEBUG }, 03:15:28 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 03:15:28 DEBUG }, 03:15:28 DEBUG 03:15:28 DEBUG 03:15:28 DEBUG "value": [ 03:15:28 DEBUG 03:15:28 DEBUG 1713060928, 03:15:28 DEBUG 03:15:28 DEBUG "value": [ 03:15:28 DEBUG 03:15:28 DEBUG }, 03:15:28 DEBUG "result": [ 03:15:28 DEBUG [...] 03:15:28 DEBUG }, 03:15:28 DEBUG "pod": "ds-idrepo-1" 03:15:28 DEBUG "result": [ 03:15:28 DEBUG 03:15:28 DEBUG "pod": "ds-idrepo-1" 03:15:28 DEBUG [...] 03:15:28 DEBUG }, 03:15:28 DEBUG 03:15:28 DEBUG { 03:15:28 DEBUG }, 03:15:28 DEBUG "metric": { 03:15:28 DEBUG "value": [ 03:15:28 DEBUG }, 03:15:28 DEBUG "value": [ 03:15:28 DEBUG 03:15:28 DEBUG 03:15:28 DEBUG 1713060928, 03:15:28 DEBUG 03:15:28 DEBUG "3881230336" 03:15:28 DEBUG "instance": "10.142.0.43:9100" 03:15:28 DEBUG 1713060928, 03:15:28 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 03:15:28 DEBUG "value": [ 03:15:28 DEBUG { 03:15:28 DEBUG 03:15:28 DEBUG "value": [ 03:15:28 DEBUG }, 03:15:28 DEBUG { 03:15:28 DEBUG [...] 03:15:28 DEBUG }, 03:15:28 DEBUG 03:15:28 DEBUG "value": [ 03:15:28 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:15:28 DEBUG "status": "success", 03:15:28 DEBUG "data": { 03:15:28 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:15:28 DEBUG 03:15:28 DEBUG "value": [ 03:15:28 DEBUG 03:15:28 DEBUG 03:15:28 DEBUG "instance": "10.142.0.43:9100" 03:15:28 DEBUG "0.027365352132061347" 03:15:28 DEBUG [...] 03:15:28 DEBUG ] 03:15:28 DEBUG }, 03:15:28 DEBUG "0" 03:15:28 DEBUG }, 03:15:28 DEBUG 03:15:28 DEBUG "metric": { 03:15:28 DEBUG 03:15:28 DEBUG 03:15:28 DEBUG "value": [ 03:15:28 DEBUG "metric": { 03:15:28 DEBUG 03:15:28 DEBUG "value": [ 03:15:28 DEBUG 03:15:28 DEBUG 1713060928, 03:15:28 DEBUG }, 03:15:28 DEBUG "value": [ 03:15:28 DEBUG "resultType": "vector", 03:15:28 DEBUG }, 03:15:28 DEBUG 03:15:28 DEBUG 1713060928, 03:15:28 DEBUG 03:15:28 DEBUG "pod": "ds-idrepo-0" 03:15:28 DEBUG }, 03:15:28 DEBUG ] 03:15:28 DEBUG 03:15:28 DEBUG } 03:15:28 DEBUG "value": [ 03:15:28 DEBUG 1713060928, 03:15:28 DEBUG "value": [ 03:15:28 DEBUG 1713060928, 03:15:28 DEBUG "pod": "ds-idrepo-1" 03:15:28 DEBUG 1713060928, 03:15:28 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:15:28 DEBUG 1713060928, 03:15:28 DEBUG "instance": "10.142.0.107:9100" 03:15:28 DEBUG 03:15:28 DEBUG 1713060928, 03:15:28 DEBUG 03:15:28 DEBUG "0" 03:15:28 DEBUG "value": [ 03:15:28 DEBUG 1713060928, 03:15:28 DEBUG "result": [ 03:15:28 DEBUG "value": [ 03:15:28 DEBUG 03:15:28 DEBUG "0" 03:15:28 DEBUG 03:15:28 DEBUG }, 03:15:28 DEBUG "value": [ 03:15:28 DEBUG } 03:15:28 DEBUG 03:15:28 DEBUG ] 03:15:28 DEBUG } 03:15:28 DEBUG "5509.166666666667" 03:15:28 DEBUG 03:15:28 DEBUG "0" 03:15:28 DEBUG }, 03:15:28 DEBUG "0" 03:15:28 DEBUG }, 03:15:28 DEBUG "0" 03:15:28 DEBUG }, 03:15:28 DEBUG 03:15:28 DEBUG "5245.565729653582" 03:15:28 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:15:28 DEBUG ] 03:15:28 DEBUG 1713060928, 03:15:28 DEBUG "0.00033333333333333365" 03:15:28 DEBUG { 03:15:28 DEBUG 03:15:28 DEBUG [...] 03:15:28 DEBUG ] 03:15:28 DEBUG [...] 03:15:28 DEBUG "value": [ 03:15:28 DEBUG 1713060928, 03:15:28 DEBUG ] 03:15:28 DEBUG 03:15:28 DEBUG ] 03:15:28 DEBUG } 03:15:28 DEBUG ] 03:15:28 DEBUG } 03:15:28 DEBUG 03:15:28 DEBUG "value": [ 03:15:28 DEBUG ] 03:15:28 DEBUG "value": [ 03:15:28 DEBUG 1713060928, 03:15:28 DEBUG "value": [ 03:15:28 DEBUG "pod": "ds-idrepo-0" 03:15:28 DEBUG ] 03:15:28 DEBUG }, 03:15:28 DEBUG } 03:15:28 DEBUG "0" 03:15:28 DEBUG ] 03:15:28 DEBUG "metric": { 03:15:28 DEBUG 03:15:28 DEBUG 03:15:28 DEBUG } 03:15:28 DEBUG 03:15:28 DEBUG 1713060928, 03:15:28 DEBUG "0" 03:15:28 DEBUG } 03:15:28 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:15:28 DEBUG } 03:15:28 DEBUG ] 03:15:28 DEBUG ] 03:15:28 DEBUG [...] 03:15:28 DEBUG 03:15:28 DEBUG } 03:15:28 DEBUG ] 03:15:28 DEBUG "0" 03:15:28 DEBUG 03:15:28 DEBUG }, 03:15:28 DEBUG } 03:15:28 DEBUG "value": [ 03:15:28 DEBUG 1713060928, 03:15:28 DEBUG ] 03:15:28 DEBUG } 03:15:28 DEBUG "instance": "10.142.0.107:9100" 03:15:28 DEBUG 03:15:28 DEBUG 03:15:28 DEBUG ] 03:15:28 DEBUG 03:15:28 DEBUG "4252.9" 03:15:28 DEBUG ] 03:15:28 DEBUG } 03:15:28 DEBUG }, 03:15:28 DEBUG ] 03:15:28 DEBUG } 03:15:28 DEBUG ] 03:15:28 DEBUG } 03:15:28 DEBUG 03:15:28 DEBUG ] 03:15:28 DEBUG } 03:15:28 DEBUG ] 03:15:28 DEBUG 03:15:28 DEBUG "value": [ 03:15:28 DEBUG ] 03:15:28 DEBUG ] 03:15:28 DEBUG "0" 03:15:28 DEBUG } 03:15:28 DEBUG ] 03:15:28 DEBUG }, 03:15:28 DEBUG [...] 03:15:28 DEBUG 03:15:28 DEBUG } 03:15:28 DEBUG 03:15:28 DEBUG ] 03:15:28 DEBUG } 03:15:28 DEBUG "value": [ 03:15:28 DEBUG } 03:15:28 DEBUG } 03:15:28 DEBUG 03:15:28 DEBUG } 03:15:28 DEBUG 03:15:28 DEBUG } 03:15:28 DEBUG ] 03:15:28 DEBUG } 03:15:28 DEBUG 03:15:28 DEBUG 1713060928, 03:15:28 DEBUG } 03:15:28 DEBUG } 03:15:28 DEBUG ] 03:15:28 DEBUG ] 03:15:28 DEBUG } 03:15:28 DEBUG "value": [ 03:15:28 DEBUG 03:15:28 DEBUG 03:15:28 DEBUG } 03:15:28 DEBUG "pod": "ds-cts-1" 03:15:28 DEBUG } 03:15:28 DEBUG ] 03:15:28 DEBUG 1713060928, 03:15:28 DEBUG } 03:15:28 DEBUG } 03:15:28 DEBUG 03:15:28 DEBUG [...] 03:15:28 DEBUG } 03:15:28 DEBUG } 03:15:28 DEBUG ] 03:15:28 DEBUG [...] 03:15:28 DEBUG "0" 03:15:28 DEBUG } 03:15:28 DEBUG } 03:15:28 DEBUG } 03:15:28 DEBUG } 03:15:28 DEBUG } 03:15:28 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:15:28 DEBUG 03:15:28 DEBUG 03:15:28 DEBUG }, 03:15:28 DEBUG ] 03:15:28 DEBUG } 03:15:28 DEBUG "0" 03:15:28 DEBUG 03:15:28 DEBUG 03:15:28 DEBUG } 03:15:28 DEBUG } 03:15:28 DEBUG 03:15:28 DEBUG ] 03:15:28 DEBUG ] 03:15:28 DEBUG } 03:15:28 DEBUG }, 03:15:28 DEBUG 03:15:28 DEBUG 03:15:28 DEBUG "value": [ 03:15:28 DEBUG } 03:15:28 DEBUG } 03:15:28 DEBUG ] 03:15:28 DEBUG "pod": "ds-idrepo-0" 03:15:28 DEBUG 03:15:28 DEBUG } 03:15:28 DEBUG 03:15:28 DEBUG } 03:15:28 DEBUG } 03:15:28 DEBUG "value": [ 03:15:28 DEBUG [...] 03:15:28 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:15:28 DEBUG 1713060928, 03:15:28 DEBUG } 03:15:28 DEBUG } 03:15:28 DEBUG }, 03:15:28 DEBUG 03:15:28 DEBUG 03:15:28 DEBUG ] 03:15:28 DEBUG } 03:15:28 DEBUG 1713060928, 03:15:28 DEBUG 03:15:28 DEBUG }, 03:15:28 DEBUG "0" 03:15:28 DEBUG ] 03:15:28 DEBUG "value": [ 03:15:28 DEBUG "pod": "ds-cts-1" 03:15:28 DEBUG "instance": "10.142.0.43:9100" 03:15:28 DEBUG } 03:15:28 DEBUG "1" 03:15:28 DEBUG 03:15:28 DEBUG "value": [ 03:15:28 DEBUG ] 03:15:28 DEBUG } 03:15:28 DEBUG 1713060928, 03:15:28 DEBUG }, 03:15:28 DEBUG }, 03:15:28 DEBUG } 03:15:28 DEBUG ] 03:15:28 DEBUG 03:15:28 DEBUG 1713060928, 03:15:28 DEBUG } 03:15:28 DEBUG } 03:15:28 DEBUG "0" 03:15:28 DEBUG "value": [ 03:15:28 DEBUG "value": [ 03:15:28 DEBUG } 03:15:28 DEBUG "instance": "10.142.0.43:9100" 03:15:28 DEBUG "0.014" 03:15:28 DEBUG ] 03:15:28 DEBUG ] 03:15:28 DEBUG 1713060928, 03:15:28 DEBUG 1713060928, 03:15:28 DEBUG ] 03:15:28 DEBUG }, 03:15:28 DEBUG ] 03:15:28 DEBUG } 03:15:28 DEBUG } 03:15:28 DEBUG "0" 03:15:28 DEBUG "12644563968" 03:15:28 DEBUG } 03:15:28 DEBUG "value": [ 03:15:28 DEBUG } 03:15:28 DEBUG } 03:15:28 DEBUG ] 03:15:28 DEBUG ] 03:15:28 DEBUG ] 03:15:28 DEBUG } 03:15:28 DEBUG 1713060928, 03:15:28 DEBUG ] 03:15:28 DEBUG } 03:15:28 DEBUG } 03:15:28 DEBUG } 03:15:28 DEBUG "0.00030000000000001136" 03:15:28 DEBUG } 03:15:28 DEBUG } 03:15:28 DEBUG ] 03:15:28 DEBUG ] 03:15:28 DEBUG ] 03:15:28 DEBUG } 03:15:28 DEBUG } 03:15:28 DEBUG } 03:15:28 DEBUG } 03:15:28 DEBUG } 03:15:28 DEBUG } 03:15:28 DEBUG ] 03:15:28 DEBUG } 03:15:28 DEBUG } 03:15:28 INFO [loop_until]: OK (rc = 0) 03:15:28 DEBUG --- stdout --- 03:15:28 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 6m 8869Mi am-7d597cf7d6-8vr8f 6m 8340Mi am-7d597cf7d6-9wz9h 5m 8972Mi ds-cts-0 13m 7752Mi ds-cts-1 12m 7950Mi ds-cts-2 12m 8027Mi ds-idrepo-0 9m 4296Mi ds-idrepo-1 9m 4284Mi ds-idrepo-2 9m 4102Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 985Mi idm-b85f9d978-rl76f 4m 983Mi lodemon-766dc9b5df-2jmd4 2m 55Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 1m 3611Mi 03:15:28 DEBUG --- stderr --- 03:15:28 DEBUG 03:15:28 INFO [loop_until]: OK (rc = 0) 03:15:28 DEBUG --- stdout --- 03:15:28 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 66m 0% 4821Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 73m 0% 10182Mi 9% gke-xlou-cdm-default-pool-1b0769c8-nhlf 61m 0% 10893Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 91m 0% 3066Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 65m 0% 6313Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 137m 0% 11260Mi 10% gke-xlou-cdm-default-pool-1b0769c8-xnmp 61m 0% 2902Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 62m 0% 9765Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 63m 0% 5485Mi 4% gke-xlou-cdm-ds-0f601751-j488 65m 0% 9446Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 72m 0% 2202Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 84m 0% 10486Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 64m 0% 5738Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 66m 0% 7360Mi 12% 03:15:28 DEBUG --- stderr --- 03:15:28 DEBUG 03:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713060988" 03:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713060988" 03:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713060988" 03:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713060988" 03:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713060988" 03:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713060988" 03:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713060988" 03:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713060988" 03:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713060988" 03:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713060988" 03:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713060988" 03:16:28 INFO [http_cmd]: http status code OK 03:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713060988" 03:16:28 DEBUG --- status code --- 03:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:28 DEBUG --- http response --- 03:16:28 INFO [http_cmd]: http status code OK 03:16:28 INFO [http_cmd]: http status code OK 03:16:28 DEBUG --- status code --- 03:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:16:28 INFO [http_cmd]: http status code OK 03:16:28 DEBUG --- status code --- 03:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:28 DEBUG ----- output ----- 03:16:28 DEBUG --- status code --- 03:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:28 DEBUG --- http response --- 03:16:28 DEBUG { 03:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:28 DEBUG --- http response --- 03:16:28 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:16:28 DEBUG "status": "success", 03:16:28 DEBUG --- http response --- 03:16:28 INFO [http_cmd]: http status code OK 03:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:16:28 INFO [http_cmd]: http status code OK 03:16:28 INFO [http_cmd]: http status code OK 03:16:28 INFO [http_cmd]: http status code OK 03:16:28 DEBUG "data": { 03:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 03:16:28 DEBUG --- status code --- 03:16:28 INFO [http_cmd]: http status code OK 03:16:28 DEBUG ----- output ----- 03:16:28 DEBUG --- status code --- 03:16:28 DEBUG --- status code --- 03:16:28 DEBUG --- status code --- 03:16:28 DEBUG "resultType": "vector", 03:16:28 DEBUG ----- output ----- 03:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:28 DEBUG --- status code --- 03:16:28 DEBUG { 03:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:28 DEBUG "result": [ 03:16:28 DEBUG { 03:16:28 DEBUG --- http response --- 03:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:28 DEBUG "status": "success", 03:16:28 DEBUG --- http response --- 03:16:28 DEBUG --- http response --- 03:16:28 DEBUG --- http response --- 03:16:28 DEBUG { 03:16:28 INFO [http_cmd]: http status code OK 03:16:28 DEBUG "status": "success", 03:16:28 DEBUG --- http response --- 03:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 03:16:28 DEBUG "data": { 03:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:16:28 DEBUG "metric": { 03:16:28 DEBUG --- status code --- 03:16:28 DEBUG "data": { 03:16:28 DEBUG ----- output ----- 03:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:16:28 DEBUG "resultType": "vector", 03:16:28 DEBUG ----- output ----- 03:16:28 DEBUG ----- output ----- 03:16:28 DEBUG ----- output ----- 03:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713060988" 03:16:28 DEBUG "pod": "ds-idrepo-1" 03:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:28 DEBUG "resultType": "vector", 03:16:28 DEBUG { 03:16:28 DEBUG ----- output ----- 03:16:28 INFO [http_cmd]: http status code OK 03:16:28 DEBUG "result": [ 03:16:28 INFO [http_cmd]: http status code OK 03:16:28 DEBUG { 03:16:28 DEBUG { 03:16:28 DEBUG { 03:16:28 DEBUG }, 03:16:28 DEBUG --- http response --- 03:16:28 DEBUG "result": [ 03:16:28 DEBUG "status": "success", 03:16:28 DEBUG { 03:16:28 DEBUG --- status code --- 03:16:28 DEBUG { 03:16:28 DEBUG --- status code --- 03:16:28 DEBUG "status": "success", 03:16:28 DEBUG "status": "success", 03:16:28 DEBUG "status": "success", 03:16:28 DEBUG "value": [ 03:16:28 DEBUG { 03:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:16:28 DEBUG "data": { 03:16:28 DEBUG "status": "success", 03:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:28 DEBUG "metric": { 03:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:28 DEBUG "data": { 03:16:28 DEBUG "data": { 03:16:28 DEBUG "data": { 03:16:28 DEBUG 03:16:28 DEBUG "metric": { 03:16:28 DEBUG ----- output ----- 03:16:28 DEBUG "resultType": "vector", 03:16:28 DEBUG "data": { 03:16:28 DEBUG --- http response --- 03:16:28 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 03:16:28 DEBUG --- http response --- 03:16:28 DEBUG "resultType": "vector", 03:16:28 DEBUG "resultType": "vector", 03:16:28 DEBUG "resultType": "vector", 03:16:28 DEBUG 03:16:28 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 03:16:28 DEBUG { 03:16:28 DEBUG "result": [ 03:16:28 DEBUG "resultType": "vector", 03:16:28 DEBUG }, 03:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:16:28 DEBUG "result": [ 03:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:16:28 DEBUG "result": [ 03:16:28 DEBUG "result": [ 03:16:28 DEBUG 03:16:28 DEBUG }, 03:16:28 DEBUG "status": "success", 03:16:28 DEBUG { 03:16:28 DEBUG "result": [ 03:16:28 DEBUG "value": [ 03:16:28 DEBUG ----- output ----- 03:16:28 DEBUG { 03:16:28 DEBUG ----- output ----- 03:16:28 DEBUG { 03:16:28 DEBUG { 03:16:28 DEBUG [...] 03:16:28 DEBUG "value": [ 03:16:28 DEBUG "data": { 03:16:28 DEBUG "metric": { 03:16:28 DEBUG { 03:16:28 DEBUG 03:16:28 DEBUG { 03:16:28 DEBUG "metric": { 03:16:28 DEBUG { 03:16:28 DEBUG "metric": { 03:16:28 DEBUG "metric": { 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG "resultType": "vector", 03:16:28 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 03:16:28 DEBUG "metric": { 03:16:28 DEBUG 03:16:28 DEBUG "status": "success", 03:16:28 DEBUG "pod": "ds-idrepo-1" 03:16:28 DEBUG "status": "success", 03:16:28 DEBUG "instance": "10.142.0.107:9100" 03:16:28 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713060988" 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG "result": [ 03:16:28 DEBUG }, 03:16:28 DEBUG "instance": "10.142.0.107:9100" 03:16:28 DEBUG 03:16:28 DEBUG "data": { 03:16:28 DEBUG }, 03:16:28 DEBUG "data": { 03:16:28 DEBUG }, 03:16:28 DEBUG }, 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG { 03:16:28 DEBUG "value": [ 03:16:28 DEBUG }, 03:16:28 DEBUG [...] 03:16:28 DEBUG "resultType": "vector", 03:16:28 DEBUG "value": [ 03:16:28 DEBUG "resultType": "vector", 03:16:28 DEBUG "value": [ 03:16:28 DEBUG "value": [ 03:16:28 DEBUG "pod": "ds-idrepo-0" 03:16:28 DEBUG [...] 03:16:28 DEBUG "metric": { 03:16:28 DEBUG 03:16:28 DEBUG "value": [ 03:16:28 DEBUG 03:16:28 DEBUG "result": [ 03:16:28 DEBUG 03:16:28 DEBUG "result": [ 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG }, 03:16:28 DEBUG 03:16:28 DEBUG "pod": "ds-idrepo-1" 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG { 03:16:28 DEBUG 03:16:28 DEBUG { 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG "value": [ 03:16:28 DEBUG 03:16:28 DEBUG }, 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG "metric": { 03:16:28 DEBUG 03:16:28 DEBUG "metric": { 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG 1713060988, 03:16:28 DEBUG 03:16:28 DEBUG "value": [ 03:16:28 DEBUG [...] 03:16:28 DEBUG 03:16:28 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:16:28 DEBUG "pod": "ds-idrepo-1" 03:16:28 DEBUG [...] 03:16:28 DEBUG "pod": "ds-idrepo-1" 03:16:28 DEBUG [...] 03:16:28 DEBUG [...] 03:16:28 DEBUG "0" 03:16:28 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG [...] 03:16:28 DEBUG }, 03:16:28 DEBUG }, 03:16:28 DEBUG 03:16:28 DEBUG }, 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG ] 03:16:28 DEBUG }, 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG "value": [ 03:16:28 DEBUG "value": [ 03:16:28 DEBUG 03:16:28 DEBUG "value": [ 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 INFO [http_cmd]: http status code OK 03:16:28 DEBUG } 03:16:28 DEBUG "value": [ 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG 1713060988, 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG --- status code --- 03:16:28 DEBUG ] 03:16:28 DEBUG 1713060988, 03:16:28 DEBUG [...] 03:16:28 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 03:16:28 DEBUG 03:16:28 DEBUG "0.27134840974742874" 03:16:28 DEBUG 03:16:28 DEBUG "pod": "ds-idrepo-0" 03:16:28 DEBUG 03:16:28 DEBUG "instance": "10.142.0.43:9100" 03:16:28 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:28 DEBUG } 03:16:28 DEBUG "3533.230571693858" 03:16:28 DEBUG 03:16:28 DEBUG }, 03:16:28 DEBUG "instance": "10.142.0.43:9100" 03:16:28 DEBUG ] 03:16:28 DEBUG 03:16:28 DEBUG }, 03:16:28 DEBUG 03:16:28 DEBUG }, 03:16:28 DEBUG }, 03:16:28 DEBUG --- http response --- 03:16:28 DEBUG } 03:16:28 DEBUG ] 03:16:28 DEBUG 03:16:28 DEBUG "value": [ 03:16:28 DEBUG }, 03:16:28 DEBUG } 03:16:28 DEBUG [...] 03:16:28 DEBUG "value": [ 03:16:28 DEBUG [...] 03:16:28 DEBUG "value": [ 03:16:28 DEBUG "value": [ 03:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:16:28 DEBUG } 03:16:28 DEBUG 03:16:28 DEBUG 1713060988, 03:16:28 DEBUG "value": [ 03:16:28 DEBUG ] 03:16:28 DEBUG 03:16:28 DEBUG 1713060988, 03:16:28 DEBUG 03:16:28 DEBUG 1713060988, 03:16:28 DEBUG 1713060988, 03:16:28 DEBUG ----- output ----- 03:16:28 DEBUG ] 03:16:28 DEBUG "pod": "ds-idrepo-0" 03:16:28 DEBUG "0" 03:16:28 DEBUG 1713060988, 03:16:28 DEBUG } 03:16:28 DEBUG 03:16:28 DEBUG "0" 03:16:28 DEBUG 03:16:28 ERROR [http_cmd]: ERROR 03:16:28 DEBUG "5583.033333333334" 03:16:28 DEBUG "0" 03:16:28 DEBUG { 03:16:28 DEBUG } 03:16:28 DEBUG }, 03:16:28 DEBUG ] 03:16:28 DEBUG "0.00033333333333333365" 03:16:28 DEBUG } 03:16:28 DEBUG 03:16:28 DEBUG ] 03:16:28 DEBUG 03:16:28 ERROR --- status code --- 03:16:28 DEBUG ] 03:16:28 DEBUG ] 03:16:28 DEBUG "status": "success", 03:16:28 DEBUG } 03:16:28 DEBUG "value": [ 03:16:28 DEBUG 1713060988, 03:16:28 DEBUG "0" 03:16:28 DEBUG ] 03:16:28 DEBUG } 03:16:28 DEBUG "pod": "ds-cts-1" 03:16:28 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:16:28 DEBUG } 03:16:28 DEBUG } 03:16:28 DEBUG "data": { 03:16:28 DEBUG } 03:16:28 DEBUG "pod": "ds-idrepo-0" 03:16:28 DEBUG ] 03:16:28 DEBUG } 03:16:28 DEBUG ] 03:16:28 DEBUG }, 03:16:28 ERROR --- http response --- 03:16:28 DEBUG ] 03:16:28 DEBUG ] 03:16:28 DEBUG "resultType": "vector", 03:16:28 DEBUG ] 03:16:28 DEBUG }, 03:16:28 DEBUG } 03:16:28 DEBUG ] 03:16:28 DEBUG } 03:16:28 DEBUG "value": [ 03:16:28 DEBUG } 03:16:28 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:16:28 DEBUG } 03:16:28 DEBUG "result": [ 03:16:28 DEBUG } 03:16:28 DEBUG "value": [ 03:16:28 DEBUG ] 03:16:28 DEBUG } 03:16:28 DEBUG } 03:16:28 DEBUG 1713060988, 03:16:28 DEBUG } 03:16:28 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:16:28 DEBUG } 03:16:28 DEBUG { 03:16:28 DEBUG } 03:16:28 DEBUG 1713060988, 03:16:28 DEBUG } 03:16:28 DEBUG } 03:16:28 DEBUG "0" 03:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713060988" 03:16:28 DEBUG "metric": { 03:16:28 DEBUG "0" 03:16:28 DEBUG } 03:16:28 DEBUG ] 03:16:28 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:16:28 DEBUG ] 03:16:28 DEBUG } 03:16:28 DEBUG }, 03:16:28 DEBUG } 03:16:28 DEBUG ] 03:16:28 DEBUG "value": [ 03:16:28 DEBUG ] 03:16:28 DEBUG } 03:16:28 DEBUG 03:16:28 DEBUG } 03:16:28 DEBUG } 03:16:28 DEBUG 03:16:28 DEBUG } 03:16:28 DEBUG 03:16:28 DEBUG [...] 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:16:28 DEBUG }, 03:16:28 DEBUG "value": [ 03:16:28 DEBUG 1713060988, 03:16:28 DEBUG "0" 03:16:28 DEBUG ] 03:16:28 DEBUG } 03:16:28 DEBUG ] 03:16:28 DEBUG } 03:16:28 DEBUG } 03:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713060988" 03:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713060988" 03:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713060988" 03:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713060988" 03:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713060988" 03:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713060988" 03:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713060988" 03:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713060988" 03:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713060988" 03:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713060988" 03:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713060988" 03:16:28 INFO [http_cmd]: http status code OK 03:16:28 INFO [http_cmd]: http status code OK 03:16:28 DEBUG --- status code --- 03:16:28 INFO [http_cmd]: http status code OK 03:16:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713060988" 03:16:28 DEBUG --- status code --- 03:16:28 INFO [http_cmd]: http status code OK 03:16:28 INFO [http_cmd]: http status code OK 03:16:28 INFO [http_cmd]: http status code OK 03:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:28 INFO [http_cmd]: http status code OK 03:16:28 DEBUG --- status code --- 03:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:28 DEBUG --- status code --- 03:16:28 INFO [http_cmd]: http status code OK 03:16:28 DEBUG --- status code --- 03:16:28 DEBUG --- status code --- 03:16:28 DEBUG --- http response --- 03:16:28 DEBUG --- status code --- 03:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:28 DEBUG --- http response --- 03:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:28 DEBUG --- status code --- 03:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:16:28 DEBUG --- http response --- 03:16:28 DEBUG --- http response --- 03:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 03:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:28 DEBUG --- http response --- 03:16:28 DEBUG --- http response --- 03:16:28 DEBUG --- http response --- 03:16:28 DEBUG ----- output ----- 03:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 03:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 03:16:28 DEBUG ----- output ----- 03:16:28 DEBUG --- http response --- 03:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:16:28 INFO [http_cmd]: http status code OK 03:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:16:28 INFO [http_cmd]: http status code OK 03:16:28 INFO [http_cmd]: http status code OK 03:16:28 INFO [http_cmd]: http status code OK 03:16:28 DEBUG { 03:16:28 DEBUG ----- output ----- 03:16:28 DEBUG ----- output ----- 03:16:28 DEBUG { 03:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:16:28 DEBUG ----- output ----- 03:16:28 DEBUG ----- output ----- 03:16:28 DEBUG --- status code --- 03:16:28 DEBUG ----- output ----- 03:16:28 DEBUG --- status code --- 03:16:28 DEBUG --- status code --- 03:16:28 DEBUG --- status code --- 03:16:28 DEBUG "status": "success", 03:16:28 DEBUG { 03:16:28 DEBUG { 03:16:28 DEBUG "status": "success", 03:16:28 DEBUG ----- output ----- 03:16:28 DEBUG { 03:16:28 DEBUG { 03:16:28 INFO [http_cmd]: http status code OK 03:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:28 DEBUG { 03:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:28 DEBUG "data": { 03:16:28 DEBUG "status": "success", 03:16:28 INFO 03:16:28 DEBUG "status": "success", 03:16:28 DEBUG "data": { 03:16:28 DEBUG { 03:16:28 DEBUG "status": "success", 03:16:28 DEBUG "status": "success", 03:16:28 DEBUG --- status code --- 03:16:28 DEBUG --- http response --- 03:16:28 DEBUG "status": "success", 03:16:28 DEBUG --- http response --- 03:16:28 DEBUG --- http response --- 03:16:28 DEBUG --- http response --- 03:16:28 DEBUG "resultType": "vector", 03:16:28 DEBUG "data": { 03:16:28 INFO [loop_until]: kubectl --namespace=xlou top pods 03:16:28 DEBUG "data": { 03:16:28 DEBUG "resultType": "vector", 03:16:28 DEBUG "status": "success", 03:16:28 DEBUG "data": { 03:16:28 DEBUG "data": { 03:16:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:28 DEBUG "data": { 03:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:16:28 DEBUG "result": [ 03:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:16:28 DEBUG "resultType": "vector", 03:16:28 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:16:28 DEBUG "resultType": "vector", 03:16:28 DEBUG "result": [ 03:16:28 DEBUG "data": { 03:16:28 DEBUG "resultType": "vector", 03:16:28 DEBUG "resultType": "vector", 03:16:28 DEBUG --- http response --- 03:16:28 DEBUG "resultType": "vector", 03:16:28 DEBUG ----- output ----- 03:16:28 DEBUG ----- output ----- 03:16:28 DEBUG ----- output ----- 03:16:28 DEBUG { 03:16:28 DEBUG ----- output ----- 03:16:28 DEBUG "result": [ 03:16:28 DEBUG "result": [ 03:16:28 DEBUG { 03:16:28 DEBUG "resultType": "vector", 03:16:28 DEBUG "result": [ 03:16:28 DEBUG "result": [ 03:16:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:16:28 DEBUG "result": [ 03:16:28 DEBUG { 03:16:28 DEBUG { 03:16:28 DEBUG { 03:16:28 DEBUG "metric": { 03:16:28 DEBUG { 03:16:28 DEBUG { 03:16:28 DEBUG { 03:16:28 DEBUG "metric": { 03:16:28 DEBUG "result": [ 03:16:28 DEBUG { 03:16:28 DEBUG { 03:16:28 DEBUG ----- output ----- 03:16:28 DEBUG { 03:16:28 DEBUG "status": "success", 03:16:28 DEBUG "status": "success", 03:16:28 DEBUG "status": "success", 03:16:28 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 03:16:28 DEBUG "status": "success", 03:16:28 DEBUG "metric": { 03:16:28 DEBUG "metric": { 03:16:28 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 03:16:28 DEBUG { 03:16:28 DEBUG "metric": { 03:16:28 DEBUG "metric": { 03:16:28 DEBUG { 03:16:28 DEBUG "metric": { 03:16:28 DEBUG "data": { 03:16:28 DEBUG "data": { 03:16:28 DEBUG "data": { 03:16:28 DEBUG }, 03:16:28 DEBUG "data": { 03:16:28 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 03:16:28 DEBUG "pod": "ds-idrepo-1" 03:16:28 DEBUG }, 03:16:28 DEBUG "metric": { 03:16:28 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:16:28 DEBUG "instance": "10.142.0.107:9100" 03:16:28 DEBUG "status": "success", 03:16:28 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:16:28 DEBUG "resultType": "vector", 03:16:28 DEBUG "resultType": "vector", 03:16:28 DEBUG "resultType": "vector", 03:16:28 DEBUG "value": [ 03:16:28 DEBUG "resultType": "vector", 03:16:28 DEBUG }, 03:16:28 DEBUG }, 03:16:28 DEBUG "value": [ 03:16:28 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:16:28 DEBUG }, 03:16:28 DEBUG }, 03:16:28 DEBUG "data": { 03:16:28 DEBUG }, 03:16:28 DEBUG "result": [ 03:16:28 DEBUG "result": [ 03:16:28 DEBUG "result": [ 03:16:28 DEBUG 03:16:28 DEBUG "result": [ 03:16:28 DEBUG "value": [ 03:16:28 DEBUG "value": [ 03:16:28 DEBUG 03:16:28 DEBUG }, 03:16:28 DEBUG "value": [ 03:16:28 DEBUG "value": [ 03:16:28 DEBUG "resultType": "vector", 03:16:28 DEBUG "value": [ 03:16:28 DEBUG { 03:16:28 DEBUG { 03:16:28 DEBUG { 03:16:28 DEBUG 03:16:28 DEBUG { 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG "value": [ 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG "result": [ 03:16:28 DEBUG 03:16:28 DEBUG "metric": { 03:16:28 DEBUG "metric": { 03:16:28 DEBUG "metric": { 03:16:28 DEBUG 03:16:28 DEBUG "metric": { 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG { 03:16:28 DEBUG 03:16:28 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:16:28 DEBUG "instance": "10.142.0.107:9100" 03:16:28 DEBUG "instance": "10.142.0.107:9100" 03:16:28 DEBUG [...] 03:16:28 DEBUG "pod": "ds-idrepo-1" 03:16:28 DEBUG 03:16:28 DEBUG [...] 03:16:28 DEBUG [...] 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG "metric": { 03:16:28 DEBUG 03:16:28 DEBUG }, 03:16:28 DEBUG }, 03:16:28 DEBUG }, 03:16:28 DEBUG 03:16:28 DEBUG }, 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG [...] 03:16:28 DEBUG [...] 03:16:28 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:16:28 DEBUG [...] 03:16:28 DEBUG "value": [ 03:16:28 DEBUG "value": [ 03:16:28 DEBUG "value": [ 03:16:28 DEBUG 03:16:28 DEBUG "value": [ 03:16:28 DEBUG [...] 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG [...] 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG }, 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG "value": [ 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 03:16:28 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG }, 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG }, 03:16:28 DEBUG }, 03:16:28 DEBUG 03:16:28 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:16:28 DEBUG "instance": "10.142.0.43:9100" 03:16:28 DEBUG 03:16:28 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:16:28 DEBUG [...] 03:16:28 DEBUG [...] 03:16:28 DEBUG [...] 03:16:28 DEBUG "value": [ 03:16:28 DEBUG [...] 03:16:28 DEBUG "pod": "ds-cts-1" 03:16:28 DEBUG "value": [ 03:16:28 DEBUG "value": [ 03:16:28 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:16:28 DEBUG }, 03:16:28 DEBUG }, 03:16:28 DEBUG 03:16:28 DEBUG }, 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG 1713060988, 03:16:28 DEBUG 03:16:28 DEBUG }, 03:16:28 DEBUG 1713060988, 03:16:28 DEBUG 1713060988, 03:16:28 DEBUG }, 03:16:28 DEBUG "value": [ 03:16:28 DEBUG "value": [ 03:16:28 DEBUG [...] 03:16:28 DEBUG "value": [ 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG "3882729472" 03:16:28 DEBUG 03:16:28 DEBUG "value": [ 03:16:28 DEBUG "1143.1040084891895" 03:16:28 DEBUG "0.15909288349277123" 03:16:28 DEBUG "value": [ 03:16:28 DEBUG 1713060988, 03:16:28 DEBUG 1713060988, 03:16:28 DEBUG 03:16:28 DEBUG 1713060988, 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG 03:16:28 DEBUG ] 03:16:28 DEBUG 03:16:28 DEBUG 1713060988, 03:16:28 DEBUG ] 03:16:28 DEBUG ] 03:16:28 DEBUG 1713060988, 03:16:28 DEBUG "0" 03:16:28 DEBUG "2972.5666666666666" 03:16:28 DEBUG 03:16:28 DEBUG "0" 03:16:28 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:16:28 DEBUG "instance": "10.142.0.43:9100" 03:16:28 DEBUG "instance": "10.142.0.43:9100" 03:16:28 DEBUG } 03:16:28 DEBUG "pod": "ds-idrepo-0" 03:16:28 DEBUG "0" 03:16:28 DEBUG } 03:16:28 DEBUG } 03:16:28 DEBUG "0" 03:16:28 DEBUG ] 03:16:28 DEBUG ] 03:16:28 DEBUG 03:16:28 DEBUG ] 03:16:28 DEBUG }, 03:16:28 DEBUG }, 03:16:28 DEBUG }, 03:16:28 DEBUG ] 03:16:28 DEBUG }, 03:16:28 DEBUG ] 03:16:28 DEBUG } 03:16:28 DEBUG ] 03:16:28 DEBUG ] 03:16:28 DEBUG } 03:16:28 DEBUG } 03:16:28 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:16:28 DEBUG } 03:16:28 DEBUG "value": [ 03:16:28 DEBUG "value": [ 03:16:28 DEBUG "value": [ 03:16:28 DEBUG } 03:16:28 DEBUG "value": [ 03:16:28 DEBUG ] 03:16:28 DEBUG ] 03:16:28 DEBUG } 03:16:28 DEBUG } 03:16:28 DEBUG ] 03:16:28 DEBUG ] 03:16:28 DEBUG }, 03:16:28 DEBUG ] 03:16:28 DEBUG 1713060988, 03:16:28 DEBUG 1713060988, 03:16:28 DEBUG 1713060988, 03:16:28 DEBUG } 03:16:28 DEBUG 1713060988, 03:16:28 DEBUG } 03:16:28 DEBUG } 03:16:28 DEBUG } 03:16:28 DEBUG ] 03:16:28 DEBUG } 03:16:28 DEBUG } 03:16:28 DEBUG "value": [ 03:16:28 DEBUG } 03:16:28 DEBUG } 03:16:28 DEBUG "12647627776" 03:16:28 DEBUG "0.0006333333333332783" 03:16:28 DEBUG "0" 03:16:28 DEBUG } 03:16:28 DEBUG } 03:16:28 DEBUG } 03:16:28 DEBUG } 03:16:28 DEBUG } 03:16:28 DEBUG 1713060988, 03:16:28 DEBUG "1" 03:16:28 DEBUG ] 03:16:28 DEBUG ] 03:16:28 DEBUG ] 03:16:28 DEBUG } 03:16:28 DEBUG "0.014" 03:16:28 DEBUG ] 03:16:28 DEBUG } 03:16:28 DEBUG } 03:16:28 DEBUG } 03:16:28 DEBUG ] 03:16:28 DEBUG } 03:16:28 DEBUG ] 03:16:28 DEBUG ] 03:16:28 DEBUG ] 03:16:28 DEBUG } 03:16:28 DEBUG ] 03:16:28 DEBUG } 03:16:28 DEBUG } 03:16:28 DEBUG } 03:16:28 DEBUG ] 03:16:28 DEBUG } 03:16:28 DEBUG } 03:16:28 DEBUG } 03:16:28 DEBUG } 03:16:28 DEBUG } 03:16:28 DEBUG } 03:16:28 DEBUG } 03:16:28 INFO 03:16:28 INFO [loop_until]: kubectl --namespace=xlou top node 03:16:28 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:16:28 INFO [loop_until]: OK (rc = 0) 03:16:28 DEBUG --- stdout --- 03:16:28 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 6m 8875Mi am-7d597cf7d6-8vr8f 7m 8340Mi am-7d597cf7d6-9wz9h 20m 8971Mi ds-cts-0 476m 7761Mi ds-cts-1 115m 7951Mi ds-cts-2 185m 8047Mi ds-idrepo-0 595m 4297Mi ds-idrepo-1 386m 4284Mi ds-idrepo-2 393m 4204Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 8m 990Mi idm-b85f9d978-rl76f 5m 983Mi lodemon-766dc9b5df-2jmd4 2m 61Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 214m 3611Mi 03:16:28 DEBUG --- stderr --- 03:16:28 DEBUG 03:16:29 INFO [loop_until]: OK (rc = 0) 03:16:29 DEBUG --- stdout --- 03:16:29 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 69m 0% 4812Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 259m 0% 10189Mi 9% gke-xlou-cdm-default-pool-1b0769c8-nhlf 53m 0% 10886Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 93m 0% 3066Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 550m 1% 6280Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 632m 2% 11223Mi 10% gke-xlou-cdm-default-pool-1b0769c8-xnmp 73m 0% 2913Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 62m 0% 9759Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 443m 1% 5464Mi 4% gke-xlou-cdm-ds-0f601751-j488 247m 0% 9398Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 65m 0% 2182Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 98m 0% 10475Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 616m 2% 5699Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 295m 1% 7364Mi 12% 03:16:29 DEBUG --- stderr --- 03:16:29 DEBUG 03:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061048" 03:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713061048" 03:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713061048" 03:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061048" 03:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061048" 03:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713061048" 03:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061048" 03:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061048" 03:17:28 ERROR [http_cmd]: ERROR 03:17:28 ERROR --- status code --- 03:17:28 INFO [http_cmd]: http status code OK 03:17:28 INFO [http_cmd]: http status code OK 03:17:28 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:17:28 DEBUG --- status code --- 03:17:28 INFO [http_cmd]: http status code OK 03:17:28 DEBUG --- status code --- 03:17:28 ERROR --- http response --- 03:17:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:28 DEBUG --- status code --- 03:17:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:28 DEBUG --- http response --- 03:17:28 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:17:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:28 DEBUG --- http response --- 03:17:28 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:17:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:17:28 DEBUG --- http response --- 03:17:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:17:28 DEBUG ----- output ----- 03:17:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:17:28 DEBUG ----- output ----- 03:17:28 DEBUG { 03:17:28 DEBUG ----- output ----- 03:17:28 DEBUG { 03:17:28 DEBUG "status": "success", 03:17:28 DEBUG { 03:17:28 DEBUG "status": "success", 03:17:28 DEBUG "data": { 03:17:28 DEBUG "status": "success", 03:17:28 DEBUG "data": { 03:17:28 DEBUG "resultType": "vector", 03:17:28 DEBUG "data": { 03:17:28 DEBUG "resultType": "vector", 03:17:28 DEBUG "result": [ 03:17:28 DEBUG "resultType": "vector", 03:17:28 DEBUG "result": [ 03:17:28 DEBUG { 03:17:28 DEBUG "result": [ 03:17:28 DEBUG { 03:17:28 DEBUG "metric": { 03:17:28 DEBUG { 03:17:28 DEBUG "metric": { 03:17:28 DEBUG "pod": "ds-idrepo-1" 03:17:28 DEBUG "metric": { 03:17:28 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 03:17:28 DEBUG }, 03:17:28 DEBUG "instance": "10.142.0.107:9100" 03:17:28 DEBUG }, 03:17:28 DEBUG "value": [ 03:17:28 DEBUG }, 03:17:28 DEBUG "value": [ 03:17:28 DEBUG 03:17:28 DEBUG "value": [ 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG [...] 03:17:28 DEBUG 03:17:28 DEBUG [...] 03:17:28 DEBUG 03:17:28 DEBUG [...] 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG "pod": "ds-idrepo-0" 03:17:28 DEBUG 03:17:28 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 03:17:28 DEBUG }, 03:17:28 DEBUG "instance": "10.142.0.43:9100" 03:17:28 DEBUG }, 03:17:28 DEBUG "value": [ 03:17:28 DEBUG }, 03:17:28 DEBUG "value": [ 03:17:28 DEBUG 1713061048, 03:17:28 DEBUG "value": [ 03:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061048" 03:17:28 DEBUG 1713061048, 03:17:28 DEBUG "0" 03:17:28 DEBUG 1713061048, 03:17:28 DEBUG "0" 03:17:28 DEBUG ] 03:17:28 DEBUG "0" 03:17:28 DEBUG ] 03:17:28 DEBUG } 03:17:28 DEBUG ] 03:17:28 DEBUG } 03:17:28 DEBUG ] 03:17:28 DEBUG } 03:17:28 DEBUG ] 03:17:28 DEBUG } 03:17:28 DEBUG ] 03:17:28 DEBUG } 03:17:28 DEBUG } 03:17:28 DEBUG } 03:17:28 DEBUG } 03:17:28 DEBUG } 03:17:28 INFO [http_cmd]: http status code OK 03:17:28 DEBUG --- status code --- 03:17:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:28 DEBUG --- http response --- 03:17:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:17:28 DEBUG ----- output ----- 03:17:28 DEBUG { 03:17:28 DEBUG "status": "success", 03:17:28 DEBUG "data": { 03:17:28 DEBUG "resultType": "vector", 03:17:28 DEBUG "result": [ 03:17:28 DEBUG { 03:17:28 DEBUG "metric": { 03:17:28 DEBUG "pod": "ds-idrepo-1" 03:17:28 DEBUG }, 03:17:28 DEBUG "value": [ 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG [...] 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG "pod": "ds-cts-1" 03:17:28 DEBUG }, 03:17:28 DEBUG "value": [ 03:17:28 DEBUG 1713061048, 03:17:28 DEBUG "0" 03:17:28 DEBUG ] 03:17:28 DEBUG } 03:17:28 DEBUG ] 03:17:28 DEBUG } 03:17:28 DEBUG } 03:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061048" 03:17:28 INFO [http_cmd]: http status code OK 03:17:28 INFO [http_cmd]: http status code OK 03:17:28 DEBUG --- status code --- 03:17:28 INFO [http_cmd]: http status code OK 03:17:28 DEBUG --- status code --- 03:17:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:28 DEBUG --- status code --- 03:17:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:28 DEBUG --- http response --- 03:17:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:28 DEBUG --- http response --- 03:17:28 DEBUG --- http response --- 03:17:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:17:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:17:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:17:28 DEBUG ----- output ----- 03:17:28 DEBUG ----- output ----- 03:17:28 DEBUG ----- output ----- 03:17:28 DEBUG { 03:17:28 DEBUG { 03:17:28 DEBUG { 03:17:28 DEBUG "status": "success", 03:17:28 DEBUG "status": "success", 03:17:28 DEBUG "status": "success", 03:17:28 DEBUG "data": { 03:17:28 DEBUG "data": { 03:17:28 DEBUG "data": { 03:17:28 DEBUG "resultType": "vector", 03:17:28 DEBUG "resultType": "vector", 03:17:28 DEBUG "resultType": "vector", 03:17:28 DEBUG "result": [ 03:17:28 DEBUG "result": [ 03:17:28 DEBUG "result": [ 03:17:28 DEBUG { 03:17:28 DEBUG { 03:17:28 DEBUG { 03:17:28 DEBUG "metric": { 03:17:28 DEBUG "metric": { 03:17:28 DEBUG "metric": { 03:17:28 DEBUG "pod": "ds-idrepo-1" 03:17:28 DEBUG "pod": "ds-idrepo-1" 03:17:28 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 03:17:28 DEBUG }, 03:17:28 DEBUG }, 03:17:28 DEBUG }, 03:17:28 DEBUG "value": [ 03:17:28 DEBUG "value": [ 03:17:28 DEBUG "value": [ 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG [...] 03:17:28 DEBUG [...] 03:17:28 DEBUG [...] 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG "pod": "ds-idrepo-0" 03:17:28 DEBUG "pod": "ds-idrepo-0" 03:17:28 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:17:28 DEBUG }, 03:17:28 DEBUG }, 03:17:28 DEBUG }, 03:17:28 DEBUG "value": [ 03:17:28 DEBUG "value": [ 03:17:28 DEBUG "value": [ 03:17:28 DEBUG 1713061048, 03:17:28 DEBUG 1713061048, 03:17:28 DEBUG 1713061048, 03:17:28 DEBUG "0" 03:17:28 DEBUG "0" 03:17:28 DEBUG "0.2301427346170219" 03:17:28 DEBUG ] 03:17:28 DEBUG ] 03:17:28 DEBUG ] 03:17:28 DEBUG } 03:17:28 DEBUG } 03:17:28 DEBUG } 03:17:28 DEBUG ] 03:17:28 DEBUG ] 03:17:28 DEBUG ] 03:17:28 DEBUG } 03:17:28 DEBUG } 03:17:28 DEBUG } 03:17:28 DEBUG } 03:17:28 DEBUG } 03:17:28 DEBUG } 03:17:28 INFO [http_cmd]: http status code OK 03:17:28 DEBUG --- status code --- 03:17:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:28 DEBUG --- http response --- 03:17:28 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:17:28 INFO [http_cmd]: http status code OK 03:17:28 DEBUG --- status code --- 03:17:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:28 DEBUG --- http response --- 03:17:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:17:28 DEBUG ----- output ----- 03:17:28 DEBUG { 03:17:28 DEBUG "status": "success", 03:17:28 DEBUG "data": { 03:17:28 DEBUG "resultType": "vector", 03:17:28 DEBUG "result": [ 03:17:28 DEBUG { 03:17:28 DEBUG "metric": { 03:17:28 DEBUG "instance": "10.142.0.107:9100" 03:17:28 DEBUG }, 03:17:28 DEBUG "value": [ 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG [...] 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG "instance": "10.142.0.43:9100" 03:17:28 DEBUG }, 03:17:28 DEBUG "value": [ 03:17:28 DEBUG 1713061048, 03:17:28 DEBUG "5542.166666666667" 03:17:28 DEBUG ] 03:17:28 DEBUG } 03:17:28 DEBUG ] 03:17:28 DEBUG } 03:17:28 DEBUG } 03:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061048" 03:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061048" 03:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061048" 03:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061048" 03:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061048" 03:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061048" 03:17:28 INFO [http_cmd]: http status code OK 03:17:28 DEBUG --- status code --- 03:17:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:28 DEBUG --- http response --- 03:17:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:17:28 DEBUG ----- output ----- 03:17:28 DEBUG { 03:17:28 DEBUG "status": "success", 03:17:28 DEBUG "data": { 03:17:28 DEBUG "resultType": "vector", 03:17:28 DEBUG "result": [ 03:17:28 DEBUG { 03:17:28 DEBUG "metric": { 03:17:28 DEBUG "pod": "ds-idrepo-1" 03:17:28 DEBUG }, 03:17:28 DEBUG "value": [ 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG [...] 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG "pod": "ds-idrepo-0" 03:17:28 DEBUG }, 03:17:28 DEBUG "value": [ 03:17:28 DEBUG 1713061048, 03:17:28 DEBUG "0" 03:17:28 DEBUG ] 03:17:28 DEBUG } 03:17:28 DEBUG ] 03:17:28 DEBUG } 03:17:28 DEBUG } 03:17:28 INFO [http_cmd]: http status code OK 03:17:28 INFO [http_cmd]: http status code OK 03:17:28 DEBUG --- status code --- 03:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061048" 03:17:28 INFO [http_cmd]: http status code OK 03:17:28 INFO [http_cmd]: http status code OK 03:17:28 INFO [http_cmd]: http status code OK 03:17:28 DEBUG --- status code --- 03:17:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:28 DEBUG --- status code --- 03:17:28 DEBUG --- status code --- 03:17:28 DEBUG --- status code --- 03:17:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:28 DEBUG --- http response --- 03:17:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:28 DEBUG --- http response --- 03:17:28 DEBUG --- http response --- 03:17:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:17:28 DEBUG --- http response --- 03:17:28 DEBUG --- http response --- 03:17:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:17:28 DEBUG ----- output ----- 03:17:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:17:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:17:28 DEBUG ----- output ----- 03:17:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:17:28 DEBUG { 03:17:28 DEBUG ----- output ----- 03:17:28 DEBUG ----- output ----- 03:17:28 DEBUG { 03:17:28 DEBUG ----- output ----- 03:17:28 DEBUG "status": "success", 03:17:28 DEBUG { 03:17:28 DEBUG { 03:17:28 DEBUG "status": "success", 03:17:28 DEBUG { 03:17:28 DEBUG "data": { 03:17:28 DEBUG "status": "success", 03:17:28 DEBUG "status": "success", 03:17:28 DEBUG "data": { 03:17:28 DEBUG "status": "success", 03:17:28 DEBUG "resultType": "vector", 03:17:28 DEBUG "data": { 03:17:28 DEBUG "data": { 03:17:28 DEBUG "resultType": "vector", 03:17:28 DEBUG "data": { 03:17:28 DEBUG "result": [ 03:17:28 DEBUG "resultType": "vector", 03:17:28 DEBUG "resultType": "vector", 03:17:28 DEBUG "result": [ 03:17:28 DEBUG "resultType": "vector", 03:17:28 DEBUG { 03:17:28 DEBUG "result": [ 03:17:28 DEBUG "result": [ 03:17:28 DEBUG { 03:17:28 DEBUG "result": [ 03:17:28 DEBUG "metric": { 03:17:28 DEBUG { 03:17:28 DEBUG { 03:17:28 DEBUG "metric": { 03:17:28 DEBUG { 03:17:28 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:17:28 DEBUG "metric": { 03:17:28 DEBUG "metric": { 03:17:28 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:17:28 DEBUG "metric": { 03:17:28 DEBUG }, 03:17:28 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 03:17:28 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:17:28 DEBUG }, 03:17:28 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 03:17:28 DEBUG "value": [ 03:17:28 DEBUG }, 03:17:28 DEBUG }, 03:17:28 DEBUG "value": [ 03:17:28 DEBUG }, 03:17:28 DEBUG 03:17:28 DEBUG "value": [ 03:17:28 DEBUG "value": [ 03:17:28 DEBUG 03:17:28 DEBUG "value": [ 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG [...] 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG [...] 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG [...] 03:17:28 DEBUG [...] 03:17:28 DEBUG 03:17:28 DEBUG [...] 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:17:28 DEBUG 03:17:28 DEBUG }, 03:17:28 DEBUG "pod": "idm-b85f9d978-j5pbm" 03:17:28 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:17:28 DEBUG }, 03:17:28 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 03:17:28 DEBUG "value": [ 03:17:28 DEBUG }, 03:17:28 DEBUG }, 03:17:28 DEBUG "value": [ 03:17:28 DEBUG }, 03:17:28 DEBUG 1713061048, 03:17:28 DEBUG "value": [ 03:17:28 DEBUG "value": [ 03:17:28 DEBUG 1713061048, 03:17:28 DEBUG "value": [ 03:17:28 DEBUG "0" 03:17:28 DEBUG 1713061048, 03:17:28 DEBUG 1713061048, 03:17:28 DEBUG "0" 03:17:28 DEBUG 1713061048, 03:17:28 INFO [http_cmd]: http status code OK 03:17:28 DEBUG ] 03:17:28 DEBUG "0" 03:17:28 DEBUG "0" 03:17:28 DEBUG ] 03:17:28 DEBUG "0" 03:17:28 DEBUG --- status code --- 03:17:28 DEBUG } 03:17:28 DEBUG ] 03:17:28 DEBUG ] 03:17:28 DEBUG } 03:17:28 DEBUG ] 03:17:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:28 DEBUG ] 03:17:28 DEBUG } 03:17:28 DEBUG } 03:17:28 DEBUG ] 03:17:28 DEBUG } 03:17:28 DEBUG --- http response --- 03:17:28 DEBUG } 03:17:28 DEBUG ] 03:17:28 DEBUG ] 03:17:28 DEBUG } 03:17:28 DEBUG ] 03:17:28 DEBUG } 03:17:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:17:28 DEBUG } 03:17:28 DEBUG } 03:17:28 DEBUG } 03:17:28 DEBUG } 03:17:28 DEBUG ----- output ----- 03:17:28 DEBUG } 03:17:28 DEBUG } 03:17:28 DEBUG } 03:17:28 DEBUG { 03:17:28 DEBUG "status": "success", 03:17:28 DEBUG "data": { 03:17:28 DEBUG "resultType": "vector", 03:17:28 DEBUG "result": [ 03:17:28 DEBUG { 03:17:28 DEBUG "metric": { 03:17:28 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:17:28 DEBUG }, 03:17:28 DEBUG "value": [ 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG [...] 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG 03:17:28 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:17:28 DEBUG }, 03:17:28 DEBUG "value": [ 03:17:28 DEBUG 1713061048, 03:17:28 DEBUG "1" 03:17:28 DEBUG ] 03:17:28 DEBUG } 03:17:28 DEBUG ] 03:17:28 DEBUG } 03:17:28 DEBUG } 03:17:28 INFO 03:17:28 INFO [loop_until]: kubectl --namespace=xlou top pods 03:17:28 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713061048" 03:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061048" 03:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061048" 03:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061048" 03:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061048" 03:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061048" 03:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061048" 03:17:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061048" 03:17:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713061049" 03:17:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061049" 03:17:29 INFO [http_cmd]: http status code OK 03:17:29 DEBUG --- status code --- 03:17:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:29 DEBUG --- http response --- 03:17:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:17:29 DEBUG ----- output ----- 03:17:29 DEBUG { 03:17:29 DEBUG "status": "success", 03:17:29 DEBUG "data": { 03:17:29 DEBUG "resultType": "vector", 03:17:29 DEBUG "result": [ 03:17:29 DEBUG { 03:17:29 DEBUG "metric": { 03:17:29 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 03:17:29 DEBUG }, 03:17:29 DEBUG "value": [ 03:17:29 DEBUG 03:17:29 DEBUG 03:17:29 DEBUG 03:17:29 DEBUG [...] 03:17:29 DEBUG 03:17:29 DEBUG 03:17:29 DEBUG 03:17:29 DEBUG "pod": "idm-b85f9d978-j5pbm" 03:17:29 DEBUG }, 03:17:29 DEBUG "value": [ 03:17:29 INFO [http_cmd]: http status code OK 03:17:29 DEBUG 1713061048, 03:17:29 INFO [http_cmd]: http status code OK 03:17:29 INFO [http_cmd]: http status code OK 03:17:29 INFO [http_cmd]: http status code OK 03:17:29 INFO [http_cmd]: http status code OK 03:17:29 DEBUG --- status code --- 03:17:29 INFO [http_cmd]: http status code OK 03:17:29 DEBUG "1178.3639417330996" 03:17:29 INFO [http_cmd]: http status code OK 03:17:29 DEBUG --- status code --- 03:17:29 DEBUG --- status code --- 03:17:29 DEBUG --- status code --- 03:17:29 INFO [http_cmd]: http status code OK 03:17:29 DEBUG --- status code --- 03:17:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:29 DEBUG --- status code --- 03:17:29 DEBUG ] 03:17:29 DEBUG --- status code --- 03:17:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:29 INFO [http_cmd]: http status code OK 03:17:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:29 DEBUG --- status code --- 03:17:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:29 DEBUG --- http response --- 03:17:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:29 DEBUG } 03:17:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:29 DEBUG --- http response --- 03:17:29 DEBUG --- status code --- 03:17:29 DEBUG --- http response --- 03:17:29 DEBUG --- http response --- 03:17:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:29 DEBUG --- http response --- 03:17:29 DEBUG --- http response --- 03:17:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:17:29 DEBUG ] 03:17:29 DEBUG --- http response --- 03:17:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:17:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:17:29 DEBUG --- http response --- 03:17:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 03:17:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:17:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:17:29 DEBUG ----- output ----- 03:17:29 DEBUG } 03:17:29 DEBUG --- http response --- 03:17:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:17:29 DEBUG ----- output ----- 03:17:29 DEBUG ----- output ----- 03:17:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:17:29 DEBUG ----- output ----- 03:17:29 DEBUG ----- output ----- 03:17:29 DEBUG ----- output ----- 03:17:29 DEBUG { 03:17:29 DEBUG } 03:17:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:17:29 DEBUG ----- output ----- 03:17:29 DEBUG { 03:17:29 DEBUG { 03:17:29 DEBUG ----- output ----- 03:17:29 DEBUG { 03:17:29 DEBUG { 03:17:29 DEBUG { 03:17:29 DEBUG "status": "success", 03:17:29 DEBUG ----- output ----- 03:17:29 DEBUG { 03:17:29 DEBUG "status": "success", 03:17:29 DEBUG "status": "success", 03:17:29 DEBUG { 03:17:29 DEBUG "status": "success", 03:17:29 DEBUG "status": "success", 03:17:29 DEBUG "status": "success", 03:17:29 DEBUG "data": { 03:17:29 DEBUG { 03:17:29 DEBUG "status": "success", 03:17:29 DEBUG "data": { 03:17:29 DEBUG "data": { 03:17:29 DEBUG "status": "success", 03:17:29 DEBUG "data": { 03:17:29 DEBUG "data": { 03:17:29 DEBUG "data": { 03:17:29 DEBUG "resultType": "vector", 03:17:29 DEBUG "status": "success", 03:17:29 DEBUG "data": { 03:17:29 DEBUG "resultType": "vector", 03:17:29 DEBUG "resultType": "vector", 03:17:29 DEBUG "data": { 03:17:29 DEBUG "resultType": "vector", 03:17:29 DEBUG "resultType": "vector", 03:17:29 DEBUG "resultType": "vector", 03:17:29 DEBUG "result": [ 03:17:29 DEBUG "data": { 03:17:29 DEBUG "resultType": "vector", 03:17:29 DEBUG "result": [ 03:17:29 DEBUG "result": [ 03:17:29 DEBUG "resultType": "vector", 03:17:29 DEBUG "result": [ 03:17:29 DEBUG "result": [ 03:17:29 DEBUG "result": [ 03:17:29 DEBUG { 03:17:29 DEBUG "resultType": "vector", 03:17:29 DEBUG "result": [ 03:17:29 DEBUG { 03:17:29 DEBUG { 03:17:29 DEBUG "result": [ 03:17:29 DEBUG { 03:17:29 DEBUG { 03:17:29 DEBUG { 03:17:29 DEBUG "metric": { 03:17:29 DEBUG "result": [ 03:17:29 DEBUG { 03:17:29 DEBUG "metric": { 03:17:29 DEBUG "metric": { 03:17:29 DEBUG { 03:17:29 DEBUG "metric": { 03:17:29 DEBUG "metric": { 03:17:29 DEBUG "metric": { 03:17:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:17:29 DEBUG { 03:17:29 DEBUG "metric": { 03:17:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:17:29 DEBUG "instance": "10.142.0.107:9100" 03:17:29 DEBUG "metric": { 03:17:29 DEBUG "pod": "ds-idrepo-1" 03:17:29 DEBUG "instance": "10.142.0.107:9100" 03:17:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 03:17:29 DEBUG }, 03:17:29 DEBUG "metric": { 03:17:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:17:29 DEBUG }, 03:17:29 DEBUG }, 03:17:29 DEBUG "pod": "ds-idrepo-1" 03:17:29 DEBUG }, 03:17:29 DEBUG }, 03:17:29 DEBUG }, 03:17:29 DEBUG "value": [ 03:17:29 DEBUG "instance": "10.142.0.107:9100" 03:17:29 DEBUG }, 03:17:29 DEBUG "value": [ 03:17:29 DEBUG "value": [ 03:17:29 DEBUG }, 03:17:29 DEBUG "value": [ 03:17:29 DEBUG "value": [ 03:17:29 DEBUG "value": [ 03:17:29 DEBUG 03:17:29 DEBUG }, 03:17:29 DEBUG "value": [ 03:17:29 DEBUG 03:17:29 DEBUG 03:17:29 DEBUG "value": [ 03:17:29 DEBUG 03:17:29 DEBUG 03:17:29 DEBUG 03:17:29 DEBUG 03:17:29 DEBUG "value": [ 03:17:29 DEBUG 03:17:29 DEBUG 03:17:29 DEBUG 03:17:29 DEBUG 03:17:29 DEBUG 03:17:29 DEBUG 03:17:29 DEBUG 03:17:29 DEBUG 03:17:29 DEBUG 03:17:29 DEBUG 03:17:29 DEBUG 03:17:29 DEBUG 03:17:29 DEBUG 03:17:29 DEBUG 03:17:29 DEBUG 03:17:29 DEBUG 03:17:29 DEBUG [...] 03:17:29 DEBUG 03:17:29 DEBUG 03:17:29 DEBUG [...] 03:17:29 DEBUG [...] 03:17:29 DEBUG 03:17:29 DEBUG [...] 03:17:29 DEBUG [...] 03:17:29 DEBUG [...] 03:17:29 DEBUG 03:17:29 DEBUG 03:17:29 DEBUG [...] 03:17:29 DEBUG 03:17:29 DEBUG 03:17:29 DEBUG [...] 03:17:29 DEBUG 03:17:29 DEBUG 03:17:29 DEBUG 03:17:29 DEBUG 03:17:29 DEBUG [...] 03:17:29 DEBUG 03:17:29 DEBUG 03:17:29 DEBUG 03:17:29 DEBUG 03:17:29 DEBUG 03:17:29 DEBUG 03:17:29 DEBUG 03:17:29 DEBUG 03:17:29 DEBUG 03:17:29 DEBUG 03:17:29 DEBUG 03:17:29 DEBUG 03:17:29 DEBUG 03:17:29 DEBUG 03:17:29 DEBUG 03:17:29 DEBUG 03:17:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:17:29 DEBUG 03:17:29 DEBUG 03:17:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:17:29 DEBUG "instance": "10.142.0.43:9100" 03:17:29 DEBUG 03:17:29 DEBUG "pod": "ds-cts-1" 03:17:29 DEBUG "instance": "10.142.0.43:9100" 03:17:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:17:29 DEBUG }, 03:17:29 DEBUG 03:17:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:17:29 DEBUG }, 03:17:29 DEBUG }, 03:17:29 DEBUG "pod": "ds-idrepo-0" 03:17:29 DEBUG }, 03:17:29 DEBUG }, 03:17:29 DEBUG }, 03:17:29 DEBUG "value": [ 03:17:29 DEBUG "instance": "10.142.0.43:9100" 03:17:29 DEBUG }, 03:17:29 DEBUG "value": [ 03:17:29 DEBUG "value": [ 03:17:29 DEBUG }, 03:17:29 DEBUG "value": [ 03:17:29 DEBUG "value": [ 03:17:29 DEBUG "value": [ 03:17:29 DEBUG 1713061048, 03:17:29 DEBUG }, 03:17:29 DEBUG "value": [ 03:17:29 DEBUG 1713061048, 03:17:29 DEBUG 1713061048, 03:17:29 DEBUG "value": [ 03:17:29 DEBUG 1713061048, 03:17:29 DEBUG 1713061048, 03:17:29 DEBUG 1713061048, 03:17:29 DEBUG "0" 03:17:29 DEBUG "value": [ 03:17:29 DEBUG 1713061049, 03:17:29 DEBUG "0" 03:17:29 DEBUG "3076.7" 03:17:29 DEBUG 1713061048, 03:17:29 DEBUG "0" 03:17:29 DEBUG "12649761792" 03:17:29 DEBUG "4025081856" 03:17:29 DEBUG ] 03:17:29 DEBUG 1713061049, 03:17:29 DEBUG "0.014" 03:17:29 DEBUG ] 03:17:29 DEBUG ] 03:17:29 DEBUG "0" 03:17:29 DEBUG ] 03:17:29 DEBUG ] 03:17:29 DEBUG ] 03:17:29 DEBUG } 03:17:29 DEBUG "0.0001666666666665151" 03:17:29 DEBUG ] 03:17:29 DEBUG } 03:17:29 DEBUG } 03:17:29 DEBUG ] 03:17:29 DEBUG } 03:17:29 DEBUG } 03:17:29 DEBUG } 03:17:29 DEBUG ] 03:17:29 DEBUG ] 03:17:29 DEBUG } 03:17:29 DEBUG ] 03:17:29 DEBUG ] 03:17:29 DEBUG } 03:17:29 DEBUG ] 03:17:29 DEBUG ] 03:17:29 DEBUG ] 03:17:29 DEBUG } 03:17:29 DEBUG } 03:17:29 DEBUG ] 03:17:29 DEBUG } 03:17:29 DEBUG } 03:17:29 DEBUG ] 03:17:29 DEBUG } 03:17:29 DEBUG } 03:17:29 DEBUG } 03:17:29 DEBUG } 03:17:29 DEBUG ] 03:17:29 DEBUG } 03:17:29 DEBUG } 03:17:29 DEBUG } 03:17:29 DEBUG } 03:17:29 DEBUG } 03:17:29 DEBUG } 03:17:29 DEBUG } 03:17:29 DEBUG } 03:17:29 DEBUG } 03:17:29 DEBUG } 03:17:29 DEBUG } 03:17:29 INFO [loop_until]: OK (rc = 0) 03:17:29 DEBUG --- stdout --- 03:17:29 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 6m 8875Mi am-7d597cf7d6-8vr8f 7m 8341Mi am-7d597cf7d6-9wz9h 9m 8972Mi ds-cts-0 13m 7761Mi ds-cts-1 12m 7952Mi ds-cts-2 12m 8047Mi ds-idrepo-0 10m 4296Mi ds-idrepo-1 9m 4286Mi ds-idrepo-2 10m 4103Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 990Mi idm-b85f9d978-rl76f 4m 983Mi lodemon-766dc9b5df-2jmd4 1m 62Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 15m 3643Mi 03:17:29 DEBUG --- stderr --- 03:17:29 DEBUG 03:17:29 INFO 03:17:29 INFO [loop_until]: kubectl --namespace=xlou top node 03:17:29 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:17:29 INFO [loop_until]: OK (rc = 0) 03:17:29 DEBUG --- stdout --- 03:17:29 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 4818Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 73m 0% 10212Mi 9% gke-xlou-cdm-default-pool-1b0769c8-nhlf 51m 0% 10886Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 86m 0% 3072Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 67m 0% 6300Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 163m 0% 11255Mi 10% gke-xlou-cdm-default-pool-1b0769c8-xnmp 66m 0% 2919Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 60m 0% 9767Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 64m 0% 5458Mi 4% gke-xlou-cdm-ds-0f601751-j488 62m 0% 9411Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 64m 0% 2179Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 86m 0% 10483Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 64m 0% 5698Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 285m 1% 7598Mi 12% 03:17:29 DEBUG --- stderr --- 03:17:29 DEBUG 03:18:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061108" 03:18:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713061108" 03:18:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713061108" 03:18:28 INFO [http_cmd]: http status code OK 03:18:28 DEBUG --- status code --- 03:18:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:28 DEBUG --- http response --- 03:18:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:18:28 DEBUG ----- output ----- 03:18:28 DEBUG { 03:18:28 DEBUG "status": "success", 03:18:28 DEBUG "data": { 03:18:28 DEBUG "resultType": "vector", 03:18:28 DEBUG "result": [ 03:18:28 DEBUG { 03:18:28 DEBUG "metric": { 03:18:28 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 03:18:28 DEBUG }, 03:18:28 DEBUG "value": [ 03:18:28 DEBUG 03:18:28 DEBUG 03:18:28 DEBUG 03:18:28 DEBUG [...] 03:18:28 DEBUG 03:18:28 DEBUG 03:18:28 DEBUG 03:18:28 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:18:28 DEBUG }, 03:18:28 DEBUG "value": [ 03:18:28 DEBUG 1713061108, 03:18:28 DEBUG "3.5698748378280936" 03:18:28 DEBUG ] 03:18:28 DEBUG } 03:18:28 DEBUG ] 03:18:28 DEBUG } 03:18:28 DEBUG } 03:18:28 INFO [http_cmd]: http status code OK 03:18:28 DEBUG --- status code --- 03:18:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:28 DEBUG --- http response --- 03:18:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:18:28 DEBUG ----- output ----- 03:18:28 DEBUG { 03:18:28 DEBUG "status": "success", 03:18:28 DEBUG "data": { 03:18:28 DEBUG "resultType": "vector", 03:18:28 DEBUG "result": [ 03:18:28 DEBUG { 03:18:28 DEBUG "metric": { 03:18:28 DEBUG "instance": "10.142.0.107:9100" 03:18:28 DEBUG }, 03:18:28 DEBUG "value": [ 03:18:28 DEBUG 03:18:28 DEBUG 03:18:28 DEBUG 03:18:28 DEBUG [...] 03:18:28 DEBUG 03:18:28 DEBUG 03:18:28 DEBUG 03:18:28 DEBUG "instance": "10.142.0.43:9100" 03:18:28 DEBUG }, 03:18:28 DEBUG "value": [ 03:18:28 DEBUG 1713061108, 03:18:28 DEBUG "0.0003333333333333262" 03:18:28 DEBUG ] 03:18:28 DEBUG } 03:18:28 DEBUG ] 03:18:28 DEBUG } 03:18:28 DEBUG } 03:18:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061108" 03:18:28 ERROR [http_cmd]: ERROR 03:18:28 ERROR --- status code --- 03:18:28 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:18:28 ERROR --- http response --- 03:18:28 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:18:28 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:18:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061108" 03:18:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061108" 03:18:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061108" 03:18:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713061108" 03:18:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061108" 03:18:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061108" 03:18:28 INFO [http_cmd]: http status code OK 03:18:28 DEBUG --- status code --- 03:18:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:28 DEBUG --- http response --- 03:18:28 INFO [http_cmd]: http status code OK 03:18:28 INFO [http_cmd]: http status code OK 03:18:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:18:28 DEBUG --- status code --- 03:18:28 DEBUG --- status code --- 03:18:28 DEBUG ----- output ----- 03:18:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:28 DEBUG { 03:18:28 DEBUG --- http response --- 03:18:28 DEBUG --- http response --- 03:18:28 DEBUG "status": "success", 03:18:28 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:18:28 DEBUG "data": { 03:18:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:18:28 DEBUG "resultType": "vector", 03:18:28 DEBUG ----- output ----- 03:18:28 DEBUG "result": [ 03:18:28 DEBUG { 03:18:28 DEBUG { 03:18:28 DEBUG "status": "success", 03:18:28 DEBUG "metric": { 03:18:28 DEBUG "data": { 03:18:28 DEBUG "pod": "ds-idrepo-1" 03:18:28 DEBUG "resultType": "vector", 03:18:28 DEBUG }, 03:18:28 DEBUG "result": [ 03:18:28 DEBUG "value": [ 03:18:28 DEBUG { 03:18:28 DEBUG 03:18:28 DEBUG "metric": { 03:18:28 DEBUG 03:18:28 DEBUG "pod": "ds-idrepo-1" 03:18:28 DEBUG 03:18:28 DEBUG }, 03:18:28 DEBUG [...] 03:18:28 DEBUG "value": [ 03:18:28 DEBUG 03:18:28 DEBUG 03:18:28 DEBUG 03:18:28 DEBUG 03:18:28 DEBUG 03:18:28 DEBUG 03:18:28 DEBUG "pod": "ds-idrepo-0" 03:18:28 DEBUG [...] 03:18:28 DEBUG }, 03:18:28 INFO [http_cmd]: http status code OK 03:18:28 INFO [http_cmd]: http status code OK 03:18:28 DEBUG 03:18:28 INFO [http_cmd]: http status code OK 03:18:28 DEBUG "value": [ 03:18:28 DEBUG --- status code --- 03:18:28 DEBUG --- status code --- 03:18:28 DEBUG 03:18:28 DEBUG --- status code --- 03:18:28 DEBUG 1713061108, 03:18:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:28 DEBUG 03:18:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:28 DEBUG "0" 03:18:28 DEBUG --- http response --- 03:18:28 DEBUG --- http response --- 03:18:28 DEBUG "pod": "ds-cts-1" 03:18:28 DEBUG --- http response --- 03:18:28 DEBUG ] 03:18:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:18:28 DEBUG }, 03:18:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061108" 03:18:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:18:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:18:28 DEBUG } 03:18:28 DEBUG ----- output ----- 03:18:28 DEBUG "value": [ 03:18:28 DEBUG ----- output ----- 03:18:28 DEBUG ----- output ----- 03:18:28 DEBUG ] 03:18:28 DEBUG { 03:18:28 DEBUG 1713061108, 03:18:28 DEBUG { 03:18:28 DEBUG { 03:18:28 DEBUG } 03:18:28 DEBUG "status": "success", 03:18:28 DEBUG "0" 03:18:28 DEBUG "status": "success", 03:18:28 DEBUG "status": "success", 03:18:28 DEBUG } 03:18:28 DEBUG "data": { 03:18:28 DEBUG ] 03:18:28 DEBUG "data": { 03:18:28 DEBUG "data": { 03:18:28 DEBUG "resultType": "vector", 03:18:28 DEBUG } 03:18:28 DEBUG "resultType": "vector", 03:18:28 DEBUG "resultType": "vector", 03:18:28 DEBUG "result": [ 03:18:28 DEBUG ] 03:18:28 DEBUG "result": [ 03:18:28 DEBUG "result": [ 03:18:28 DEBUG { 03:18:28 DEBUG } 03:18:28 INFO [http_cmd]: http status code OK 03:18:28 DEBUG { 03:18:28 DEBUG { 03:18:28 DEBUG "metric": { 03:18:28 DEBUG } 03:18:28 DEBUG --- status code --- 03:18:28 DEBUG "metric": { 03:18:28 DEBUG "metric": { 03:18:28 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:18:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:28 DEBUG "pod": "ds-idrepo-1" 03:18:28 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 03:18:28 DEBUG }, 03:18:28 DEBUG --- http response --- 03:18:28 DEBUG }, 03:18:28 DEBUG }, 03:18:28 DEBUG "value": [ 03:18:28 DEBUG "value": [ 03:18:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:18:28 DEBUG "value": [ 03:18:28 DEBUG 03:18:28 DEBUG 03:18:28 DEBUG ----- output ----- 03:18:28 DEBUG 03:18:28 DEBUG 03:18:28 DEBUG 03:18:28 DEBUG { 03:18:28 DEBUG 03:18:28 DEBUG 03:18:28 DEBUG 03:18:28 DEBUG "status": "success", 03:18:28 DEBUG 03:18:28 DEBUG [...] 03:18:28 DEBUG [...] 03:18:28 DEBUG "data": { 03:18:28 DEBUG [...] 03:18:28 DEBUG 03:18:28 DEBUG 03:18:28 DEBUG "resultType": "vector", 03:18:28 DEBUG 03:18:28 DEBUG 03:18:28 DEBUG 03:18:28 DEBUG "result": [ 03:18:28 DEBUG 03:18:28 DEBUG 03:18:28 DEBUG 03:18:28 DEBUG { 03:18:28 DEBUG 03:18:28 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:18:28 DEBUG "pod": "ds-idrepo-0" 03:18:28 DEBUG "metric": { 03:18:28 DEBUG "pod": "login-ui-5fdf49fc66-2wnx7" 03:18:28 DEBUG }, 03:18:28 DEBUG }, 03:18:28 DEBUG "pod": "ds-idrepo-1" 03:18:28 DEBUG }, 03:18:28 DEBUG "value": [ 03:18:28 DEBUG "value": [ 03:18:28 DEBUG }, 03:18:28 DEBUG "value": [ 03:18:28 DEBUG 1713061108, 03:18:28 DEBUG 1713061108, 03:18:28 DEBUG "value": [ 03:18:28 DEBUG 1713061108, 03:18:28 DEBUG "0" 03:18:28 DEBUG "0" 03:18:28 DEBUG 03:18:28 DEBUG "0" 03:18:28 DEBUG ] 03:18:28 DEBUG ] 03:18:28 DEBUG 03:18:28 DEBUG ] 03:18:28 DEBUG } 03:18:28 DEBUG } 03:18:28 DEBUG 03:18:28 DEBUG } 03:18:28 DEBUG ] 03:18:28 DEBUG ] 03:18:28 DEBUG [...] 03:18:28 DEBUG ] 03:18:28 DEBUG } 03:18:28 DEBUG } 03:18:28 DEBUG 03:18:28 DEBUG } 03:18:28 DEBUG } 03:18:28 DEBUG } 03:18:28 DEBUG 03:18:28 DEBUG } 03:18:28 DEBUG 03:18:28 DEBUG "pod": "ds-idrepo-0" 03:18:28 DEBUG }, 03:18:28 DEBUG "value": [ 03:18:28 DEBUG 1713061108, 03:18:28 DEBUG "0" 03:18:28 DEBUG ] 03:18:28 DEBUG } 03:18:28 DEBUG ] 03:18:28 DEBUG } 03:18:28 DEBUG } 03:18:28 INFO [http_cmd]: http status code OK 03:18:28 DEBUG --- status code --- 03:18:28 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:28 DEBUG --- http response --- 03:18:28 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:18:28 DEBUG ----- output ----- 03:18:28 DEBUG { 03:18:28 DEBUG "status": "success", 03:18:28 DEBUG "data": { 03:18:28 DEBUG "resultType": "vector", 03:18:28 DEBUG "result": [ 03:18:29 DEBUG { 03:18:29 DEBUG "metric": { 03:18:29 DEBUG "instance": "10.142.0.107:9100" 03:18:29 DEBUG }, 03:18:29 DEBUG "value": [ 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG [...] 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG "instance": "10.142.0.43:9100" 03:18:29 DEBUG }, 03:18:29 DEBUG "value": [ 03:18:29 DEBUG 1713061108, 03:18:29 DEBUG "5650.3" 03:18:29 DEBUG ] 03:18:29 DEBUG } 03:18:29 DEBUG ] 03:18:29 DEBUG } 03:18:29 DEBUG } 03:18:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061109" 03:18:29 INFO [http_cmd]: http status code OK 03:18:29 DEBUG --- status code --- 03:18:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:29 DEBUG --- http response --- 03:18:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:18:29 DEBUG ----- output ----- 03:18:29 DEBUG { 03:18:29 DEBUG "status": "success", 03:18:29 DEBUG "data": { 03:18:29 DEBUG "resultType": "vector", 03:18:29 DEBUG "result": [ 03:18:29 DEBUG { 03:18:29 DEBUG "metric": { 03:18:29 DEBUG "pod": "ds-idrepo-1" 03:18:29 DEBUG }, 03:18:29 DEBUG "value": [ 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG [...] 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG "pod": "ds-idrepo-0" 03:18:29 DEBUG }, 03:18:29 DEBUG "value": [ 03:18:29 DEBUG 1713061109, 03:18:29 DEBUG "0" 03:18:29 DEBUG ] 03:18:29 DEBUG } 03:18:29 DEBUG ] 03:18:29 DEBUG } 03:18:29 DEBUG } 03:18:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061109" 03:18:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061109" 03:18:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061109" 03:18:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061109" 03:18:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061109" 03:18:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713061109" 03:18:29 INFO [http_cmd]: http status code OK 03:18:29 DEBUG --- status code --- 03:18:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:29 DEBUG --- http response --- 03:18:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:18:29 DEBUG ----- output ----- 03:18:29 DEBUG { 03:18:29 DEBUG "status": "success", 03:18:29 DEBUG "data": { 03:18:29 DEBUG "resultType": "vector", 03:18:29 DEBUG "result": [ 03:18:29 DEBUG { 03:18:29 DEBUG "metric": { 03:18:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:18:29 DEBUG }, 03:18:29 DEBUG "value": [ 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG [...] 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:18:29 DEBUG }, 03:18:29 DEBUG "value": [ 03:18:29 DEBUG 1713061109, 03:18:29 DEBUG "0" 03:18:29 DEBUG ] 03:18:29 DEBUG } 03:18:29 DEBUG ] 03:18:29 DEBUG } 03:18:29 DEBUG } 03:18:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713061109" 03:18:29 INFO [http_cmd]: http status code OK 03:18:29 INFO [http_cmd]: http status code OK 03:18:29 INFO [http_cmd]: http status code OK 03:18:29 INFO [http_cmd]: http status code OK 03:18:29 DEBUG --- status code --- 03:18:29 DEBUG --- status code --- 03:18:29 DEBUG --- status code --- 03:18:29 DEBUG --- status code --- 03:18:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:29 DEBUG --- http response --- 03:18:29 DEBUG --- http response --- 03:18:29 DEBUG --- http response --- 03:18:29 DEBUG --- http response --- 03:18:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:18:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:18:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:18:29 DEBUG ----- output ----- 03:18:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:18:29 DEBUG ----- output ----- 03:18:29 DEBUG ----- output ----- 03:18:29 DEBUG { 03:18:29 DEBUG ----- output ----- 03:18:29 DEBUG { 03:18:29 DEBUG { 03:18:29 DEBUG "status": "success", 03:18:29 DEBUG { 03:18:29 DEBUG "status": "success", 03:18:29 DEBUG "status": "success", 03:18:29 DEBUG "data": { 03:18:29 DEBUG "status": "success", 03:18:29 DEBUG "data": { 03:18:29 DEBUG "data": { 03:18:29 DEBUG "resultType": "vector", 03:18:29 DEBUG "data": { 03:18:29 DEBUG "resultType": "vector", 03:18:29 DEBUG "resultType": "vector", 03:18:29 DEBUG "result": [ 03:18:29 DEBUG "resultType": "vector", 03:18:29 DEBUG "result": [ 03:18:29 DEBUG "result": [ 03:18:29 DEBUG { 03:18:29 DEBUG "result": [ 03:18:29 DEBUG { 03:18:29 DEBUG { 03:18:29 DEBUG "metric": { 03:18:29 DEBUG { 03:18:29 DEBUG "metric": { 03:18:29 DEBUG "metric": { 03:18:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:18:29 DEBUG "metric": { 03:18:29 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 03:18:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:18:29 DEBUG }, 03:18:29 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 03:18:29 DEBUG }, 03:18:29 DEBUG }, 03:18:29 DEBUG "value": [ 03:18:29 DEBUG }, 03:18:29 DEBUG "value": [ 03:18:29 DEBUG "value": [ 03:18:29 DEBUG 03:18:29 DEBUG "value": [ 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG [...] 03:18:29 DEBUG 03:18:29 DEBUG [...] 03:18:29 DEBUG [...] 03:18:29 DEBUG 03:18:29 DEBUG [...] 03:18:29 INFO [http_cmd]: http status code OK 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG --- status code --- 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:18:29 DEBUG 03:18:29 DEBUG --- http response --- 03:18:29 DEBUG "pod": "login-ui-5fdf49fc66-2wnx7" 03:18:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:18:29 DEBUG }, 03:18:29 DEBUG "pod": "login-ui-5fdf49fc66-2wnx7" 03:18:29 DEBUG }, 03:18:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:18:29 DEBUG }, 03:18:29 DEBUG "value": [ 03:18:29 DEBUG }, 03:18:29 DEBUG "value": [ 03:18:29 DEBUG ----- output ----- 03:18:29 DEBUG "value": [ 03:18:29 DEBUG 1713061109, 03:18:29 DEBUG "value": [ 03:18:29 DEBUG 1713061109, 03:18:29 DEBUG { 03:18:29 DEBUG 1713061109, 03:18:29 DEBUG "2285.1666666666665" 03:18:29 DEBUG 1713061109, 03:18:29 DEBUG "0" 03:18:29 DEBUG "status": "success", 03:18:29 DEBUG "40781" 03:18:29 DEBUG ] 03:18:29 DEBUG "0.000020163249441259287" 03:18:29 DEBUG ] 03:18:29 DEBUG "data": { 03:18:29 DEBUG ] 03:18:29 DEBUG } 03:18:29 DEBUG ] 03:18:29 DEBUG } 03:18:29 DEBUG "resultType": "vector", 03:18:29 DEBUG } 03:18:29 DEBUG ] 03:18:29 DEBUG } 03:18:29 DEBUG ] 03:18:29 DEBUG "result": [ 03:18:29 DEBUG ] 03:18:29 DEBUG } 03:18:29 DEBUG ] 03:18:29 DEBUG } 03:18:29 DEBUG { 03:18:29 DEBUG } 03:18:29 DEBUG } 03:18:29 DEBUG } 03:18:29 DEBUG } 03:18:29 DEBUG "metric": { 03:18:29 DEBUG } 03:18:29 DEBUG } 03:18:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 03:18:29 DEBUG }, 03:18:29 DEBUG "value": [ 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG [...] 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:18:29 DEBUG }, 03:18:29 DEBUG "value": [ 03:18:29 DEBUG 1713061109, 03:18:29 DEBUG "4347514880" 03:18:29 DEBUG ] 03:18:29 DEBUG } 03:18:29 DEBUG ] 03:18:29 DEBUG } 03:18:29 DEBUG } 03:18:29 INFO [http_cmd]: http status code OK 03:18:29 DEBUG --- status code --- 03:18:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:29 DEBUG --- http response --- 03:18:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:18:29 DEBUG ----- output ----- 03:18:29 DEBUG { 03:18:29 DEBUG "status": "success", 03:18:29 DEBUG "data": { 03:18:29 DEBUG "resultType": "vector", 03:18:29 DEBUG "result": [ 03:18:29 DEBUG { 03:18:29 DEBUG "metric": { 03:18:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:18:29 DEBUG }, 03:18:29 DEBUG "value": [ 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG [...] 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:18:29 DEBUG }, 03:18:29 DEBUG "value": [ 03:18:29 DEBUG 1713061109, 03:18:29 DEBUG "0.014" 03:18:29 DEBUG ] 03:18:29 DEBUG } 03:18:29 DEBUG ] 03:18:29 DEBUG } 03:18:29 DEBUG } 03:18:29 INFO 03:18:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061109" 03:18:29 INFO [loop_until]: kubectl --namespace=xlou top pods 03:18:29 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:18:29 INFO [http_cmd]: http status code OK 03:18:29 DEBUG --- status code --- 03:18:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:29 DEBUG --- http response --- 03:18:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:18:29 DEBUG ----- output ----- 03:18:29 DEBUG { 03:18:29 DEBUG "status": "success", 03:18:29 DEBUG "data": { 03:18:29 DEBUG "resultType": "vector", 03:18:29 DEBUG "result": [ 03:18:29 DEBUG { 03:18:29 DEBUG "metric": { 03:18:29 DEBUG "pod": "ds-idrepo-1" 03:18:29 DEBUG }, 03:18:29 DEBUG "value": [ 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG [...] 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG "pod": "ds-idrepo-0" 03:18:29 DEBUG }, 03:18:29 DEBUG "value": [ 03:18:29 DEBUG 1713061109, 03:18:29 DEBUG "0" 03:18:29 DEBUG ] 03:18:29 DEBUG } 03:18:29 DEBUG ] 03:18:29 DEBUG } 03:18:29 DEBUG } 03:18:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061109" 03:18:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061109" 03:18:29 INFO [http_cmd]: http status code OK 03:18:29 DEBUG --- status code --- 03:18:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:29 DEBUG --- http response --- 03:18:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 03:18:29 DEBUG ----- output ----- 03:18:29 DEBUG { 03:18:29 DEBUG "status": "success", 03:18:29 DEBUG "data": { 03:18:29 DEBUG "resultType": "vector", 03:18:29 DEBUG "result": [ 03:18:29 DEBUG { 03:18:29 DEBUG "metric": { 03:18:29 DEBUG "pod": "ds-idrepo-1" 03:18:29 DEBUG }, 03:18:29 DEBUG "value": [ 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG [...] 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG "pod": "ds-cts-1" 03:18:29 DEBUG }, 03:18:29 DEBUG "value": [ 03:18:29 DEBUG 1713061109, 03:18:29 DEBUG "0.00002709722222222222" 03:18:29 DEBUG ] 03:18:29 DEBUG } 03:18:29 DEBUG ] 03:18:29 DEBUG } 03:18:29 DEBUG } 03:18:29 INFO [http_cmd]: http status code OK 03:18:29 DEBUG --- status code --- 03:18:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:29 DEBUG --- http response --- 03:18:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:18:29 DEBUG ----- output ----- 03:18:29 DEBUG { 03:18:29 DEBUG "status": "success", 03:18:29 DEBUG "data": { 03:18:29 DEBUG "resultType": "vector", 03:18:29 DEBUG "result": [ 03:18:29 DEBUG { 03:18:29 DEBUG "metric": { 03:18:29 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 03:18:29 DEBUG }, 03:18:29 DEBUG "value": [ 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG [...] 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG "pod": "login-ui-5fdf49fc66-2wnx7" 03:18:29 DEBUG }, 03:18:29 DEBUG "value": [ 03:18:29 DEBUG 1713061109, 03:18:29 DEBUG "0" 03:18:29 DEBUG ] 03:18:29 DEBUG } 03:18:29 DEBUG ] 03:18:29 DEBUG } 03:18:29 DEBUG } 03:18:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061109" 03:18:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061109" 03:18:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061109" 03:18:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061109" 03:18:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061109" 03:18:29 INFO [http_cmd]: http status code OK 03:18:29 DEBUG --- status code --- 03:18:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:29 DEBUG --- http response --- 03:18:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:18:29 DEBUG ----- output ----- 03:18:29 DEBUG { 03:18:29 DEBUG "status": "success", 03:18:29 DEBUG "data": { 03:18:29 DEBUG "resultType": "vector", 03:18:29 DEBUG "result": [ 03:18:29 DEBUG { 03:18:29 DEBUG "metric": { 03:18:29 DEBUG "instance": "10.142.0.107:9100" 03:18:29 DEBUG }, 03:18:29 DEBUG "value": [ 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG [...] 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG "instance": "10.142.0.43:9100" 03:18:29 DEBUG }, 03:18:29 DEBUG "value": [ 03:18:29 DEBUG 1713061109, 03:18:29 DEBUG "2952.1" 03:18:29 DEBUG ] 03:18:29 DEBUG } 03:18:29 DEBUG ] 03:18:29 DEBUG } 03:18:29 DEBUG } 03:18:29 INFO [http_cmd]: http status code OK 03:18:29 DEBUG --- status code --- 03:18:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:29 DEBUG --- http response --- 03:18:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:18:29 DEBUG ----- output ----- 03:18:29 DEBUG { 03:18:29 DEBUG "status": "success", 03:18:29 DEBUG "data": { 03:18:29 DEBUG "resultType": "vector", 03:18:29 DEBUG "result": [ 03:18:29 DEBUG { 03:18:29 DEBUG "metric": { 03:18:29 DEBUG "instance": "10.142.0.107:9100" 03:18:29 DEBUG }, 03:18:29 DEBUG "value": [ 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG [...] 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG "instance": "10.142.0.43:9100" 03:18:29 DEBUG }, 03:18:29 DEBUG "value": [ 03:18:29 DEBUG 1713061109, 03:18:29 DEBUG "12650359808" 03:18:29 DEBUG ] 03:18:29 DEBUG } 03:18:29 DEBUG ] 03:18:29 DEBUG } 03:18:29 DEBUG } 03:18:29 INFO [http_cmd]: http status code OK 03:18:29 INFO [http_cmd]: http status code OK 03:18:29 DEBUG --- status code --- 03:18:29 INFO [http_cmd]: http status code OK 03:18:29 DEBUG --- status code --- 03:18:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:29 DEBUG --- status code --- 03:18:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:29 DEBUG --- http response --- 03:18:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:29 DEBUG --- http response --- 03:18:29 DEBUG --- http response --- 03:18:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:18:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:18:29 DEBUG ----- output ----- 03:18:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:18:29 DEBUG ----- output ----- 03:18:29 DEBUG { 03:18:29 DEBUG ----- output ----- 03:18:29 DEBUG { 03:18:29 DEBUG "status": "success", 03:18:29 DEBUG { 03:18:29 DEBUG "status": "success", 03:18:29 DEBUG "data": { 03:18:29 DEBUG "status": "success", 03:18:29 DEBUG "data": { 03:18:29 DEBUG "resultType": "vector", 03:18:29 DEBUG "data": { 03:18:29 DEBUG "resultType": "vector", 03:18:29 DEBUG "result": [ 03:18:29 DEBUG "resultType": "vector", 03:18:29 DEBUG "result": [ 03:18:29 DEBUG { 03:18:29 DEBUG "result": [ 03:18:29 DEBUG { 03:18:29 DEBUG "metric": { 03:18:29 DEBUG { 03:18:29 DEBUG "metric": { 03:18:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:18:29 DEBUG "metric": { 03:18:29 DEBUG "instance": "10.142.0.107:9100" 03:18:29 DEBUG }, 03:18:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:18:29 DEBUG }, 03:18:29 DEBUG "value": [ 03:18:29 DEBUG }, 03:18:29 DEBUG "value": [ 03:18:29 DEBUG 03:18:29 DEBUG "value": [ 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG [...] 03:18:29 DEBUG 03:18:29 DEBUG [...] 03:18:29 DEBUG 03:18:29 DEBUG [...] 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG 03:18:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:18:29 DEBUG 03:18:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:18:29 DEBUG }, 03:18:29 DEBUG "instance": "10.142.0.43:9100" 03:18:29 DEBUG }, 03:18:29 DEBUG "value": [ 03:18:29 DEBUG }, 03:18:29 DEBUG "value": [ 03:18:29 DEBUG 1713061109, 03:18:29 DEBUG "value": [ 03:18:29 DEBUG 1713061109, 03:18:29 DEBUG "2285.3" 03:18:29 DEBUG 1713061109, 03:18:29 DEBUG "2285.2" 03:18:29 DEBUG ] 03:18:29 DEBUG "0.000333333333333267" 03:18:29 DEBUG ] 03:18:29 DEBUG } 03:18:29 DEBUG ] 03:18:29 DEBUG } 03:18:29 DEBUG ] 03:18:29 DEBUG } 03:18:29 DEBUG ] 03:18:29 DEBUG } 03:18:29 DEBUG ] 03:18:29 DEBUG } 03:18:29 DEBUG } 03:18:29 DEBUG } 03:18:29 DEBUG } 03:18:29 DEBUG } 03:18:29 INFO [loop_until]: OK (rc = 0) 03:18:29 DEBUG --- stdout --- 03:18:29 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 10867m 7313Mi am-7d597cf7d6-8vr8f 9234m 8368Mi am-7d597cf7d6-9wz9h 12457m 8968Mi ds-cts-0 4818m 7799Mi ds-cts-1 4592m 7985Mi ds-cts-2 4072m 8051Mi ds-idrepo-0 11213m 4291Mi ds-idrepo-1 1724m 4286Mi ds-idrepo-2 1885m 4102Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 10m 990Mi idm-b85f9d978-rl76f 16m 984Mi lodemon-766dc9b5df-2jmd4 2m 62Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 5752m 5305Mi 03:18:29 DEBUG --- stderr --- 03:18:29 DEBUG 03:18:29 INFO 03:18:29 INFO [loop_until]: kubectl --namespace=xlou top node 03:18:29 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:18:29 INFO [loop_until]: OK (rc = 0) 03:18:29 DEBUG --- stdout --- 03:18:29 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 72m 0% 4825Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5079m 17% 10287Mi 9% gke-xlou-cdm-default-pool-1b0769c8-nhlf 10434m 34% 9359Mi 8% gke-xlou-cdm-default-pool-1b0769c8-qkns 19971m 66% 3487Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 1926m 6% 6352Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 4902m 16% 11261Mi 10% gke-xlou-cdm-default-pool-1b0769c8-xnmp 72m 0% 2913Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 9867m 33% 9793Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1761m 5% 5525Mi 4% gke-xlou-cdm-ds-0f601751-j488 5050m 16% 9529Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 65m 0% 2185Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 13040m 43% 10505Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 11776m 39% 5744Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 7567m 47% 9715Mi 16% 03:18:29 DEBUG --- stderr --- 03:18:29 DEBUG 03:19:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061168" 03:19:28 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061168" 03:19:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061169" 03:19:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713061169" 03:19:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713061169" 03:19:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061169" 03:19:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061169" 03:19:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061169" 03:19:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713061169" 03:19:29 INFO [http_cmd]: http status code OK 03:19:29 INFO [http_cmd]: http status code OK 03:19:29 INFO [http_cmd]: http status code OK 03:19:29 DEBUG --- status code --- 03:19:29 INFO [http_cmd]: http status code OK 03:19:29 INFO [http_cmd]: http status code OK 03:19:29 DEBUG --- status code --- 03:19:29 DEBUG --- status code --- 03:19:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:29 DEBUG --- status code --- 03:19:29 DEBUG --- status code --- 03:19:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:29 DEBUG --- http response --- 03:19:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:29 DEBUG --- http response --- 03:19:29 DEBUG --- http response --- 03:19:29 DEBUG --- http response --- 03:19:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:19:29 DEBUG --- http response --- 03:19:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:19:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:19:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:19:29 DEBUG ----- output ----- 03:19:29 DEBUG ----- output ----- 03:19:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 03:19:29 DEBUG ----- output ----- 03:19:29 DEBUG ----- output ----- 03:19:29 DEBUG { 03:19:29 DEBUG { 03:19:29 DEBUG ----- output ----- 03:19:29 DEBUG { 03:19:29 DEBUG { 03:19:29 DEBUG "status": "success", 03:19:29 DEBUG "status": "success", 03:19:29 DEBUG { 03:19:29 DEBUG "status": "success", 03:19:29 DEBUG "status": "success", 03:19:29 DEBUG "data": { 03:19:29 DEBUG "data": { 03:19:29 DEBUG "status": "success", 03:19:29 DEBUG "data": { 03:19:29 DEBUG "data": { 03:19:29 DEBUG "resultType": "vector", 03:19:29 DEBUG "resultType": "vector", 03:19:29 DEBUG "data": { 03:19:29 DEBUG "resultType": "vector", 03:19:29 DEBUG "resultType": "vector", 03:19:29 DEBUG "result": [ 03:19:29 DEBUG "result": [ 03:19:29 DEBUG "resultType": "vector", 03:19:29 DEBUG "result": [ 03:19:29 DEBUG "result": [ 03:19:29 DEBUG { 03:19:29 DEBUG { 03:19:29 DEBUG "result": [ 03:19:29 DEBUG { 03:19:29 DEBUG { 03:19:29 DEBUG "metric": { 03:19:29 DEBUG "metric": { 03:19:29 DEBUG { 03:19:29 DEBUG "metric": { 03:19:29 DEBUG "metric": { 03:19:29 ERROR [http_cmd]: ERROR 03:19:29 INFO [http_cmd]: http status code OK 03:19:29 DEBUG "pod": "ds-idrepo-1" 03:19:29 INFO [http_cmd]: http status code OK 03:19:29 DEBUG "pod": "ds-idrepo-1" 03:19:29 DEBUG "metric": { 03:19:29 DEBUG "pod": "ds-idrepo-1" 03:19:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 03:19:29 ERROR --- status code --- 03:19:29 DEBUG --- status code --- 03:19:29 DEBUG }, 03:19:29 DEBUG --- status code --- 03:19:29 DEBUG }, 03:19:29 DEBUG "pod": "ds-cts-2" 03:19:29 DEBUG }, 03:19:29 DEBUG }, 03:19:29 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:19:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:29 DEBUG "value": [ 03:19:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:29 DEBUG "value": [ 03:19:29 DEBUG }, 03:19:29 DEBUG "value": [ 03:19:29 DEBUG "value": [ 03:19:29 ERROR --- http response --- 03:19:29 DEBUG --- http response --- 03:19:29 DEBUG 03:19:29 DEBUG --- http response --- 03:19:29 DEBUG 03:19:29 DEBUG "value": [ 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:19:29 DEBUG 03:19:29 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:19:29 INFO [http_cmd]: http status code OK 03:19:29 DEBUG 03:19:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:19:29 DEBUG 03:19:29 DEBUG --- status code --- 03:19:29 DEBUG 03:19:29 DEBUG ----- output ----- 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG [...] 03:19:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:29 DEBUG [...] 03:19:29 DEBUG { 03:19:29 DEBUG 03:19:29 DEBUG [...] 03:19:29 DEBUG [...] 03:19:29 DEBUG 03:19:29 DEBUG --- http response --- 03:19:29 DEBUG 03:19:29 DEBUG "status": "success", 03:19:29 DEBUG [...] 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:19:29 DEBUG "data": { 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG ----- output ----- 03:19:29 DEBUG "resultType": "vector", 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG "pod": "ds-cts-1" 03:19:29 DEBUG "pod": "ds-idrepo-0" 03:19:29 DEBUG { 03:19:29 DEBUG "result": [ 03:19:29 DEBUG 03:19:29 DEBUG "pod": "ds-idrepo-0" 03:19:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:19:29 DEBUG }, 03:19:29 DEBUG }, 03:19:29 DEBUG "status": "success", 03:19:29 DEBUG { 03:19:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:19:29 DEBUG }, 03:19:29 DEBUG }, 03:19:29 DEBUG "value": [ 03:19:29 DEBUG "value": [ 03:19:29 DEBUG "data": { 03:19:29 DEBUG "metric": { 03:19:29 DEBUG }, 03:19:29 DEBUG "value": [ 03:19:29 DEBUG "value": [ 03:19:29 DEBUG 1713061169, 03:19:29 DEBUG 1713061169, 03:19:29 DEBUG "resultType": "vector", 03:19:29 DEBUG "pod": "ds-idrepo-1" 03:19:29 DEBUG "value": [ 03:19:29 DEBUG 1713061168, 03:19:29 DEBUG 1713061169, 03:19:29 DEBUG "0" 03:19:29 DEBUG "0" 03:19:29 DEBUG "result": [ 03:19:29 DEBUG }, 03:19:29 DEBUG 1713061168, 03:19:29 DEBUG "0" 03:19:29 DEBUG "8.02374318556942" 03:19:29 DEBUG ] 03:19:29 DEBUG ] 03:19:29 DEBUG { 03:19:29 DEBUG "value": [ 03:19:29 DEBUG "294760771.76706576" 03:19:29 DEBUG ] 03:19:29 DEBUG ] 03:19:29 DEBUG } 03:19:29 DEBUG } 03:19:29 DEBUG "metric": { 03:19:29 DEBUG 03:19:29 DEBUG ] 03:19:29 DEBUG } 03:19:29 DEBUG } 03:19:29 DEBUG ] 03:19:29 DEBUG ] 03:19:29 DEBUG "instance": "10.142.0.107:9100" 03:19:29 DEBUG 03:19:29 DEBUG } 03:19:29 DEBUG ] 03:19:29 DEBUG ] 03:19:29 DEBUG } 03:19:29 DEBUG } 03:19:29 DEBUG }, 03:19:29 DEBUG 03:19:29 DEBUG ] 03:19:29 DEBUG } 03:19:29 DEBUG } 03:19:29 DEBUG } 03:19:29 DEBUG } 03:19:29 DEBUG "value": [ 03:19:29 DEBUG [...] 03:19:29 DEBUG } 03:19:29 DEBUG } 03:19:29 DEBUG } 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG } 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG [...] 03:19:29 DEBUG "pod": "ds-idrepo-0" 03:19:29 DEBUG 03:19:29 DEBUG }, 03:19:29 DEBUG 03:19:29 DEBUG "value": [ 03:19:29 DEBUG 03:19:29 DEBUG 1713061169, 03:19:29 DEBUG "instance": "10.142.0.43:9100" 03:19:29 DEBUG "0" 03:19:29 DEBUG }, 03:19:29 DEBUG ] 03:19:29 DEBUG "value": [ 03:19:29 DEBUG } 03:19:29 DEBUG 1713061169, 03:19:29 DEBUG ] 03:19:29 DEBUG "0.00033333333333333365" 03:19:29 DEBUG } 03:19:29 DEBUG ] 03:19:29 DEBUG } 03:19:29 DEBUG } 03:19:29 DEBUG ] 03:19:29 DEBUG } 03:19:29 DEBUG } 03:19:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061169" 03:19:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061169" 03:19:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061169" 03:19:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061169" 03:19:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061169" 03:19:29 INFO [http_cmd]: http status code OK 03:19:29 DEBUG --- status code --- 03:19:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:29 DEBUG --- http response --- 03:19:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:19:29 DEBUG ----- output ----- 03:19:29 DEBUG { 03:19:29 DEBUG "status": "success", 03:19:29 DEBUG "data": { 03:19:29 DEBUG "resultType": "vector", 03:19:29 DEBUG "result": [ 03:19:29 DEBUG { 03:19:29 DEBUG "metric": { 03:19:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:19:29 DEBUG }, 03:19:29 DEBUG "value": [ 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG [...] 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:19:29 DEBUG }, 03:19:29 DEBUG "value": [ 03:19:29 DEBUG 1713061169, 03:19:29 DEBUG "0" 03:19:29 DEBUG ] 03:19:29 DEBUG } 03:19:29 DEBUG ] 03:19:29 DEBUG } 03:19:29 DEBUG } 03:19:29 INFO [http_cmd]: http status code OK 03:19:29 INFO [http_cmd]: http status code OK 03:19:29 DEBUG --- status code --- 03:19:29 DEBUG --- status code --- 03:19:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:29 DEBUG --- http response --- 03:19:29 DEBUG --- http response --- 03:19:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:19:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:19:29 DEBUG ----- output ----- 03:19:29 DEBUG ----- output ----- 03:19:29 DEBUG { 03:19:29 DEBUG { 03:19:29 DEBUG "status": "success", 03:19:29 DEBUG "status": "success", 03:19:29 DEBUG "data": { 03:19:29 DEBUG "data": { 03:19:29 DEBUG "resultType": "vector", 03:19:29 DEBUG "resultType": "vector", 03:19:29 DEBUG "result": [ 03:19:29 DEBUG "result": [ 03:19:29 DEBUG { 03:19:29 DEBUG { 03:19:29 DEBUG "metric": { 03:19:29 DEBUG "metric": { 03:19:29 DEBUG "instance": "10.142.0.107:9100" 03:19:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:19:29 DEBUG }, 03:19:29 DEBUG }, 03:19:29 DEBUG "value": [ 03:19:29 DEBUG "value": [ 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG [...] 03:19:29 DEBUG [...] 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 INFO [http_cmd]: http status code OK 03:19:29 DEBUG 03:19:29 DEBUG "instance": "10.142.0.43:9100" 03:19:29 DEBUG --- status code --- 03:19:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:19:29 DEBUG }, 03:19:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:29 DEBUG }, 03:19:29 DEBUG "value": [ 03:19:29 DEBUG --- http response --- 03:19:29 DEBUG "value": [ 03:19:29 DEBUG 1713061169, 03:19:29 DEBUG 1713061169, 03:19:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:19:29 DEBUG "5590.733333333334" 03:19:29 DEBUG "138031" 03:19:29 DEBUG ----- output ----- 03:19:29 DEBUG ] 03:19:29 DEBUG ] 03:19:29 DEBUG { 03:19:29 INFO [http_cmd]: http status code OK 03:19:29 DEBUG } 03:19:29 DEBUG } 03:19:29 DEBUG "status": "success", 03:19:29 DEBUG --- status code --- 03:19:29 DEBUG ] 03:19:29 DEBUG ] 03:19:29 DEBUG "data": { 03:19:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:29 DEBUG } 03:19:29 DEBUG } 03:19:29 DEBUG "resultType": "vector", 03:19:29 DEBUG --- http response --- 03:19:29 DEBUG } 03:19:29 DEBUG } 03:19:29 DEBUG "result": [ 03:19:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:19:29 DEBUG { 03:19:29 DEBUG ----- output ----- 03:19:29 DEBUG "metric": { 03:19:29 DEBUG { 03:19:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:19:29 DEBUG "status": "success", 03:19:29 DEBUG }, 03:19:29 DEBUG "data": { 03:19:29 DEBUG "value": [ 03:19:29 DEBUG "resultType": "vector", 03:19:29 DEBUG 03:19:29 DEBUG "result": [ 03:19:29 DEBUG 03:19:29 DEBUG { 03:19:29 DEBUG 03:19:29 DEBUG "metric": { 03:19:29 DEBUG [...] 03:19:29 DEBUG "pod": "ds-idrepo-1" 03:19:29 DEBUG 03:19:29 DEBUG }, 03:19:29 DEBUG 03:19:29 DEBUG "value": [ 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:19:29 DEBUG 03:19:29 DEBUG }, 03:19:29 DEBUG 03:19:29 DEBUG "value": [ 03:19:29 DEBUG [...] 03:19:29 DEBUG 1713061169, 03:19:29 DEBUG 03:19:29 DEBUG "1254.8333333333333" 03:19:29 DEBUG 03:19:29 DEBUG ] 03:19:29 DEBUG 03:19:29 DEBUG } 03:19:29 DEBUG "pod": "ds-idrepo-0" 03:19:29 DEBUG ] 03:19:29 DEBUG }, 03:19:29 DEBUG } 03:19:29 DEBUG "value": [ 03:19:29 DEBUG } 03:19:29 DEBUG 1713061169, 03:19:29 DEBUG "0" 03:19:29 DEBUG ] 03:19:29 DEBUG } 03:19:29 DEBUG ] 03:19:29 DEBUG } 03:19:29 DEBUG } 03:19:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061169" 03:19:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061169" 03:19:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061169" 03:19:29 INFO [http_cmd]: http status code OK 03:19:29 DEBUG --- status code --- 03:19:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:29 DEBUG --- http response --- 03:19:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:19:29 DEBUG ----- output ----- 03:19:29 DEBUG { 03:19:29 DEBUG "status": "success", 03:19:29 DEBUG "data": { 03:19:29 DEBUG "resultType": "vector", 03:19:29 DEBUG "result": [ 03:19:29 DEBUG { 03:19:29 DEBUG "metric": { 03:19:29 DEBUG "pod": "ds-cts-2" 03:19:29 DEBUG }, 03:19:29 DEBUG "value": [ 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG [...] 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:19:29 DEBUG }, 03:19:29 DEBUG "value": [ 03:19:29 DEBUG 1713061169, 03:19:29 DEBUG "0" 03:19:29 DEBUG ] 03:19:29 DEBUG } 03:19:29 DEBUG ] 03:19:29 DEBUG } 03:19:29 DEBUG } 03:19:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713061169" 03:19:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061169" 03:19:29 INFO [http_cmd]: http status code OK 03:19:29 INFO [http_cmd]: http status code OK 03:19:29 DEBUG --- status code --- 03:19:29 DEBUG --- status code --- 03:19:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713061169" 03:19:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:29 DEBUG --- http response --- 03:19:29 DEBUG --- http response --- 03:19:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:19:29 DEBUG ----- output ----- 03:19:29 DEBUG { 03:19:29 DEBUG "status": "success", 03:19:29 DEBUG "data": { 03:19:29 DEBUG "resultType": "vector", 03:19:29 DEBUG "result": [ 03:19:29 DEBUG { 03:19:29 DEBUG "metric": { 03:19:29 DEBUG "pod": "ds-idrepo-1" 03:19:29 DEBUG }, 03:19:29 DEBUG "value": [ 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG [...] 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG "pod": "ds-idrepo-0" 03:19:29 DEBUG }, 03:19:29 DEBUG "value": [ 03:19:29 DEBUG 1713061169, 03:19:29 DEBUG "0" 03:19:29 DEBUG ] 03:19:29 DEBUG } 03:19:29 DEBUG ] 03:19:29 DEBUG } 03:19:29 DEBUG } 03:19:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:19:29 DEBUG ----- output ----- 03:19:29 DEBUG { 03:19:29 DEBUG "status": "success", 03:19:29 DEBUG "data": { 03:19:29 DEBUG "resultType": "vector", 03:19:29 DEBUG "result": [ 03:19:29 DEBUG { 03:19:29 DEBUG "metric": { 03:19:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:19:29 DEBUG }, 03:19:29 DEBUG "value": [ 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG [...] 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:19:29 DEBUG }, 03:19:29 DEBUG "value": [ 03:19:29 DEBUG 1713061169, 03:19:29 DEBUG "0" 03:19:29 DEBUG ] 03:19:29 DEBUG } 03:19:29 DEBUG ] 03:19:29 DEBUG } 03:19:29 DEBUG } 03:19:29 INFO [http_cmd]: http status code OK 03:19:29 DEBUG --- status code --- 03:19:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:29 DEBUG --- http response --- 03:19:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:19:29 DEBUG ----- output ----- 03:19:29 DEBUG { 03:19:29 DEBUG "status": "success", 03:19:29 DEBUG "data": { 03:19:29 DEBUG "resultType": "vector", 03:19:29 DEBUG "result": [ 03:19:29 DEBUG { 03:19:29 DEBUG "metric": { 03:19:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:19:29 DEBUG }, 03:19:29 DEBUG "value": [ 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG [...] 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:19:29 DEBUG }, 03:19:29 DEBUG "value": [ 03:19:29 DEBUG 1713061169, 03:19:29 DEBUG "0.014499999999999999" 03:19:29 DEBUG ] 03:19:29 DEBUG } 03:19:29 DEBUG ] 03:19:29 DEBUG } 03:19:29 DEBUG } 03:19:29 INFO [http_cmd]: http status code OK 03:19:29 DEBUG --- status code --- 03:19:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:29 DEBUG --- http response --- 03:19:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 03:19:29 DEBUG ----- output ----- 03:19:29 DEBUG { 03:19:29 DEBUG "status": "success", 03:19:29 DEBUG "data": { 03:19:29 DEBUG "resultType": "vector", 03:19:29 DEBUG "result": [ 03:19:29 DEBUG { 03:19:29 DEBUG "metric": { 03:19:29 DEBUG "pod": "ds-cts-2" 03:19:29 DEBUG }, 03:19:29 DEBUG "value": [ 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG [...] 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:19:29 DEBUG }, 03:19:29 DEBUG "value": [ 03:19:29 DEBUG 1713061169, 03:19:29 DEBUG "6.847668087507154" 03:19:29 DEBUG ] 03:19:29 DEBUG } 03:19:29 DEBUG ] 03:19:29 DEBUG } 03:19:29 DEBUG } 03:19:29 INFO [http_cmd]: http status code OK 03:19:29 DEBUG --- status code --- 03:19:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:29 DEBUG --- http response --- 03:19:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:19:29 DEBUG ----- output ----- 03:19:29 DEBUG { 03:19:29 DEBUG "status": "success", 03:19:29 DEBUG "data": { 03:19:29 DEBUG "resultType": "vector", 03:19:29 DEBUG "result": [ 03:19:29 DEBUG { 03:19:29 DEBUG "metric": { 03:19:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 03:19:29 DEBUG }, 03:19:29 DEBUG "value": [ 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG [...] 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:19:29 DEBUG }, 03:19:29 DEBUG "value": [ 03:19:29 DEBUG 1713061169, 03:19:29 DEBUG "6941437952" 03:19:29 DEBUG ] 03:19:29 DEBUG } 03:19:29 DEBUG ] 03:19:29 DEBUG } 03:19:29 DEBUG } 03:19:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061169" 03:19:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061169" 03:19:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061169" 03:19:29 INFO [http_cmd]: http status code OK 03:19:29 DEBUG --- status code --- 03:19:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:29 DEBUG --- http response --- 03:19:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:19:29 DEBUG ----- output ----- 03:19:29 DEBUG { 03:19:29 DEBUG "status": "success", 03:19:29 DEBUG "data": { 03:19:29 DEBUG "resultType": "vector", 03:19:29 DEBUG "result": [ 03:19:29 DEBUG { 03:19:29 DEBUG "metric": { 03:19:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:19:29 DEBUG }, 03:19:29 DEBUG "value": [ 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG [...] 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:19:29 DEBUG }, 03:19:29 DEBUG "value": [ 03:19:29 DEBUG 1713061169, 03:19:29 DEBUG "1254.8333333333333" 03:19:29 DEBUG ] 03:19:29 DEBUG } 03:19:29 DEBUG ] 03:19:29 DEBUG } 03:19:29 DEBUG } 03:19:29 INFO 03:19:29 INFO [loop_until]: kubectl --namespace=xlou top pods 03:19:29 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:19:29 INFO [http_cmd]: http status code OK 03:19:29 INFO [http_cmd]: http status code OK 03:19:29 DEBUG --- status code --- 03:19:29 DEBUG --- status code --- 03:19:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:29 DEBUG --- http response --- 03:19:29 DEBUG --- http response --- 03:19:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 03:19:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:19:29 DEBUG ----- output ----- 03:19:29 DEBUG ----- output ----- 03:19:29 DEBUG { 03:19:29 DEBUG { 03:19:29 DEBUG "status": "success", 03:19:29 DEBUG "status": "success", 03:19:29 DEBUG "data": { 03:19:29 DEBUG "data": { 03:19:29 DEBUG "resultType": "vector", 03:19:29 DEBUG "resultType": "vector", 03:19:29 DEBUG "result": [ 03:19:29 DEBUG "result": [ 03:19:29 DEBUG { 03:19:29 DEBUG { 03:19:29 DEBUG "metric": { 03:19:29 DEBUG "metric": { 03:19:29 DEBUG "pod": "ds-idrepo-1" 03:19:29 DEBUG "pod": "ds-cts-2" 03:19:29 DEBUG }, 03:19:29 DEBUG }, 03:19:29 DEBUG "value": [ 03:19:29 DEBUG "value": [ 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG [...] 03:19:29 DEBUG [...] 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG "pod": "ds-cts-1" 03:19:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:19:29 DEBUG }, 03:19:29 DEBUG }, 03:19:29 DEBUG "value": [ 03:19:29 DEBUG "value": [ 03:19:29 DEBUG 1713061169, 03:19:29 DEBUG 1713061169, 03:19:29 DEBUG "0" 03:19:29 DEBUG "2794256.127374162" 03:19:29 DEBUG ] 03:19:29 DEBUG ] 03:19:29 DEBUG } 03:19:29 DEBUG } 03:19:29 DEBUG ] 03:19:29 DEBUG ] 03:19:29 DEBUG } 03:19:29 DEBUG } 03:19:29 DEBUG } 03:19:29 DEBUG } 03:19:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061169" 03:19:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061169" 03:19:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061169" 03:19:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061169" 03:19:29 INFO [http_cmd]: http status code OK 03:19:29 DEBUG --- status code --- 03:19:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:29 DEBUG --- http response --- 03:19:29 INFO [http_cmd]: http status code OK 03:19:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:19:29 DEBUG --- status code --- 03:19:29 DEBUG ----- output ----- 03:19:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:29 DEBUG { 03:19:29 DEBUG --- http response --- 03:19:29 DEBUG "status": "success", 03:19:29 DEBUG "data": { 03:19:29 DEBUG "resultType": "vector", 03:19:29 DEBUG "result": [ 03:19:29 DEBUG { 03:19:29 DEBUG "metric": { 03:19:29 DEBUG "instance": "10.142.0.107:9100" 03:19:29 DEBUG }, 03:19:29 DEBUG "value": [ 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG [...] 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG "instance": "10.142.0.43:9100" 03:19:29 DEBUG }, 03:19:29 DEBUG "value": [ 03:19:29 DEBUG 1713061169, 03:19:29 DEBUG "12650888192" 03:19:29 DEBUG ] 03:19:29 DEBUG } 03:19:29 DEBUG ] 03:19:29 DEBUG } 03:19:29 DEBUG } 03:19:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:19:29 DEBUG ----- output ----- 03:19:29 DEBUG { 03:19:29 DEBUG "status": "success", 03:19:29 DEBUG "data": { 03:19:29 DEBUG "resultType": "vector", 03:19:29 DEBUG "result": [ 03:19:29 DEBUG { 03:19:29 DEBUG "metric": { 03:19:29 DEBUG "instance": "10.142.0.107:9100" 03:19:29 DEBUG }, 03:19:29 DEBUG "value": [ 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG [...] 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG "instance": "10.142.0.43:9100" 03:19:29 DEBUG }, 03:19:29 DEBUG "value": [ 03:19:29 DEBUG 1713061169, 03:19:29 DEBUG "4439.4" 03:19:29 DEBUG ] 03:19:29 DEBUG } 03:19:29 DEBUG ] 03:19:29 DEBUG } 03:19:29 DEBUG } 03:19:29 INFO [http_cmd]: http status code OK 03:19:29 DEBUG --- status code --- 03:19:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:29 DEBUG --- http response --- 03:19:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:19:29 DEBUG ----- output ----- 03:19:29 DEBUG { 03:19:29 DEBUG "status": "success", 03:19:29 DEBUG "data": { 03:19:29 DEBUG "resultType": "vector", 03:19:29 DEBUG "result": [ 03:19:29 DEBUG { 03:19:29 DEBUG "metric": { 03:19:29 DEBUG "instance": "10.142.0.107:9100" 03:19:29 DEBUG }, 03:19:29 DEBUG "value": [ 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG [...] 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG "instance": "10.142.0.43:9100" 03:19:29 DEBUG }, 03:19:29 DEBUG "value": [ 03:19:29 DEBUG 1713061169, 03:19:29 DEBUG "0.00026666666666651885" 03:19:29 DEBUG ] 03:19:29 DEBUG } 03:19:29 DEBUG ] 03:19:29 DEBUG } 03:19:29 DEBUG } 03:19:29 INFO [http_cmd]: http status code OK 03:19:29 DEBUG --- status code --- 03:19:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:29 DEBUG --- http response --- 03:19:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:19:29 DEBUG ----- output ----- 03:19:29 DEBUG { 03:19:29 DEBUG "status": "success", 03:19:29 DEBUG "data": { 03:19:29 DEBUG "resultType": "vector", 03:19:29 DEBUG "result": [ 03:19:29 DEBUG { 03:19:29 DEBUG "metric": { 03:19:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:19:29 DEBUG }, 03:19:29 DEBUG "value": [ 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG [...] 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG 03:19:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:19:29 DEBUG }, 03:19:29 DEBUG "value": [ 03:19:29 DEBUG 1713061169, 03:19:29 DEBUG "1256.0333333333333" 03:19:29 DEBUG ] 03:19:29 DEBUG } 03:19:29 DEBUG ] 03:19:29 DEBUG } 03:19:29 DEBUG } 03:19:29 INFO [loop_until]: OK (rc = 0) 03:19:29 DEBUG --- stdout --- 03:19:29 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 5615m 7320Mi am-7d597cf7d6-8vr8f 5093m 8370Mi am-7d597cf7d6-9wz9h 6704m 8968Mi ds-cts-0 2552m 7800Mi ds-cts-1 2638m 7993Mi ds-cts-2 2668m 8089Mi ds-idrepo-0 6435m 4366Mi ds-idrepo-1 1046m 4285Mi ds-idrepo-2 970m 4102Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 5m 990Mi idm-b85f9d978-rl76f 4m 983Mi lodemon-766dc9b5df-2jmd4 2m 62Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 8001m 7641Mi 03:19:29 DEBUG --- stderr --- 03:19:29 DEBUG 03:19:29 INFO 03:19:29 INFO [loop_until]: kubectl --namespace=xlou top node 03:19:29 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:19:29 INFO [loop_until]: OK (rc = 0) 03:19:29 DEBUG --- stdout --- 03:19:29 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 4817Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 2798m 9% 10288Mi 9% gke-xlou-cdm-default-pool-1b0769c8-nhlf 5600m 18% 9358Mi 8% gke-xlou-cdm-default-pool-1b0769c8-qkns 11012m 36% 3621Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 1095m 3% 6338Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 2650m 8% 11238Mi 10% gke-xlou-cdm-default-pool-1b0769c8-xnmp 62m 0% 2895Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 5377m 18% 9800Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1002m 3% 5518Mi 4% gke-xlou-cdm-ds-0f601751-j488 2739m 9% 9516Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 59m 0% 2191Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 7169m 24% 10517Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 6569m 22% 5835Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 7991m 50% 10900Mi 18% 03:19:29 DEBUG --- stderr --- 03:19:29 DEBUG 03:20:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061229" 03:20:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713061229" 03:20:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061229" 03:20:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061229" 03:20:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061229" 03:20:29 INFO [http_cmd]: http status code OK 03:20:29 INFO [http_cmd]: http status code OK 03:20:29 DEBUG --- status code --- 03:20:29 DEBUG --- status code --- 03:20:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:29 DEBUG --- http response --- 03:20:29 DEBUG --- http response --- 03:20:29 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:20:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:20:29 DEBUG ----- output ----- 03:20:29 DEBUG { 03:20:29 DEBUG "status": "success", 03:20:29 DEBUG "data": { 03:20:29 DEBUG "resultType": "vector", 03:20:29 DEBUG "result": [ 03:20:29 DEBUG { 03:20:29 DEBUG "metric": { 03:20:29 DEBUG "pod": "ds-idrepo-1" 03:20:29 DEBUG }, 03:20:29 DEBUG "value": [ 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG [...] 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG "pod": "ds-cts-1" 03:20:29 DEBUG }, 03:20:29 DEBUG "value": [ 03:20:29 DEBUG 1713061229, 03:20:29 DEBUG "0" 03:20:29 DEBUG ] 03:20:29 DEBUG } 03:20:29 DEBUG ] 03:20:29 DEBUG } 03:20:29 DEBUG } 03:20:29 INFO [http_cmd]: http status code OK 03:20:29 INFO [http_cmd]: http status code OK 03:20:29 DEBUG --- status code --- 03:20:29 DEBUG --- status code --- 03:20:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:29 DEBUG --- http response --- 03:20:29 DEBUG --- http response --- 03:20:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:20:29 DEBUG ----- output ----- 03:20:29 DEBUG { 03:20:29 DEBUG "status": "success", 03:20:29 DEBUG "data": { 03:20:29 DEBUG "resultType": "vector", 03:20:29 DEBUG "result": [ 03:20:29 DEBUG { 03:20:29 DEBUG "metric": { 03:20:29 DEBUG "pod": "ds-idrepo-1" 03:20:29 DEBUG }, 03:20:29 DEBUG "value": [ 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG [...] 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG "pod": "ds-idrepo-0" 03:20:29 DEBUG }, 03:20:29 DEBUG "value": [ 03:20:29 DEBUG 1713061229, 03:20:29 DEBUG "0" 03:20:29 DEBUG ] 03:20:29 DEBUG } 03:20:29 DEBUG ] 03:20:29 DEBUG } 03:20:29 DEBUG } 03:20:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:20:29 DEBUG ----- output ----- 03:20:29 DEBUG { 03:20:29 DEBUG "status": "success", 03:20:29 DEBUG "data": { 03:20:29 DEBUG "resultType": "vector", 03:20:29 DEBUG "result": [ 03:20:29 DEBUG { 03:20:29 DEBUG "metric": { 03:20:29 DEBUG "pod": "ds-idrepo-1" 03:20:29 DEBUG }, 03:20:29 DEBUG "value": [ 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG [...] 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG "pod": "ds-idrepo-0" 03:20:29 DEBUG }, 03:20:29 DEBUG "value": [ 03:20:29 DEBUG 1713061229, 03:20:29 DEBUG "0" 03:20:29 DEBUG ] 03:20:29 DEBUG } 03:20:29 DEBUG ] 03:20:29 DEBUG } 03:20:29 DEBUG } 03:20:29 INFO [http_cmd]: http status code OK 03:20:29 DEBUG --- status code --- 03:20:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:29 DEBUG --- http response --- 03:20:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:20:29 DEBUG ----- output ----- 03:20:29 DEBUG { 03:20:29 DEBUG "status": "success", 03:20:29 DEBUG "data": { 03:20:29 DEBUG "resultType": "vector", 03:20:29 DEBUG "result": [ 03:20:29 DEBUG { 03:20:29 DEBUG "metric": { 03:20:29 DEBUG "pod": "ds-idrepo-1" 03:20:29 DEBUG }, 03:20:29 DEBUG "value": [ 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG [...] 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG "pod": "ds-idrepo-0" 03:20:29 DEBUG }, 03:20:29 DEBUG "value": [ 03:20:29 DEBUG 1713061229, 03:20:29 DEBUG "0" 03:20:29 DEBUG ] 03:20:29 DEBUG } 03:20:29 DEBUG ] 03:20:29 DEBUG } 03:20:29 DEBUG } 03:20:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713061229" 03:20:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713061229" 03:20:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061229" 03:20:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061229" 03:20:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061229" 03:20:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061229" 03:20:29 ERROR [http_cmd]: ERROR 03:20:29 ERROR --- status code --- 03:20:29 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:20:29 ERROR --- http response --- 03:20:29 INFO [http_cmd]: http status code OK 03:20:29 INFO [http_cmd]: http status code OK 03:20:29 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:20:29 DEBUG --- status code --- 03:20:29 DEBUG --- status code --- 03:20:29 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:20:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:29 DEBUG --- http response --- 03:20:29 DEBUG --- http response --- 03:20:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:20:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:20:29 DEBUG ----- output ----- 03:20:29 DEBUG ----- output ----- 03:20:29 DEBUG { 03:20:29 DEBUG { 03:20:29 DEBUG "status": "success", 03:20:29 DEBUG "status": "success", 03:20:29 DEBUG "data": { 03:20:29 DEBUG "data": { 03:20:29 DEBUG "resultType": "vector", 03:20:29 DEBUG "resultType": "vector", 03:20:29 DEBUG "result": [ 03:20:29 DEBUG "result": [ 03:20:29 DEBUG { 03:20:29 DEBUG { 03:20:29 DEBUG "metric": { 03:20:29 DEBUG "metric": { 03:20:29 DEBUG "pod": "ds-idrepo-1" 03:20:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:20:29 DEBUG }, 03:20:29 DEBUG }, 03:20:29 DEBUG "value": [ 03:20:29 DEBUG "value": [ 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG [...] 03:20:29 DEBUG [...] 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG "pod": "ds-idrepo-0" 03:20:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:20:29 DEBUG }, 03:20:29 DEBUG }, 03:20:29 DEBUG "value": [ 03:20:29 DEBUG "value": [ 03:20:29 DEBUG 1713061229, 03:20:29 DEBUG 1713061229, 03:20:29 DEBUG "0" 03:20:29 DEBUG "215089.5" 03:20:29 DEBUG ] 03:20:29 DEBUG ] 03:20:29 DEBUG } 03:20:29 DEBUG } 03:20:29 DEBUG ] 03:20:29 DEBUG ] 03:20:29 DEBUG } 03:20:29 DEBUG } 03:20:29 DEBUG } 03:20:29 DEBUG } 03:20:29 INFO [http_cmd]: http status code OK 03:20:29 DEBUG --- status code --- 03:20:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:29 DEBUG --- http response --- 03:20:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:20:29 DEBUG ----- output ----- 03:20:29 DEBUG { 03:20:29 DEBUG "status": "success", 03:20:29 DEBUG "data": { 03:20:29 DEBUG "resultType": "vector", 03:20:29 DEBUG "result": [ 03:20:29 DEBUG { 03:20:29 DEBUG "metric": { 03:20:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 03:20:29 DEBUG }, 03:20:29 DEBUG "value": [ 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG [...] 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:20:29 DEBUG }, 03:20:29 DEBUG "value": [ 03:20:29 DEBUG 1713061229, 03:20:29 DEBUG "8.031619280705748" 03:20:29 DEBUG ] 03:20:29 DEBUG } 03:20:29 DEBUG ] 03:20:29 DEBUG } 03:20:29 DEBUG } 03:20:29 INFO [http_cmd]: http status code OK 03:20:29 DEBUG --- status code --- 03:20:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:29 DEBUG --- http response --- 03:20:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:20:29 INFO [http_cmd]: http status code OK 03:20:29 DEBUG ----- output ----- 03:20:29 DEBUG --- status code --- 03:20:29 DEBUG { 03:20:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:29 DEBUG "status": "success", 03:20:29 DEBUG --- http response --- 03:20:29 DEBUG "data": { 03:20:29 DEBUG "resultType": "vector", 03:20:29 DEBUG "result": [ 03:20:29 DEBUG { 03:20:29 DEBUG "metric": { 03:20:29 DEBUG "instance": "10.142.0.107:9100" 03:20:29 DEBUG }, 03:20:29 DEBUG "value": [ 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG [...] 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG "instance": "10.142.0.43:9100" 03:20:29 DEBUG }, 03:20:29 DEBUG "value": [ 03:20:29 DEBUG 1713061229, 03:20:29 DEBUG "0" 03:20:29 DEBUG ] 03:20:29 DEBUG } 03:20:29 DEBUG ] 03:20:29 DEBUG } 03:20:29 DEBUG } 03:20:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:20:29 DEBUG ----- output ----- 03:20:29 DEBUG { 03:20:29 DEBUG "status": "success", 03:20:29 DEBUG "data": { 03:20:29 DEBUG "resultType": "vector", 03:20:29 DEBUG "result": [ 03:20:29 DEBUG { 03:20:29 DEBUG "metric": { 03:20:29 DEBUG "pod": "ds-cts-2" 03:20:29 DEBUG }, 03:20:29 DEBUG "value": [ 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG [...] 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 03:20:29 DEBUG }, 03:20:29 DEBUG "value": [ 03:20:29 DEBUG 1713061229, 03:20:29 DEBUG "0" 03:20:29 DEBUG ] 03:20:29 DEBUG } 03:20:29 DEBUG ] 03:20:29 DEBUG } 03:20:29 DEBUG } 03:20:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061229" 03:20:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061229" 03:20:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713061229" 03:20:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061229" 03:20:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713061229" 03:20:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061229" 03:20:29 INFO [http_cmd]: http status code OK 03:20:29 INFO [http_cmd]: http status code OK 03:20:29 DEBUG --- status code --- 03:20:29 DEBUG --- status code --- 03:20:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:29 DEBUG --- http response --- 03:20:29 DEBUG --- http response --- 03:20:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:20:29 DEBUG ----- output ----- 03:20:29 DEBUG { 03:20:29 DEBUG "status": "success", 03:20:29 DEBUG "data": { 03:20:29 DEBUG "resultType": "vector", 03:20:29 DEBUG "result": [ 03:20:29 DEBUG { 03:20:29 DEBUG "metric": { 03:20:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:20:29 DEBUG }, 03:20:29 DEBUG "value": [ 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG [...] 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:20:29 DEBUG }, 03:20:29 DEBUG "value": [ 03:20:29 DEBUG 1713061229, 03:20:29 DEBUG "0" 03:20:29 DEBUG ] 03:20:29 DEBUG } 03:20:29 DEBUG ] 03:20:29 DEBUG } 03:20:29 DEBUG } 03:20:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:20:29 DEBUG ----- output ----- 03:20:29 DEBUG { 03:20:29 DEBUG "status": "success", 03:20:29 DEBUG "data": { 03:20:29 DEBUG "resultType": "vector", 03:20:29 DEBUG "result": [ 03:20:29 DEBUG { 03:20:29 DEBUG "metric": { 03:20:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:20:29 DEBUG }, 03:20:29 DEBUG "value": [ 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG [...] 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:20:29 DEBUG }, 03:20:29 DEBUG "value": [ 03:20:29 DEBUG 1713061229, 03:20:29 DEBUG "0" 03:20:29 DEBUG ] 03:20:29 DEBUG } 03:20:29 DEBUG ] 03:20:29 DEBUG } 03:20:29 DEBUG } 03:20:29 INFO [http_cmd]: http status code OK 03:20:29 DEBUG --- status code --- 03:20:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:29 DEBUG --- http response --- 03:20:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:20:29 DEBUG ----- output ----- 03:20:29 DEBUG { 03:20:29 DEBUG "status": "success", 03:20:29 DEBUG "data": { 03:20:29 DEBUG "resultType": "vector", 03:20:29 DEBUG "result": [ 03:20:29 DEBUG { 03:20:29 DEBUG "metric": { 03:20:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 03:20:29 DEBUG }, 03:20:29 DEBUG "value": [ 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG [...] 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:20:29 DEBUG }, 03:20:29 DEBUG "value": [ 03:20:29 DEBUG 1713061229, 03:20:29 DEBUG "8250564608" 03:20:29 DEBUG ] 03:20:29 DEBUG } 03:20:29 DEBUG ] 03:20:29 DEBUG } 03:20:29 DEBUG } 03:20:29 INFO [http_cmd]: http status code OK 03:20:29 INFO [http_cmd]: http status code OK 03:20:29 DEBUG --- status code --- 03:20:29 DEBUG --- status code --- 03:20:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:29 DEBUG --- http response --- 03:20:29 DEBUG --- http response --- 03:20:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:20:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:20:29 DEBUG ----- output ----- 03:20:29 DEBUG ----- output ----- 03:20:29 DEBUG { 03:20:29 DEBUG { 03:20:29 DEBUG "status": "success", 03:20:29 DEBUG "status": "success", 03:20:29 DEBUG "data": { 03:20:29 DEBUG "data": { 03:20:29 DEBUG "resultType": "vector", 03:20:29 DEBUG "resultType": "vector", 03:20:29 DEBUG "result": [ 03:20:29 DEBUG "result": [ 03:20:29 DEBUG { 03:20:29 DEBUG { 03:20:29 DEBUG "metric": { 03:20:29 DEBUG "metric": { 03:20:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:20:29 DEBUG "instance": "10.142.0.107:9100" 03:20:29 DEBUG }, 03:20:29 DEBUG }, 03:20:29 DEBUG "value": [ 03:20:29 DEBUG "value": [ 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG [...] 03:20:29 DEBUG [...] 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:20:29 DEBUG "instance": "10.142.0.43:9100" 03:20:29 DEBUG }, 03:20:29 DEBUG }, 03:20:29 DEBUG "value": [ 03:20:29 DEBUG "value": [ 03:20:29 DEBUG 1713061229, 03:20:29 DEBUG 1713061229, 03:20:29 DEBUG "0.015" 03:20:29 DEBUG "5100.166666666667" 03:20:29 DEBUG ] 03:20:29 DEBUG ] 03:20:29 DEBUG } 03:20:29 INFO [http_cmd]: http status code OK 03:20:29 DEBUG } 03:20:29 DEBUG ] 03:20:29 DEBUG --- status code --- 03:20:29 DEBUG ] 03:20:29 DEBUG } 03:20:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:29 DEBUG } 03:20:29 DEBUG } 03:20:29 DEBUG --- http response --- 03:20:29 DEBUG } 03:20:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:20:29 DEBUG ----- output ----- 03:20:29 DEBUG { 03:20:29 DEBUG "status": "success", 03:20:29 DEBUG "data": { 03:20:29 DEBUG "resultType": "vector", 03:20:29 DEBUG "result": [ 03:20:29 DEBUG { 03:20:29 DEBUG "metric": { 03:20:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:20:29 DEBUG }, 03:20:29 DEBUG "value": [ 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG [...] 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:20:29 DEBUG }, 03:20:29 DEBUG "value": [ 03:20:29 DEBUG 1713061229, 03:20:29 DEBUG "1312.5" 03:20:29 DEBUG ] 03:20:29 DEBUG } 03:20:29 DEBUG ] 03:20:29 DEBUG } 03:20:29 DEBUG } 03:20:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061229" 03:20:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061229" 03:20:29 INFO [http_cmd]: http status code OK 03:20:29 DEBUG --- status code --- 03:20:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:29 DEBUG --- http response --- 03:20:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 03:20:29 DEBUG ----- output ----- 03:20:29 DEBUG { 03:20:29 DEBUG "status": "success", 03:20:29 DEBUG "data": { 03:20:29 DEBUG "resultType": "vector", 03:20:29 DEBUG "result": [ 03:20:29 DEBUG { 03:20:29 DEBUG "metric": { 03:20:29 DEBUG "pod": "ds-idrepo-1" 03:20:29 DEBUG }, 03:20:29 DEBUG "value": [ 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG [...] 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG "pod": "ds-cts-1" 03:20:29 DEBUG }, 03:20:29 DEBUG "value": [ 03:20:29 DEBUG 1713061229, 03:20:29 DEBUG "0.00005419444444444444" 03:20:29 DEBUG ] 03:20:29 DEBUG } 03:20:29 DEBUG ] 03:20:29 DEBUG } 03:20:29 DEBUG } 03:20:29 INFO [http_cmd]: http status code OK 03:20:29 DEBUG --- status code --- 03:20:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:29 DEBUG --- http response --- 03:20:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:20:29 DEBUG ----- output ----- 03:20:29 DEBUG { 03:20:29 DEBUG "status": "success", 03:20:29 DEBUG "data": { 03:20:29 DEBUG "resultType": "vector", 03:20:29 DEBUG "result": [ 03:20:29 DEBUG { 03:20:29 DEBUG "metric": { 03:20:29 DEBUG "pod": "ds-cts-2" 03:20:29 DEBUG }, 03:20:29 DEBUG "value": [ 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG [...] 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 03:20:29 DEBUG }, 03:20:29 DEBUG "value": [ 03:20:29 DEBUG 1713061229, 03:20:29 DEBUG "0" 03:20:29 DEBUG ] 03:20:29 DEBUG } 03:20:29 DEBUG ] 03:20:29 DEBUG } 03:20:29 DEBUG } 03:20:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061229" 03:20:29 INFO [http_cmd]: http status code OK 03:20:29 DEBUG --- status code --- 03:20:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:29 DEBUG --- http response --- 03:20:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:20:29 DEBUG ----- output ----- 03:20:29 DEBUG { 03:20:29 DEBUG "status": "success", 03:20:29 DEBUG "data": { 03:20:29 DEBUG "resultType": "vector", 03:20:29 DEBUG "result": [ 03:20:29 DEBUG { 03:20:29 DEBUG "metric": { 03:20:29 DEBUG "pod": "ds-idrepo-1" 03:20:29 DEBUG }, 03:20:29 DEBUG "value": [ 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG [...] 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG "pod": "ds-idrepo-0" 03:20:29 DEBUG }, 03:20:29 DEBUG "value": [ 03:20:29 DEBUG 1713061229, 03:20:29 DEBUG "0" 03:20:29 DEBUG ] 03:20:29 DEBUG } 03:20:29 DEBUG ] 03:20:29 DEBUG } 03:20:29 DEBUG } 03:20:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061229" 03:20:29 INFO [http_cmd]: http status code OK 03:20:29 DEBUG --- status code --- 03:20:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:29 DEBUG --- http response --- 03:20:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:20:29 DEBUG ----- output ----- 03:20:29 DEBUG { 03:20:29 DEBUG "status": "success", 03:20:29 DEBUG "data": { 03:20:29 DEBUG "resultType": "vector", 03:20:29 DEBUG "result": [ 03:20:29 DEBUG { 03:20:29 DEBUG "metric": { 03:20:29 DEBUG "pod": "ds-cts-2" 03:20:29 DEBUG }, 03:20:29 DEBUG "value": [ 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG [...] 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 03:20:29 DEBUG }, 03:20:29 DEBUG "value": [ 03:20:29 DEBUG 1713061229, 03:20:29 DEBUG "0" 03:20:29 DEBUG ] 03:20:29 DEBUG } 03:20:29 DEBUG ] 03:20:29 DEBUG } 03:20:29 DEBUG } 03:20:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061229" 03:20:29 INFO [http_cmd]: http status code OK 03:20:29 DEBUG --- status code --- 03:20:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:29 DEBUG --- http response --- 03:20:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:20:29 DEBUG ----- output ----- 03:20:29 DEBUG { 03:20:29 DEBUG "status": "success", 03:20:29 DEBUG "data": { 03:20:29 DEBUG "resultType": "vector", 03:20:29 DEBUG "result": [ 03:20:29 DEBUG { 03:20:29 DEBUG "metric": { 03:20:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:20:29 DEBUG }, 03:20:29 DEBUG "value": [ 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG [...] 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:20:29 DEBUG }, 03:20:29 DEBUG "value": [ 03:20:29 DEBUG 1713061229, 03:20:29 DEBUG "1312.5333333333333" 03:20:29 DEBUG ] 03:20:29 DEBUG } 03:20:29 DEBUG ] 03:20:29 DEBUG } 03:20:29 DEBUG } 03:20:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061229" 03:20:29 INFO [http_cmd]: http status code OK 03:20:29 DEBUG --- status code --- 03:20:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:29 DEBUG --- http response --- 03:20:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:20:29 DEBUG ----- output ----- 03:20:29 DEBUG { 03:20:29 DEBUG "status": "success", 03:20:29 DEBUG "data": { 03:20:29 DEBUG "resultType": "vector", 03:20:29 DEBUG "result": [ 03:20:29 DEBUG { 03:20:29 DEBUG "metric": { 03:20:29 DEBUG "pod": "ds-cts-2" 03:20:29 DEBUG }, 03:20:29 DEBUG "value": [ 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG [...] 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 03:20:29 DEBUG }, 03:20:29 DEBUG "value": [ 03:20:29 DEBUG 1713061229, 03:20:29 DEBUG "0" 03:20:29 DEBUG ] 03:20:29 DEBUG } 03:20:29 DEBUG ] 03:20:29 DEBUG } 03:20:29 DEBUG } 03:20:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061229" 03:20:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061229" 03:20:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061229" 03:20:29 INFO [http_cmd]: http status code OK 03:20:29 DEBUG --- status code --- 03:20:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:29 DEBUG --- http response --- 03:20:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:20:29 DEBUG ----- output ----- 03:20:29 DEBUG { 03:20:29 DEBUG "status": "success", 03:20:29 DEBUG "data": { 03:20:29 DEBUG "resultType": "vector", 03:20:29 DEBUG "result": [ 03:20:29 DEBUG { 03:20:29 DEBUG "metric": { 03:20:29 DEBUG "instance": "10.142.0.107:9100" 03:20:29 DEBUG }, 03:20:29 DEBUG "value": [ 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG [...] 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG "instance": "10.142.0.43:9100" 03:20:29 DEBUG }, 03:20:29 DEBUG "value": [ 03:20:29 DEBUG 1713061229, 03:20:29 DEBUG "12651369472" 03:20:29 DEBUG ] 03:20:29 DEBUG } 03:20:29 DEBUG ] 03:20:29 DEBUG } 03:20:29 DEBUG } 03:20:29 INFO [http_cmd]: http status code OK 03:20:29 DEBUG --- status code --- 03:20:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:29 DEBUG --- http response --- 03:20:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:20:29 DEBUG ----- output ----- 03:20:29 DEBUG { 03:20:29 DEBUG "status": "success", 03:20:29 DEBUG "data": { 03:20:29 DEBUG "resultType": "vector", 03:20:29 DEBUG "result": [ 03:20:29 DEBUG { 03:20:29 DEBUG "metric": { 03:20:29 DEBUG "instance": "10.142.0.107:9100" 03:20:29 DEBUG }, 03:20:29 DEBUG "value": [ 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG [...] 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG "instance": "10.142.0.43:9100" 03:20:29 DEBUG }, 03:20:29 DEBUG "value": [ 03:20:29 DEBUG 1713061229, 03:20:29 DEBUG "2811.2666666666664" 03:20:29 DEBUG ] 03:20:29 DEBUG } 03:20:29 DEBUG ] 03:20:29 DEBUG } 03:20:29 DEBUG } 03:20:29 INFO [http_cmd]: http status code OK 03:20:29 DEBUG --- status code --- 03:20:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:29 DEBUG --- http response --- 03:20:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:20:29 DEBUG ----- output ----- 03:20:29 DEBUG { 03:20:29 DEBUG "status": "success", 03:20:29 DEBUG "data": { 03:20:29 DEBUG "resultType": "vector", 03:20:29 DEBUG "result": [ 03:20:29 DEBUG { 03:20:29 DEBUG "metric": { 03:20:29 DEBUG "instance": "10.142.0.107:9100" 03:20:29 DEBUG }, 03:20:29 DEBUG "value": [ 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG [...] 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG "instance": "10.142.0.43:9100" 03:20:29 DEBUG }, 03:20:29 DEBUG "value": [ 03:20:29 DEBUG 1713061229, 03:20:29 DEBUG "0.0002333333333332632" 03:20:29 DEBUG ] 03:20:29 DEBUG } 03:20:29 DEBUG ] 03:20:29 DEBUG } 03:20:29 DEBUG } 03:20:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061229" 03:20:29 INFO [http_cmd]: http status code OK 03:20:29 DEBUG --- status code --- 03:20:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:29 DEBUG --- http response --- 03:20:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:20:29 DEBUG ----- output ----- 03:20:29 DEBUG { 03:20:29 DEBUG "status": "success", 03:20:29 DEBUG "data": { 03:20:29 DEBUG "resultType": "vector", 03:20:29 DEBUG "result": [ 03:20:29 DEBUG { 03:20:29 DEBUG "metric": { 03:20:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:20:29 DEBUG }, 03:20:29 DEBUG "value": [ 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG [...] 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG 03:20:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:20:29 DEBUG }, 03:20:29 DEBUG "value": [ 03:20:29 DEBUG 1713061229, 03:20:29 DEBUG "1312.8" 03:20:29 DEBUG ] 03:20:29 DEBUG } 03:20:29 DEBUG ] 03:20:29 DEBUG } 03:20:29 DEBUG } 03:20:29 INFO 03:20:29 INFO [loop_until]: kubectl --namespace=xlou top pods 03:20:29 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:20:29 INFO [loop_until]: OK (rc = 0) 03:20:29 DEBUG --- stdout --- 03:20:29 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 5445m 7317Mi am-7d597cf7d6-8vr8f 5353m 8370Mi am-7d597cf7d6-9wz9h 6703m 8968Mi ds-cts-0 2699m 7835Mi ds-cts-1 2778m 8020Mi ds-cts-2 2802m 8100Mi ds-idrepo-0 6588m 4420Mi ds-idrepo-1 1014m 4286Mi ds-idrepo-2 969m 4103Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 990Mi idm-b85f9d978-rl76f 4m 984Mi lodemon-766dc9b5df-2jmd4 2m 62Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 7864m 7908Mi 03:20:29 DEBUG --- stderr --- 03:20:29 DEBUG 03:20:29 INFO 03:20:29 INFO [loop_until]: kubectl --namespace=xlou top node 03:20:29 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:20:29 INFO [loop_until]: OK (rc = 0) 03:20:29 DEBUG --- stdout --- 03:20:29 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 4819Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 2720m 9% 10293Mi 9% gke-xlou-cdm-default-pool-1b0769c8-nhlf 5709m 19% 9362Mi 8% gke-xlou-cdm-default-pool-1b0769c8-qkns 10800m 36% 3610Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 1143m 3% 6349Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 2720m 9% 11247Mi 10% gke-xlou-cdm-default-pool-1b0769c8-xnmp 66m 0% 2906Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 5549m 18% 9796Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1078m 3% 5515Mi 4% gke-xlou-cdm-ds-0f601751-j488 2765m 9% 9519Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 63m 0% 2186Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 7130m 23% 10520Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 6481m 21% 5880Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 8242m 51% 11025Mi 18% 03:20:29 DEBUG --- stderr --- 03:20:29 DEBUG 03:21:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061289" 03:21:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061289" 03:21:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713061289" 03:21:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061289" 03:21:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061289" 03:21:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061289" 03:21:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713061289" 03:21:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061289" 03:21:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713061289" 03:21:29 INFO [http_cmd]: http status code OK 03:21:29 INFO [http_cmd]: http status code OK 03:21:29 DEBUG --- status code --- 03:21:29 DEBUG --- status code --- 03:21:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:29 DEBUG --- http response --- 03:21:29 DEBUG --- http response --- 03:21:29 INFO [http_cmd]: http status code OK 03:21:29 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:21:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:21:29 DEBUG --- status code --- 03:21:29 DEBUG ----- output ----- 03:21:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:29 DEBUG { 03:21:29 DEBUG --- http response --- 03:21:29 DEBUG "status": "success", 03:21:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:21:29 DEBUG "data": { 03:21:29 INFO [http_cmd]: http status code OK 03:21:29 DEBUG ----- output ----- 03:21:29 DEBUG "resultType": "vector", 03:21:29 DEBUG --- status code --- 03:21:29 DEBUG { 03:21:29 DEBUG "result": [ 03:21:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:29 DEBUG "status": "success", 03:21:29 DEBUG { 03:21:29 DEBUG --- http response --- 03:21:29 DEBUG "data": { 03:21:29 DEBUG "metric": { 03:21:29 DEBUG "resultType": "vector", 03:21:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:21:29 DEBUG "pod": "ds-idrepo-1" 03:21:29 DEBUG "result": [ 03:21:29 DEBUG ----- output ----- 03:21:29 DEBUG }, 03:21:29 DEBUG { 03:21:29 DEBUG { 03:21:29 DEBUG "value": [ 03:21:29 DEBUG "metric": { 03:21:29 DEBUG "status": "success", 03:21:29 DEBUG 03:21:29 DEBUG "instance": "10.142.0.107:9100" 03:21:29 DEBUG "data": { 03:21:29 DEBUG 03:21:29 DEBUG }, 03:21:29 DEBUG "resultType": "vector", 03:21:29 DEBUG 03:21:29 DEBUG "value": [ 03:21:29 DEBUG "result": [ 03:21:29 DEBUG [...] 03:21:29 DEBUG 03:21:29 DEBUG { 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG "metric": { 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG "pod": "ds-idrepo-1" 03:21:29 DEBUG 03:21:29 DEBUG [...] 03:21:29 DEBUG }, 03:21:29 DEBUG "pod": "ds-idrepo-0" 03:21:29 DEBUG 03:21:29 DEBUG "value": [ 03:21:29 DEBUG }, 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG "value": [ 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 1713061289, 03:21:29 DEBUG "instance": "10.142.0.43:9100" 03:21:29 DEBUG 03:21:29 DEBUG "0" 03:21:29 DEBUG }, 03:21:29 DEBUG [...] 03:21:29 DEBUG ] 03:21:29 DEBUG "value": [ 03:21:29 DEBUG 03:21:29 DEBUG } 03:21:29 DEBUG 1713061289, 03:21:29 DEBUG 03:21:29 DEBUG ] 03:21:29 DEBUG "0.00033333333333333365" 03:21:29 DEBUG 03:21:29 DEBUG } 03:21:29 DEBUG ] 03:21:29 DEBUG "pod": "ds-cts-1" 03:21:29 DEBUG } 03:21:29 DEBUG } 03:21:29 DEBUG }, 03:21:29 DEBUG ] 03:21:29 DEBUG "value": [ 03:21:29 DEBUG } 03:21:29 DEBUG 1713061289, 03:21:29 DEBUG } 03:21:29 DEBUG "0" 03:21:29 DEBUG ] 03:21:29 DEBUG } 03:21:29 DEBUG ] 03:21:29 DEBUG } 03:21:29 DEBUG } 03:21:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061289" 03:21:29 INFO [http_cmd]: http status code OK 03:21:29 INFO [http_cmd]: http status code OK 03:21:29 DEBUG --- status code --- 03:21:29 INFO [http_cmd]: http status code OK 03:21:29 INFO [http_cmd]: http status code OK 03:21:29 DEBUG --- status code --- 03:21:29 ERROR [http_cmd]: ERROR 03:21:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:29 DEBUG --- status code --- 03:21:29 DEBUG --- status code --- 03:21:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:29 ERROR --- status code --- 03:21:29 DEBUG --- http response --- 03:21:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:29 DEBUG --- http response --- 03:21:29 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:21:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:21:29 DEBUG --- http response --- 03:21:29 DEBUG --- http response --- 03:21:29 ERROR --- http response --- 03:21:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:21:29 DEBUG ----- output ----- 03:21:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:21:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:21:29 DEBUG ----- output ----- 03:21:29 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:21:29 DEBUG { 03:21:29 DEBUG ----- output ----- 03:21:29 DEBUG ----- output ----- 03:21:29 DEBUG { 03:21:29 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:21:29 DEBUG "status": "success", 03:21:29 DEBUG { 03:21:29 DEBUG { 03:21:29 DEBUG "status": "success", 03:21:29 DEBUG "data": { 03:21:29 DEBUG "status": "success", 03:21:29 DEBUG "status": "success", 03:21:29 DEBUG "data": { 03:21:29 DEBUG "resultType": "vector", 03:21:29 DEBUG "data": { 03:21:29 DEBUG "data": { 03:21:29 DEBUG "resultType": "vector", 03:21:29 DEBUG "result": [ 03:21:29 DEBUG "resultType": "vector", 03:21:29 DEBUG "resultType": "vector", 03:21:29 DEBUG "result": [ 03:21:29 DEBUG { 03:21:29 DEBUG "result": [ 03:21:29 DEBUG "result": [ 03:21:29 DEBUG { 03:21:29 DEBUG "metric": { 03:21:29 DEBUG { 03:21:29 DEBUG { 03:21:29 DEBUG "metric": { 03:21:29 DEBUG "pod": "ds-idrepo-1" 03:21:29 DEBUG "metric": { 03:21:29 DEBUG "metric": { 03:21:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 03:21:29 DEBUG }, 03:21:29 DEBUG "pod": "ds-idrepo-1" 03:21:29 DEBUG "pod": "ds-idrepo-1" 03:21:29 DEBUG }, 03:21:29 DEBUG "value": [ 03:21:29 DEBUG }, 03:21:29 DEBUG }, 03:21:29 DEBUG "value": [ 03:21:29 DEBUG 03:21:29 DEBUG "value": [ 03:21:29 DEBUG "value": [ 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG [...] 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG [...] 03:21:29 DEBUG 03:21:29 DEBUG [...] 03:21:29 DEBUG [...] 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG "pod": "ds-idrepo-0" 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:21:29 DEBUG }, 03:21:29 DEBUG "pod": "ds-idrepo-0" 03:21:29 DEBUG "pod": "ds-idrepo-0" 03:21:29 DEBUG }, 03:21:29 DEBUG "value": [ 03:21:29 DEBUG }, 03:21:29 DEBUG }, 03:21:29 DEBUG "value": [ 03:21:29 DEBUG 1713061289, 03:21:29 DEBUG "value": [ 03:21:29 DEBUG "value": [ 03:21:29 DEBUG 1713061289, 03:21:29 DEBUG "0" 03:21:29 DEBUG 1713061289, 03:21:29 DEBUG 1713061289, 03:21:29 DEBUG "8.038908162731294" 03:21:29 DEBUG ] 03:21:29 DEBUG "0" 03:21:29 DEBUG "0" 03:21:29 DEBUG ] 03:21:29 DEBUG } 03:21:29 DEBUG ] 03:21:29 DEBUG ] 03:21:29 DEBUG } 03:21:29 DEBUG ] 03:21:29 DEBUG } 03:21:29 DEBUG } 03:21:29 DEBUG ] 03:21:29 DEBUG } 03:21:29 DEBUG ] 03:21:29 DEBUG ] 03:21:29 DEBUG } 03:21:29 DEBUG } 03:21:29 DEBUG } 03:21:29 DEBUG } 03:21:29 DEBUG } 03:21:29 DEBUG } 03:21:29 DEBUG } 03:21:29 INFO [http_cmd]: http status code OK 03:21:29 DEBUG --- status code --- 03:21:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:29 DEBUG --- http response --- 03:21:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:21:29 DEBUG ----- output ----- 03:21:29 DEBUG { 03:21:29 DEBUG "status": "success", 03:21:29 DEBUG "data": { 03:21:29 DEBUG "resultType": "vector", 03:21:29 DEBUG "result": [ 03:21:29 DEBUG { 03:21:29 DEBUG "metric": { 03:21:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:21:29 DEBUG }, 03:21:29 DEBUG "value": [ 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG [...] 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:21:29 DEBUG }, 03:21:29 DEBUG "value": [ 03:21:29 DEBUG 1713061289, 03:21:29 DEBUG "247066.5" 03:21:29 DEBUG ] 03:21:29 DEBUG } 03:21:29 DEBUG ] 03:21:29 DEBUG } 03:21:29 DEBUG } 03:21:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713061289" 03:21:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713061289" 03:21:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061289" 03:21:29 INFO [http_cmd]: http status code OK 03:21:29 DEBUG --- status code --- 03:21:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:29 DEBUG --- http response --- 03:21:29 INFO [http_cmd]: http status code OK 03:21:29 INFO [http_cmd]: http status code OK 03:21:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:21:29 DEBUG --- status code --- 03:21:29 DEBUG --- status code --- 03:21:29 DEBUG ----- output ----- 03:21:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:29 DEBUG { 03:21:29 DEBUG --- http response --- 03:21:29 DEBUG --- http response --- 03:21:29 DEBUG "status": "success", 03:21:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 03:21:29 DEBUG "data": { 03:21:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:21:29 DEBUG ----- output ----- 03:21:29 DEBUG "resultType": "vector", 03:21:29 DEBUG ----- output ----- 03:21:29 DEBUG { 03:21:29 DEBUG "result": [ 03:21:29 DEBUG { 03:21:29 DEBUG "status": "success", 03:21:29 DEBUG { 03:21:29 DEBUG "status": "success", 03:21:29 DEBUG "data": { 03:21:29 DEBUG "metric": { 03:21:29 DEBUG "data": { 03:21:29 DEBUG "resultType": "vector", 03:21:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:21:29 DEBUG "resultType": "vector", 03:21:29 DEBUG "result": [ 03:21:29 DEBUG }, 03:21:29 DEBUG "result": [ 03:21:29 DEBUG { 03:21:29 DEBUG "value": [ 03:21:29 DEBUG { 03:21:29 DEBUG "metric": { 03:21:29 DEBUG 03:21:29 DEBUG "metric": { 03:21:29 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 03:21:29 DEBUG 03:21:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 03:21:29 DEBUG }, 03:21:29 DEBUG 03:21:29 DEBUG }, 03:21:29 DEBUG "value": [ 03:21:29 DEBUG [...] 03:21:29 DEBUG "value": [ 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG [...] 03:21:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:21:29 DEBUG [...] 03:21:29 DEBUG 03:21:29 DEBUG }, 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG "value": [ 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 1713061289, 03:21:29 DEBUG 03:21:29 DEBUG "pod": "end-user-ui-58878d7668-zg4h2" 03:21:29 DEBUG "0.015" 03:21:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:21:29 DEBUG }, 03:21:29 DEBUG ] 03:21:29 DEBUG }, 03:21:29 DEBUG "value": [ 03:21:29 DEBUG } 03:21:29 DEBUG "value": [ 03:21:29 DEBUG 1713061289, 03:21:29 DEBUG ] 03:21:29 DEBUG 1713061289, 03:21:29 DEBUG "0" 03:21:29 DEBUG } 03:21:29 DEBUG "8349700096" 03:21:29 DEBUG ] 03:21:29 DEBUG } 03:21:29 DEBUG ] 03:21:29 DEBUG } 03:21:29 DEBUG } 03:21:29 DEBUG ] 03:21:29 DEBUG ] 03:21:29 DEBUG } 03:21:29 DEBUG } 03:21:29 DEBUG } 03:21:29 DEBUG } 03:21:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061289" 03:21:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061289" 03:21:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061289" 03:21:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061289" 03:21:29 INFO [http_cmd]: http status code OK 03:21:29 DEBUG --- status code --- 03:21:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:29 DEBUG --- http response --- 03:21:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:21:29 DEBUG ----- output ----- 03:21:29 DEBUG { 03:21:29 DEBUG "status": "success", 03:21:29 DEBUG "data": { 03:21:29 DEBUG "resultType": "vector", 03:21:29 DEBUG "result": [ 03:21:29 DEBUG { 03:21:29 DEBUG "metric": { 03:21:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:21:29 DEBUG }, 03:21:29 DEBUG "value": [ 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG [...] 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:21:29 DEBUG }, 03:21:29 DEBUG "value": [ 03:21:29 DEBUG 1713061289, 03:21:29 DEBUG "0" 03:21:29 DEBUG ] 03:21:29 DEBUG } 03:21:29 DEBUG ] 03:21:29 DEBUG } 03:21:29 DEBUG } 03:21:29 INFO [http_cmd]: http status code OK 03:21:29 DEBUG --- status code --- 03:21:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:29 DEBUG --- http response --- 03:21:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:21:29 DEBUG ----- output ----- 03:21:29 DEBUG { 03:21:29 DEBUG "status": "success", 03:21:29 DEBUG "data": { 03:21:29 DEBUG "resultType": "vector", 03:21:29 DEBUG "result": [ 03:21:29 DEBUG { 03:21:29 DEBUG "metric": { 03:21:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:21:29 DEBUG }, 03:21:29 DEBUG "value": [ 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG [...] 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:21:29 DEBUG }, 03:21:29 DEBUG "value": [ 03:21:29 DEBUG 1713061289, 03:21:29 DEBUG "2284.9666666666667" 03:21:29 DEBUG ] 03:21:29 DEBUG } 03:21:29 DEBUG ] 03:21:29 DEBUG } 03:21:29 DEBUG } 03:21:29 INFO [http_cmd]: http status code OK 03:21:29 DEBUG --- status code --- 03:21:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:29 DEBUG --- http response --- 03:21:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:21:29 DEBUG ----- output ----- 03:21:29 DEBUG { 03:21:29 DEBUG "status": "success", 03:21:29 DEBUG "data": { 03:21:29 DEBUG "resultType": "vector", 03:21:29 DEBUG "result": [ 03:21:29 DEBUG { 03:21:29 DEBUG "metric": { 03:21:29 DEBUG "instance": "10.142.0.107:9100" 03:21:29 DEBUG }, 03:21:29 DEBUG "value": [ 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 INFO [http_cmd]: http status code OK 03:21:29 DEBUG 03:21:29 DEBUG --- status code --- 03:21:29 DEBUG [...] 03:21:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:29 DEBUG 03:21:29 DEBUG --- http response --- 03:21:29 DEBUG 03:21:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:21:29 DEBUG 03:21:29 DEBUG ----- output ----- 03:21:29 DEBUG "instance": "10.142.0.43:9100" 03:21:29 DEBUG { 03:21:29 DEBUG }, 03:21:29 DEBUG "status": "success", 03:21:29 DEBUG "value": [ 03:21:29 DEBUG "data": { 03:21:29 DEBUG 1713061289, 03:21:29 DEBUG "resultType": "vector", 03:21:29 DEBUG "5136.033333333334" 03:21:29 DEBUG "result": [ 03:21:29 DEBUG ] 03:21:29 DEBUG { 03:21:29 DEBUG } 03:21:29 DEBUG "metric": { 03:21:29 DEBUG ] 03:21:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:21:29 DEBUG } 03:21:29 DEBUG }, 03:21:29 DEBUG } 03:21:29 DEBUG "value": [ 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG [...] 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:21:29 DEBUG }, 03:21:29 DEBUG "value": [ 03:21:29 DEBUG 1713061289, 03:21:29 DEBUG "1145.5333333333333" 03:21:29 DEBUG ] 03:21:29 DEBUG } 03:21:29 DEBUG ] 03:21:29 DEBUG } 03:21:29 DEBUG } 03:21:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061289" 03:21:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061289" 03:21:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061289" 03:21:29 INFO [http_cmd]: http status code OK 03:21:29 DEBUG --- status code --- 03:21:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:29 DEBUG --- http response --- 03:21:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 03:21:29 DEBUG ----- output ----- 03:21:29 DEBUG { 03:21:29 DEBUG "status": "success", 03:21:29 DEBUG "data": { 03:21:29 DEBUG "resultType": "vector", 03:21:29 DEBUG "result": [ 03:21:29 DEBUG { 03:21:29 DEBUG "metric": { 03:21:29 DEBUG "pod": "ds-idrepo-1" 03:21:29 DEBUG }, 03:21:29 DEBUG "value": [ 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG [...] 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG "pod": "ds-cts-1" 03:21:29 DEBUG }, 03:21:29 DEBUG "value": [ 03:21:29 DEBUG 1713061289, 03:21:29 DEBUG "0.00017925" 03:21:29 DEBUG ] 03:21:29 DEBUG } 03:21:29 DEBUG ] 03:21:29 DEBUG } 03:21:29 DEBUG } 03:21:29 INFO [http_cmd]: http status code OK 03:21:29 DEBUG --- status code --- 03:21:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:29 DEBUG --- http response --- 03:21:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:21:29 DEBUG ----- output ----- 03:21:29 DEBUG { 03:21:29 DEBUG "status": "success", 03:21:29 DEBUG "data": { 03:21:29 DEBUG "resultType": "vector", 03:21:29 DEBUG "result": [ 03:21:29 DEBUG { 03:21:29 DEBUG "metric": { 03:21:29 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 03:21:29 DEBUG }, 03:21:29 DEBUG "value": [ 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG [...] 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG "pod": "end-user-ui-58878d7668-zg4h2" 03:21:29 DEBUG }, 03:21:29 DEBUG "value": [ 03:21:29 DEBUG 1713061289, 03:21:29 DEBUG "0" 03:21:29 DEBUG ] 03:21:29 DEBUG } 03:21:29 DEBUG ] 03:21:29 DEBUG } 03:21:29 DEBUG } 03:21:29 INFO [http_cmd]: http status code OK 03:21:29 DEBUG --- status code --- 03:21:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:29 DEBUG --- http response --- 03:21:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:21:29 DEBUG ----- output ----- 03:21:29 DEBUG { 03:21:29 DEBUG "status": "success", 03:21:29 DEBUG "data": { 03:21:29 DEBUG "resultType": "vector", 03:21:29 DEBUG "result": [ 03:21:29 DEBUG { 03:21:29 DEBUG "metric": { 03:21:29 DEBUG "pod": "ds-idrepo-1" 03:21:29 DEBUG }, 03:21:29 DEBUG "value": [ 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG [...] 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG "pod": "ds-idrepo-0" 03:21:29 DEBUG }, 03:21:29 DEBUG "value": [ 03:21:29 DEBUG 1713061289, 03:21:29 DEBUG "0" 03:21:29 DEBUG ] 03:21:29 DEBUG } 03:21:29 DEBUG ] 03:21:29 DEBUG } 03:21:29 DEBUG } 03:21:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061289" 03:21:29 INFO [http_cmd]: http status code OK 03:21:29 DEBUG --- status code --- 03:21:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:29 DEBUG --- http response --- 03:21:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 03:21:29 DEBUG ----- output ----- 03:21:29 DEBUG { 03:21:29 DEBUG "status": "success", 03:21:29 DEBUG "data": { 03:21:29 DEBUG "resultType": "vector", 03:21:29 DEBUG "result": [ 03:21:29 DEBUG { 03:21:29 DEBUG "metric": { 03:21:29 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 03:21:29 DEBUG }, 03:21:29 DEBUG "value": [ 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG [...] 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG "pod": "end-user-ui-58878d7668-zg4h2" 03:21:29 DEBUG }, 03:21:29 DEBUG "value": [ 03:21:29 DEBUG 1713061289, 03:21:29 DEBUG "0.000023632255435786065" 03:21:29 DEBUG ] 03:21:29 DEBUG } 03:21:29 DEBUG ] 03:21:29 DEBUG } 03:21:29 DEBUG } 03:21:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061289" 03:21:29 INFO [http_cmd]: http status code OK 03:21:29 DEBUG --- status code --- 03:21:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:29 DEBUG --- http response --- 03:21:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:21:29 DEBUG ----- output ----- 03:21:29 DEBUG { 03:21:29 DEBUG "status": "success", 03:21:29 DEBUG "data": { 03:21:29 DEBUG "resultType": "vector", 03:21:29 DEBUG "result": [ 03:21:29 DEBUG { 03:21:29 DEBUG "metric": { 03:21:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:21:29 DEBUG }, 03:21:29 DEBUG "value": [ 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG [...] 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:21:29 DEBUG }, 03:21:29 DEBUG "value": [ 03:21:29 DEBUG 1713061289, 03:21:29 DEBUG "1145.5333333333333" 03:21:29 DEBUG ] 03:21:29 DEBUG } 03:21:29 DEBUG ] 03:21:29 DEBUG } 03:21:29 DEBUG } 03:21:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061289" 03:21:29 INFO [http_cmd]: http status code OK 03:21:29 DEBUG --- status code --- 03:21:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:29 DEBUG --- http response --- 03:21:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:21:29 DEBUG ----- output ----- 03:21:29 DEBUG { 03:21:29 DEBUG "status": "success", 03:21:29 DEBUG "data": { 03:21:29 DEBUG "resultType": "vector", 03:21:29 DEBUG "result": [ 03:21:29 DEBUG { 03:21:29 DEBUG "metric": { 03:21:29 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 03:21:29 DEBUG }, 03:21:29 DEBUG "value": [ 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG [...] 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG "pod": "end-user-ui-58878d7668-zg4h2" 03:21:29 DEBUG }, 03:21:29 DEBUG "value": [ 03:21:29 DEBUG 1713061289, 03:21:29 DEBUG "0" 03:21:29 DEBUG ] 03:21:29 DEBUG } 03:21:29 DEBUG ] 03:21:29 DEBUG } 03:21:29 DEBUG } 03:21:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061289" 03:21:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061289" 03:21:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061289" 03:21:29 INFO [http_cmd]: http status code OK 03:21:29 DEBUG --- status code --- 03:21:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:29 DEBUG --- http response --- 03:21:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:21:29 DEBUG ----- output ----- 03:21:29 DEBUG { 03:21:29 DEBUG "status": "success", 03:21:29 DEBUG "data": { 03:21:29 DEBUG "resultType": "vector", 03:21:29 DEBUG "result": [ 03:21:29 DEBUG { 03:21:29 DEBUG "metric": { 03:21:29 DEBUG "instance": "10.142.0.107:9100" 03:21:29 DEBUG }, 03:21:29 DEBUG "value": [ 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG [...] 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG "instance": "10.142.0.43:9100" 03:21:29 DEBUG }, 03:21:29 DEBUG "value": [ 03:21:29 DEBUG 1713061289, 03:21:29 DEBUG "12651955200" 03:21:29 DEBUG ] 03:21:29 DEBUG } 03:21:29 DEBUG ] 03:21:29 DEBUG } 03:21:29 DEBUG } 03:21:29 INFO [http_cmd]: http status code OK 03:21:29 DEBUG --- status code --- 03:21:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:29 DEBUG --- http response --- 03:21:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:21:29 DEBUG ----- output ----- 03:21:29 DEBUG { 03:21:29 DEBUG "status": "success", 03:21:29 DEBUG "data": { 03:21:29 DEBUG "resultType": "vector", 03:21:29 DEBUG "result": [ 03:21:29 DEBUG { 03:21:29 DEBUG "metric": { 03:21:29 DEBUG "instance": "10.142.0.107:9100" 03:21:29 DEBUG }, 03:21:29 DEBUG "value": [ 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG [...] 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG "instance": "10.142.0.43:9100" 03:21:29 DEBUG }, 03:21:29 DEBUG "value": [ 03:21:29 DEBUG 1713061289, 03:21:29 DEBUG "2767.9333333333334" 03:21:29 DEBUG ] 03:21:29 DEBUG } 03:21:29 DEBUG ] 03:21:29 DEBUG } 03:21:29 DEBUG } 03:21:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061289" 03:21:29 INFO [http_cmd]: http status code OK 03:21:29 DEBUG --- status code --- 03:21:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:29 DEBUG --- http response --- 03:21:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:21:29 DEBUG ----- output ----- 03:21:29 DEBUG { 03:21:29 DEBUG "status": "success", 03:21:29 DEBUG "data": { 03:21:29 DEBUG "resultType": "vector", 03:21:29 DEBUG "result": [ 03:21:29 DEBUG { 03:21:29 DEBUG "metric": { 03:21:29 DEBUG "instance": "10.142.0.107:9100" 03:21:29 DEBUG }, 03:21:29 DEBUG "value": [ 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG [...] 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG "instance": "10.142.0.43:9100" 03:21:29 DEBUG }, 03:21:29 DEBUG "value": [ 03:21:29 DEBUG 1713061289, 03:21:29 DEBUG "0.0002333333333332632" 03:21:29 DEBUG ] 03:21:29 DEBUG } 03:21:29 DEBUG ] 03:21:29 DEBUG } 03:21:29 DEBUG } 03:21:29 INFO [http_cmd]: http status code OK 03:21:29 DEBUG --- status code --- 03:21:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:29 DEBUG --- http response --- 03:21:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:21:29 DEBUG ----- output ----- 03:21:29 DEBUG { 03:21:29 DEBUG "status": "success", 03:21:29 DEBUG "data": { 03:21:29 DEBUG "resultType": "vector", 03:21:29 DEBUG "result": [ 03:21:29 DEBUG { 03:21:29 DEBUG "metric": { 03:21:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:21:29 DEBUG }, 03:21:29 DEBUG "value": [ 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG [...] 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG 03:21:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:21:29 DEBUG }, 03:21:29 DEBUG "value": [ 03:21:29 DEBUG 1713061289, 03:21:29 DEBUG "1145.6" 03:21:29 DEBUG ] 03:21:29 DEBUG } 03:21:29 DEBUG ] 03:21:29 DEBUG } 03:21:29 DEBUG } 03:21:29 INFO 03:21:29 INFO [loop_until]: kubectl --namespace=xlou top pods 03:21:29 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:21:29 INFO [loop_until]: OK (rc = 0) 03:21:29 DEBUG --- stdout --- 03:21:29 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 7494m 7346Mi am-7d597cf7d6-8vr8f 6463m 8382Mi am-7d597cf7d6-9wz9h 9767m 8989Mi ds-cts-0 7168m 7858Mi ds-cts-1 7832m 8042Mi ds-cts-2 7693m 8135Mi ds-idrepo-0 6329m 4454Mi ds-idrepo-1 1033m 4286Mi ds-idrepo-2 1084m 4104Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 5m 990Mi idm-b85f9d978-rl76f 4m 983Mi lodemon-766dc9b5df-2jmd4 2m 62Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 7993m 1229Mi 03:21:29 DEBUG --- stderr --- 03:21:29 DEBUG 03:21:29 INFO 03:21:29 INFO [loop_until]: kubectl --namespace=xlou top node 03:21:29 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:21:29 INFO [loop_until]: OK (rc = 0) 03:21:29 DEBUG --- stdout --- 03:21:29 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 61m 0% 4813Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7462m 24% 10279Mi 9% gke-xlou-cdm-default-pool-1b0769c8-nhlf 7191m 24% 9377Mi 8% gke-xlou-cdm-default-pool-1b0769c8-qkns 7634m 25% 3103Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 1140m 3% 6349Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 7611m 25% 11247Mi 10% gke-xlou-cdm-default-pool-1b0769c8-xnmp 65m 0% 2902Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 6767m 22% 9826Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1141m 3% 5516Mi 4% gke-xlou-cdm-ds-0f601751-j488 7957m 26% 9514Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 60m 0% 2192Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 8394m 28% 10601Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 6618m 22% 5915Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6470m 40% 3823Mi 6% 03:21:29 DEBUG --- stderr --- 03:21:29 DEBUG 03:21:44 WARNING File /tmp/lodemon_data-20240414_031429/node_disk_read_bytes_total.json does not exist. Skipping average value computation... 127.0.0.1 - - [14/Apr/2024 03:21:44] "GET /monitoring/average?start_time=20240414_021918&stop_time=20240414_021944 HTTP/1.1" 200 - 03:22:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061349" 03:22:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061349" 03:22:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061349" 03:22:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713061349" 03:22:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713061349" 03:22:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061349" 03:22:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061349" 03:22:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713061349" 03:22:29 INFO [http_cmd]: http status code OK 03:22:29 DEBUG --- status code --- 03:22:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:29 DEBUG --- http response --- 03:22:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:22:29 DEBUG ----- output ----- 03:22:29 DEBUG { 03:22:29 DEBUG "status": "success", 03:22:29 DEBUG "data": { 03:22:29 DEBUG "resultType": "vector", 03:22:29 DEBUG "result": [ 03:22:29 DEBUG { 03:22:29 DEBUG "metric": { 03:22:29 DEBUG "pod": "ds-idrepo-1" 03:22:29 DEBUG }, 03:22:29 DEBUG "value": [ 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG [...] 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG "pod": "ds-idrepo-0" 03:22:29 DEBUG }, 03:22:29 DEBUG "value": [ 03:22:29 DEBUG 1713061349, 03:22:29 DEBUG "0" 03:22:29 DEBUG ] 03:22:29 DEBUG } 03:22:29 DEBUG ] 03:22:29 DEBUG } 03:22:29 DEBUG } 03:22:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061349" 03:22:29 ERROR [http_cmd]: ERROR 03:22:29 INFO [http_cmd]: http status code OK 03:22:29 ERROR --- status code --- 03:22:29 DEBUG --- status code --- 03:22:29 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:22:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:29 ERROR --- http response --- 03:22:29 DEBUG --- http response --- 03:22:29 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:22:29 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:22:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:22:29 DEBUG ----- output ----- 03:22:29 INFO [http_cmd]: http status code OK 03:22:29 DEBUG { 03:22:29 DEBUG --- status code --- 03:22:29 DEBUG "status": "success", 03:22:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:29 DEBUG "data": { 03:22:29 DEBUG --- http response --- 03:22:29 DEBUG "resultType": "vector", 03:22:29 DEBUG "result": [ 03:22:29 DEBUG { 03:22:29 DEBUG "metric": { 03:22:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:22:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:22:29 DEBUG }, 03:22:29 DEBUG ----- output ----- 03:22:29 DEBUG "value": [ 03:22:29 INFO [http_cmd]: http status code OK 03:22:29 INFO [http_cmd]: http status code OK 03:22:29 INFO [http_cmd]: http status code OK 03:22:29 DEBUG { 03:22:29 DEBUG 03:22:29 DEBUG --- status code --- 03:22:29 DEBUG --- status code --- 03:22:29 DEBUG --- status code --- 03:22:29 DEBUG "status": "success", 03:22:29 DEBUG 03:22:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:29 DEBUG "data": { 03:22:29 DEBUG 03:22:29 DEBUG --- http response --- 03:22:29 DEBUG --- http response --- 03:22:29 DEBUG --- http response --- 03:22:29 DEBUG "resultType": "vector", 03:22:29 DEBUG [...] 03:22:29 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:22:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:22:29 DEBUG "result": [ 03:22:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:22:29 DEBUG 03:22:29 DEBUG ----- output ----- 03:22:29 DEBUG { 03:22:29 DEBUG ----- output ----- 03:22:29 DEBUG 03:22:29 DEBUG { 03:22:29 DEBUG "metric": { 03:22:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 03:22:29 DEBUG 03:22:29 DEBUG "status": "success", 03:22:29 DEBUG { 03:22:29 DEBUG }, 03:22:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:22:29 DEBUG "data": { 03:22:29 DEBUG "status": "success", 03:22:29 DEBUG "value": [ 03:22:29 DEBUG }, 03:22:29 DEBUG "resultType": "vector", 03:22:29 DEBUG "data": { 03:22:29 DEBUG 03:22:29 DEBUG "value": [ 03:22:29 DEBUG "result": [ 03:22:29 DEBUG "resultType": "vector", 03:22:29 DEBUG 03:22:29 DEBUG 1713061349, 03:22:29 DEBUG { 03:22:29 DEBUG "result": [ 03:22:29 DEBUG 03:22:29 DEBUG "172444" 03:22:29 DEBUG "metric": { 03:22:29 DEBUG { 03:22:29 DEBUG [...] 03:22:29 DEBUG ] 03:22:29 DEBUG "pod": "ds-idrepo-1" 03:22:29 DEBUG "metric": { 03:22:29 DEBUG 03:22:29 DEBUG } 03:22:29 DEBUG }, 03:22:29 DEBUG "pod": "ds-idrepo-1" 03:22:29 DEBUG 03:22:29 DEBUG ] 03:22:29 DEBUG "value": [ 03:22:29 DEBUG }, 03:22:29 DEBUG 03:22:29 DEBUG } 03:22:29 DEBUG 03:22:29 DEBUG "value": [ 03:22:29 INFO [http_cmd]: http status code OK 03:22:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:22:29 DEBUG } 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG --- status code --- 03:22:29 DEBUG }, 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:29 DEBUG "value": [ 03:22:29 DEBUG [...] 03:22:29 DEBUG 03:22:29 DEBUG --- http response --- 03:22:29 DEBUG 1713061349, 03:22:29 DEBUG 03:22:29 DEBUG [...] 03:22:29 DEBUG "1.2555425602041954" 03:22:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG ] 03:22:29 DEBUG ----- output ----- 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG } 03:22:29 DEBUG { 03:22:29 DEBUG "pod": "ds-idrepo-0" 03:22:29 DEBUG 03:22:29 DEBUG ] 03:22:29 DEBUG "status": "success", 03:22:29 DEBUG }, 03:22:29 DEBUG "pod": "ds-cts-1" 03:22:29 DEBUG } 03:22:29 DEBUG "data": { 03:22:29 DEBUG "value": [ 03:22:29 DEBUG }, 03:22:29 DEBUG } 03:22:29 DEBUG "resultType": "vector", 03:22:29 DEBUG 1713061349, 03:22:29 DEBUG "value": [ 03:22:29 DEBUG "result": [ 03:22:29 DEBUG "0" 03:22:29 DEBUG 1713061349, 03:22:29 DEBUG { 03:22:29 DEBUG ] 03:22:29 DEBUG "0" 03:22:29 DEBUG "metric": { 03:22:29 DEBUG } 03:22:29 DEBUG ] 03:22:29 DEBUG "instance": "10.142.0.107:9100" 03:22:29 DEBUG ] 03:22:29 DEBUG } 03:22:29 DEBUG }, 03:22:29 DEBUG } 03:22:29 DEBUG ] 03:22:29 DEBUG "value": [ 03:22:29 DEBUG } 03:22:29 DEBUG } 03:22:29 DEBUG 03:22:29 DEBUG } 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG [...] 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG "instance": "10.142.0.43:9100" 03:22:29 DEBUG }, 03:22:29 DEBUG "value": [ 03:22:29 DEBUG 1713061349, 03:22:29 DEBUG "0" 03:22:29 DEBUG ] 03:22:29 DEBUG } 03:22:29 DEBUG ] 03:22:29 DEBUG } 03:22:29 DEBUG } 03:22:29 INFO [http_cmd]: http status code OK 03:22:29 DEBUG --- status code --- 03:22:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:29 DEBUG --- http response --- 03:22:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:22:29 DEBUG ----- output ----- 03:22:29 DEBUG { 03:22:29 DEBUG "status": "success", 03:22:29 DEBUG "data": { 03:22:29 DEBUG "resultType": "vector", 03:22:29 DEBUG "result": [ 03:22:29 DEBUG { 03:22:29 DEBUG "metric": { 03:22:29 DEBUG "pod": "ds-idrepo-1" 03:22:29 DEBUG }, 03:22:29 DEBUG "value": [ 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG [...] 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG "pod": "ds-idrepo-0" 03:22:29 DEBUG }, 03:22:29 DEBUG "value": [ 03:22:29 DEBUG 1713061349, 03:22:29 DEBUG "0" 03:22:29 DEBUG ] 03:22:29 DEBUG } 03:22:29 DEBUG ] 03:22:29 DEBUG } 03:22:29 DEBUG } 03:22:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713061349" 03:22:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713061349" 03:22:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061349" 03:22:29 INFO [http_cmd]: http status code OK 03:22:29 DEBUG --- status code --- 03:22:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:29 DEBUG --- http response --- 03:22:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:22:29 DEBUG ----- output ----- 03:22:29 DEBUG { 03:22:29 DEBUG "status": "success", 03:22:29 DEBUG "data": { 03:22:29 DEBUG "resultType": "vector", 03:22:29 DEBUG "result": [ 03:22:29 DEBUG { 03:22:29 DEBUG "metric": { 03:22:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:22:29 DEBUG }, 03:22:29 DEBUG "value": [ 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG [...] 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:22:29 DEBUG }, 03:22:29 DEBUG "value": [ 03:22:29 DEBUG 1713061349, 03:22:29 DEBUG "0.015" 03:22:29 DEBUG ] 03:22:29 DEBUG } 03:22:29 DEBUG ] 03:22:29 DEBUG } 03:22:29 DEBUG } 03:22:29 INFO [http_cmd]: http status code OK 03:22:29 DEBUG --- status code --- 03:22:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:29 DEBUG --- http response --- 03:22:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:22:29 DEBUG ----- output ----- 03:22:29 DEBUG { 03:22:29 DEBUG "status": "success", 03:22:29 DEBUG "data": { 03:22:29 DEBUG "resultType": "vector", 03:22:29 DEBUG "result": [ 03:22:29 DEBUG { 03:22:29 DEBUG "metric": { 03:22:29 DEBUG "pod": "ds-idrepo-1" 03:22:29 DEBUG }, 03:22:29 DEBUG "value": [ 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG [...] 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG "pod": "ds-idrepo-0" 03:22:29 DEBUG }, 03:22:29 DEBUG "value": [ 03:22:29 DEBUG 1713061349, 03:22:29 DEBUG "0" 03:22:29 DEBUG ] 03:22:29 DEBUG } 03:22:29 DEBUG ] 03:22:29 DEBUG } 03:22:29 DEBUG } 03:22:29 INFO [http_cmd]: http status code OK 03:22:29 DEBUG --- status code --- 03:22:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:29 DEBUG --- http response --- 03:22:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:22:29 DEBUG ----- output ----- 03:22:29 DEBUG { 03:22:29 DEBUG "status": "success", 03:22:29 DEBUG "data": { 03:22:29 DEBUG "resultType": "vector", 03:22:29 DEBUG "result": [ 03:22:29 DEBUG { 03:22:29 DEBUG "metric": { 03:22:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 03:22:29 DEBUG }, 03:22:29 DEBUG "value": [ 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG [...] 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:22:29 DEBUG }, 03:22:29 DEBUG "value": [ 03:22:29 DEBUG 1713061349, 03:22:29 DEBUG "250368000" 03:22:29 DEBUG ] 03:22:29 DEBUG } 03:22:29 DEBUG ] 03:22:29 DEBUG } 03:22:29 DEBUG } 03:22:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061349" 03:22:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061349" 03:22:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061349" 03:22:29 INFO [http_cmd]: http status code OK 03:22:29 DEBUG --- status code --- 03:22:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:29 DEBUG --- http response --- 03:22:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:22:29 DEBUG ----- output ----- 03:22:29 DEBUG { 03:22:29 DEBUG "status": "success", 03:22:29 DEBUG "data": { 03:22:29 DEBUG "resultType": "vector", 03:22:29 DEBUG "result": [ 03:22:29 DEBUG { 03:22:29 DEBUG "metric": { 03:22:29 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 03:22:29 DEBUG }, 03:22:29 DEBUG "value": [ 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG [...] 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG "pod": "login-ui-5fdf49fc66-2wnx7" 03:22:29 DEBUG }, 03:22:29 DEBUG "value": [ 03:22:29 DEBUG 1713061349, 03:22:29 DEBUG "0" 03:22:29 DEBUG ] 03:22:29 DEBUG } 03:22:29 DEBUG ] 03:22:29 DEBUG } 03:22:29 DEBUG } 03:22:29 INFO [http_cmd]: http status code OK 03:22:29 DEBUG --- status code --- 03:22:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:29 DEBUG --- http response --- 03:22:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:22:29 DEBUG ----- output ----- 03:22:29 INFO [http_cmd]: http status code OK 03:22:29 DEBUG { 03:22:29 DEBUG --- status code --- 03:22:29 DEBUG "status": "success", 03:22:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:29 DEBUG "data": { 03:22:29 DEBUG --- http response --- 03:22:29 DEBUG "resultType": "vector", 03:22:29 DEBUG "result": [ 03:22:29 DEBUG { 03:22:29 DEBUG "metric": { 03:22:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:22:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:22:29 DEBUG ----- output ----- 03:22:29 DEBUG }, 03:22:29 DEBUG { 03:22:29 DEBUG "value": [ 03:22:29 DEBUG "status": "success", 03:22:29 DEBUG 03:22:29 DEBUG "data": { 03:22:29 DEBUG 03:22:29 DEBUG "resultType": "vector", 03:22:29 DEBUG 03:22:29 DEBUG "result": [ 03:22:29 DEBUG [...] 03:22:29 DEBUG { 03:22:29 DEBUG 03:22:29 DEBUG "metric": { 03:22:29 DEBUG 03:22:29 DEBUG "instance": "10.142.0.107:9100" 03:22:29 DEBUG 03:22:29 DEBUG }, 03:22:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:22:29 DEBUG "value": [ 03:22:29 DEBUG }, 03:22:29 DEBUG 03:22:29 DEBUG "value": [ 03:22:29 DEBUG 03:22:29 DEBUG 1713061349, 03:22:29 DEBUG 03:22:29 DEBUG "0" 03:22:29 DEBUG [...] 03:22:29 DEBUG ] 03:22:29 DEBUG 03:22:29 DEBUG } 03:22:29 DEBUG 03:22:29 DEBUG ] 03:22:29 DEBUG 03:22:29 DEBUG } 03:22:29 DEBUG "instance": "10.142.0.43:9100" 03:22:29 DEBUG } 03:22:29 DEBUG }, 03:22:29 DEBUG "value": [ 03:22:29 DEBUG 1713061349, 03:22:29 DEBUG "5394.166666666667" 03:22:29 DEBUG ] 03:22:29 DEBUG } 03:22:29 DEBUG ] 03:22:29 DEBUG } 03:22:29 DEBUG } 03:22:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061349" 03:22:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061349" 03:22:29 INFO [http_cmd]: http status code OK 03:22:29 DEBUG --- status code --- 03:22:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:29 DEBUG --- http response --- 03:22:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:22:29 DEBUG ----- output ----- 03:22:29 DEBUG { 03:22:29 DEBUG "status": "success", 03:22:29 DEBUG "data": { 03:22:29 DEBUG "resultType": "vector", 03:22:29 DEBUG "result": [ 03:22:29 DEBUG { 03:22:29 DEBUG "metric": { 03:22:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:22:29 DEBUG }, 03:22:29 DEBUG "value": [ 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG [...] 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:22:29 DEBUG }, 03:22:29 DEBUG "value": [ 03:22:29 DEBUG 1713061349, 03:22:29 DEBUG "1256.7666666666667" 03:22:29 DEBUG ] 03:22:29 DEBUG } 03:22:29 DEBUG ] 03:22:29 DEBUG } 03:22:29 DEBUG } 03:22:29 INFO [http_cmd]: http status code OK 03:22:29 DEBUG --- status code --- 03:22:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:29 DEBUG --- http response --- 03:22:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:22:29 DEBUG ----- output ----- 03:22:29 DEBUG { 03:22:29 DEBUG "status": "success", 03:22:29 DEBUG "data": { 03:22:29 DEBUG "resultType": "vector", 03:22:29 DEBUG "result": [ 03:22:29 DEBUG { 03:22:29 DEBUG "metric": { 03:22:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:22:29 DEBUG }, 03:22:29 DEBUG "value": [ 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG [...] 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:22:29 DEBUG }, 03:22:29 DEBUG "value": [ 03:22:29 DEBUG 1713061349, 03:22:29 DEBUG "361.73333333333335" 03:22:29 DEBUG ] 03:22:29 DEBUG } 03:22:29 DEBUG ] 03:22:29 DEBUG } 03:22:29 DEBUG } 03:22:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061349" 03:22:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061349" 03:22:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061349" 03:22:29 INFO [http_cmd]: http status code OK 03:22:29 INFO [http_cmd]: http status code OK 03:22:29 INFO [http_cmd]: http status code OK 03:22:29 DEBUG --- status code --- 03:22:29 DEBUG --- status code --- 03:22:29 DEBUG --- status code --- 03:22:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:29 DEBUG --- http response --- 03:22:29 DEBUG --- http response --- 03:22:29 DEBUG --- http response --- 03:22:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 03:22:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:22:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 03:22:29 DEBUG ----- output ----- 03:22:29 DEBUG ----- output ----- 03:22:29 DEBUG ----- output ----- 03:22:29 DEBUG { 03:22:29 DEBUG { 03:22:29 DEBUG { 03:22:29 DEBUG "status": "success", 03:22:29 DEBUG "status": "success", 03:22:29 DEBUG "status": "success", 03:22:29 DEBUG "data": { 03:22:29 DEBUG "data": { 03:22:29 DEBUG "data": { 03:22:29 DEBUG "resultType": "vector", 03:22:29 DEBUG "resultType": "vector", 03:22:29 DEBUG "resultType": "vector", 03:22:29 DEBUG "result": [ 03:22:29 DEBUG "result": [ 03:22:29 DEBUG "result": [ 03:22:29 DEBUG { 03:22:29 DEBUG { 03:22:29 DEBUG { 03:22:29 DEBUG "metric": { 03:22:29 DEBUG "metric": { 03:22:29 DEBUG "metric": { 03:22:29 DEBUG "pod": "ds-idrepo-1" 03:22:29 DEBUG "pod": "ds-idrepo-1" 03:22:29 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 03:22:29 DEBUG }, 03:22:29 DEBUG }, 03:22:29 DEBUG }, 03:22:29 DEBUG "value": [ 03:22:29 DEBUG "value": [ 03:22:29 DEBUG "value": [ 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG [...] 03:22:29 DEBUG [...] 03:22:29 DEBUG [...] 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG "pod": "ds-cts-1" 03:22:29 DEBUG "pod": "ds-idrepo-0" 03:22:29 DEBUG "pod": "login-ui-5fdf49fc66-2wnx7" 03:22:29 DEBUG }, 03:22:29 DEBUG }, 03:22:29 DEBUG }, 03:22:29 DEBUG "value": [ 03:22:29 DEBUG "value": [ 03:22:29 DEBUG "value": [ 03:22:29 DEBUG 1713061349, 03:22:29 DEBUG 1713061349, 03:22:29 DEBUG 1713061349, 03:22:29 DEBUG "0.00006666666666666667" 03:22:29 DEBUG "0" 03:22:29 DEBUG "0" 03:22:29 DEBUG ] 03:22:29 DEBUG ] 03:22:29 DEBUG ] 03:22:29 DEBUG } 03:22:29 DEBUG } 03:22:29 DEBUG } 03:22:29 DEBUG ] 03:22:29 DEBUG ] 03:22:29 DEBUG ] 03:22:29 DEBUG } 03:22:29 DEBUG } 03:22:29 DEBUG } 03:22:29 DEBUG } 03:22:29 DEBUG } 03:22:29 DEBUG } 03:22:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061349" 03:22:29 INFO [http_cmd]: http status code OK 03:22:29 DEBUG --- status code --- 03:22:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:29 DEBUG --- http response --- 03:22:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:22:29 DEBUG ----- output ----- 03:22:29 DEBUG { 03:22:29 DEBUG "status": "success", 03:22:29 DEBUG "data": { 03:22:29 DEBUG "resultType": "vector", 03:22:29 DEBUG "result": [ 03:22:29 DEBUG { 03:22:29 DEBUG "metric": { 03:22:29 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 03:22:29 DEBUG }, 03:22:29 DEBUG "value": [ 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG [...] 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG "pod": "login-ui-5fdf49fc66-2wnx7" 03:22:29 DEBUG }, 03:22:29 DEBUG "value": [ 03:22:29 DEBUG 1713061349, 03:22:29 DEBUG "0.00002175140304851596" 03:22:29 DEBUG ] 03:22:29 DEBUG } 03:22:29 DEBUG ] 03:22:29 DEBUG } 03:22:29 DEBUG } 03:22:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061349" 03:22:29 INFO [http_cmd]: http status code OK 03:22:29 DEBUG --- status code --- 03:22:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:29 DEBUG --- http response --- 03:22:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:22:29 DEBUG ----- output ----- 03:22:29 DEBUG { 03:22:29 DEBUG "status": "success", 03:22:29 DEBUG "data": { 03:22:29 DEBUG "resultType": "vector", 03:22:29 DEBUG "result": [ 03:22:29 DEBUG { 03:22:29 DEBUG "metric": { 03:22:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:22:29 DEBUG }, 03:22:29 DEBUG "value": [ 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG [...] 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:22:29 DEBUG }, 03:22:29 DEBUG "value": [ 03:22:29 DEBUG 1713061349, 03:22:29 DEBUG "361.6333333333333" 03:22:29 DEBUG ] 03:22:29 DEBUG } 03:22:29 DEBUG ] 03:22:29 DEBUG } 03:22:29 DEBUG } 03:22:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061349" 03:22:29 INFO [http_cmd]: http status code OK 03:22:29 DEBUG --- status code --- 03:22:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:29 DEBUG --- http response --- 03:22:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 03:22:29 DEBUG ----- output ----- 03:22:29 DEBUG { 03:22:29 DEBUG "status": "success", 03:22:29 DEBUG "data": { 03:22:29 DEBUG "resultType": "vector", 03:22:29 DEBUG "result": [ 03:22:29 DEBUG { 03:22:29 DEBUG "metric": { 03:22:29 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 03:22:29 DEBUG }, 03:22:29 DEBUG "value": [ 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG [...] 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG "pod": "login-ui-5fdf49fc66-2wnx7" 03:22:29 DEBUG }, 03:22:29 DEBUG "value": [ 03:22:29 DEBUG 1713061349, 03:22:29 DEBUG "0" 03:22:29 DEBUG ] 03:22:29 DEBUG } 03:22:29 DEBUG ] 03:22:29 DEBUG } 03:22:29 DEBUG } 03:22:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061349" 03:22:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061349" 03:22:29 INFO [http_cmd]: http status code OK 03:22:29 DEBUG --- status code --- 03:22:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:29 DEBUG --- http response --- 03:22:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:22:29 DEBUG ----- output ----- 03:22:29 DEBUG { 03:22:29 DEBUG "status": "success", 03:22:29 DEBUG "data": { 03:22:29 DEBUG "resultType": "vector", 03:22:29 DEBUG "result": [ 03:22:29 DEBUG { 03:22:29 DEBUG "metric": { 03:22:29 DEBUG "instance": "10.142.0.107:9100" 03:22:29 DEBUG }, 03:22:29 DEBUG "value": [ 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG [...] 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG "instance": "10.142.0.43:9100" 03:22:29 DEBUG }, 03:22:29 DEBUG "value": [ 03:22:29 DEBUG 1713061349, 03:22:29 DEBUG "12652510208" 03:22:29 DEBUG ] 03:22:29 DEBUG } 03:22:29 DEBUG ] 03:22:29 DEBUG } 03:22:29 DEBUG } 03:22:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061349" 03:22:29 INFO [http_cmd]: http status code OK 03:22:29 DEBUG --- status code --- 03:22:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:29 DEBUG --- http response --- 03:22:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061349" 03:22:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:22:29 DEBUG ----- output ----- 03:22:29 DEBUG { 03:22:29 DEBUG "status": "success", 03:22:29 DEBUG "data": { 03:22:29 DEBUG "resultType": "vector", 03:22:29 DEBUG "result": [ 03:22:29 DEBUG { 03:22:29 DEBUG "metric": { 03:22:29 DEBUG "instance": "10.142.0.107:9100" 03:22:29 DEBUG }, 03:22:29 DEBUG "value": [ 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG [...] 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG "instance": "10.142.0.43:9100" 03:22:29 DEBUG }, 03:22:29 DEBUG "value": [ 03:22:29 DEBUG 1713061349, 03:22:29 DEBUG "3077.4333333333334" 03:22:29 DEBUG ] 03:22:29 DEBUG } 03:22:29 DEBUG ] 03:22:29 DEBUG } 03:22:29 DEBUG } 03:22:29 INFO [http_cmd]: http status code OK 03:22:29 DEBUG --- status code --- 03:22:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:29 DEBUG --- http response --- 03:22:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:22:29 DEBUG ----- output ----- 03:22:29 DEBUG { 03:22:29 DEBUG "status": "success", 03:22:29 DEBUG "data": { 03:22:29 DEBUG "resultType": "vector", 03:22:29 DEBUG "result": [ 03:22:29 DEBUG { 03:22:29 DEBUG "metric": { 03:22:29 DEBUG "instance": "10.142.0.107:9100" 03:22:29 DEBUG }, 03:22:29 DEBUG "value": [ 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG [...] 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG "instance": "10.142.0.43:9100" 03:22:29 DEBUG }, 03:22:29 DEBUG "value": [ 03:22:29 DEBUG 1713061349, 03:22:29 DEBUG "0.0002333333333332632" 03:22:29 DEBUG ] 03:22:29 DEBUG } 03:22:29 DEBUG ] 03:22:29 DEBUG } 03:22:29 DEBUG } 03:22:29 INFO [http_cmd]: http status code OK 03:22:29 DEBUG --- status code --- 03:22:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:29 DEBUG --- http response --- 03:22:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:22:29 DEBUG ----- output ----- 03:22:29 DEBUG { 03:22:29 DEBUG "status": "success", 03:22:29 DEBUG "data": { 03:22:29 DEBUG "resultType": "vector", 03:22:29 DEBUG "result": [ 03:22:29 DEBUG { 03:22:29 DEBUG "metric": { 03:22:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:22:29 DEBUG }, 03:22:29 DEBUG "value": [ 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG [...] 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG 03:22:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:22:29 DEBUG }, 03:22:29 DEBUG "value": [ 03:22:29 DEBUG 1713061349, 03:22:29 DEBUG "361.3" 03:22:29 DEBUG ] 03:22:29 DEBUG } 03:22:29 DEBUG ] 03:22:29 DEBUG } 03:22:29 DEBUG } 03:22:29 INFO 03:22:29 INFO [loop_until]: kubectl --namespace=xlou top pods 03:22:29 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:22:29 INFO [loop_until]: OK (rc = 0) 03:22:29 DEBUG --- stdout --- 03:22:29 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 2215m 7338Mi am-7d597cf7d6-8vr8f 2672m 8381Mi am-7d597cf7d6-9wz9h 4371m 8989Mi ds-cts-0 3362m 7838Mi ds-cts-1 4347m 8024Mi ds-cts-2 3254m 8144Mi ds-idrepo-0 4773m 4520Mi ds-idrepo-1 528m 4286Mi ds-idrepo-2 160m 4103Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 5m 990Mi idm-b85f9d978-rl76f 5m 984Mi lodemon-766dc9b5df-2jmd4 2m 63Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 3358m 438Mi 03:22:29 DEBUG --- stderr --- 03:22:29 DEBUG 03:22:29 INFO 03:22:29 INFO [loop_until]: kubectl --namespace=xlou top node 03:22:29 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:22:29 INFO [loop_until]: OK (rc = 0) 03:22:29 DEBUG --- stdout --- 03:22:29 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 4812Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 4530m 15% 10260Mi 9% gke-xlou-cdm-default-pool-1b0769c8-nhlf 3206m 10% 9406Mi 8% gke-xlou-cdm-default-pool-1b0769c8-qkns 6156m 20% 3031Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 439m 1% 6333Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 4307m 14% 11218Mi 10% gke-xlou-cdm-default-pool-1b0769c8-xnmp 66m 0% 2897Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 3448m 11% 9825Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 383m 1% 5507Mi 4% gke-xlou-cdm-ds-0f601751-j488 4380m 14% 9505Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 61m 0% 2194Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 5419m 18% 10552Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 3854m 12% 5978Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 4099m 25% 4261Mi 7% 03:22:29 DEBUG --- stderr --- 03:22:29 DEBUG 03:23:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061409" 03:23:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713061409" 03:23:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713061409" 03:23:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061409" 03:23:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713061409" 03:23:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061409" 03:23:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061409" 03:23:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713061409" 03:23:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713061409" 03:23:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061409" 03:23:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061409" 03:23:29 ERROR [http_cmd]: ERROR 03:23:29 ERROR --- status code --- 03:23:29 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:23:29 ERROR --- http response --- 03:23:29 INFO [http_cmd]: http status code OK 03:23:29 INFO [http_cmd]: http status code OK 03:23:29 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:23:29 DEBUG --- status code --- 03:23:29 DEBUG --- status code --- 03:23:29 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:23:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:29 DEBUG --- http response --- 03:23:29 DEBUG --- http response --- 03:23:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:23:29 DEBUG ----- output ----- 03:23:29 DEBUG { 03:23:29 DEBUG "status": "success", 03:23:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:23:29 DEBUG "data": { 03:23:29 DEBUG ----- output ----- 03:23:29 DEBUG "resultType": "vector", 03:23:29 DEBUG { 03:23:29 DEBUG "result": [ 03:23:29 DEBUG "status": "success", 03:23:29 DEBUG { 03:23:29 DEBUG "data": { 03:23:29 DEBUG "metric": { 03:23:29 DEBUG "resultType": "vector", 03:23:29 DEBUG "pod": "ds-idrepo-1" 03:23:29 DEBUG "result": [ 03:23:29 DEBUG }, 03:23:29 DEBUG { 03:23:29 DEBUG "value": [ 03:23:29 DEBUG "metric": { 03:23:29 DEBUG 03:23:29 DEBUG "pod": "ds-idrepo-1" 03:23:29 DEBUG 03:23:29 DEBUG }, 03:23:29 DEBUG 03:23:29 DEBUG "value": [ 03:23:29 DEBUG [...] 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG [...] 03:23:29 DEBUG "pod": "ds-idrepo-0" 03:23:29 DEBUG 03:23:29 DEBUG }, 03:23:29 DEBUG 03:23:29 DEBUG "value": [ 03:23:29 DEBUG 03:23:29 DEBUG 1713061409, 03:23:29 DEBUG "pod": "ds-cts-1" 03:23:29 DEBUG "0" 03:23:29 DEBUG }, 03:23:29 DEBUG ] 03:23:29 DEBUG "value": [ 03:23:29 DEBUG } 03:23:29 DEBUG 1713061409, 03:23:29 DEBUG ] 03:23:29 DEBUG "0" 03:23:29 DEBUG } 03:23:29 DEBUG ] 03:23:29 DEBUG } 03:23:29 DEBUG } 03:23:29 DEBUG ] 03:23:29 DEBUG } 03:23:29 DEBUG } 03:23:29 INFO [http_cmd]: http status code OK 03:23:29 DEBUG --- status code --- 03:23:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:29 DEBUG --- http response --- 03:23:29 INFO [http_cmd]: http status code OK 03:23:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:23:29 INFO [http_cmd]: http status code OK 03:23:29 DEBUG --- status code --- 03:23:29 DEBUG ----- output ----- 03:23:29 DEBUG --- status code --- 03:23:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:29 DEBUG { 03:23:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:29 DEBUG --- http response --- 03:23:29 DEBUG "status": "success", 03:23:29 DEBUG --- http response --- 03:23:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:23:29 DEBUG "data": { 03:23:29 DEBUG ----- output ----- 03:23:29 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:23:29 DEBUG "resultType": "vector", 03:23:29 DEBUG { 03:23:29 DEBUG "result": [ 03:23:29 DEBUG "status": "success", 03:23:29 DEBUG { 03:23:29 DEBUG "data": { 03:23:29 DEBUG "metric": { 03:23:29 DEBUG "resultType": "vector", 03:23:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:23:29 DEBUG "result": [ 03:23:29 DEBUG }, 03:23:29 DEBUG { 03:23:29 DEBUG "value": [ 03:23:29 DEBUG "metric": { 03:23:29 DEBUG 03:23:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:23:29 DEBUG 03:23:29 DEBUG }, 03:23:29 DEBUG 03:23:29 DEBUG "value": [ 03:23:29 DEBUG [...] 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 INFO [http_cmd]: http status code OK 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 INFO [http_cmd]: http status code OK 03:23:29 DEBUG --- status code --- 03:23:29 DEBUG 03:23:29 DEBUG [...] 03:23:29 DEBUG --- status code --- 03:23:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:23:29 DEBUG 03:23:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:29 DEBUG --- http response --- 03:23:29 DEBUG }, 03:23:29 DEBUG 03:23:29 DEBUG --- http response --- 03:23:29 DEBUG "value": [ 03:23:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:23:29 DEBUG 03:23:29 INFO [http_cmd]: http status code OK 03:23:29 DEBUG 1713061409, 03:23:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:23:29 DEBUG ----- output ----- 03:23:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:23:29 DEBUG --- status code --- 03:23:29 DEBUG "178607" 03:23:29 DEBUG ----- output ----- 03:23:29 DEBUG { 03:23:29 DEBUG }, 03:23:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:29 DEBUG ] 03:23:29 DEBUG { 03:23:29 DEBUG "status": "success", 03:23:29 DEBUG "value": [ 03:23:29 DEBUG --- http response --- 03:23:29 DEBUG } 03:23:29 DEBUG "status": "success", 03:23:29 DEBUG "data": { 03:23:29 DEBUG 1713061409, 03:23:29 DEBUG ] 03:23:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:23:29 DEBUG "data": { 03:23:29 DEBUG "resultType": "vector", 03:23:29 DEBUG "0.015" 03:23:29 DEBUG } 03:23:29 DEBUG ----- output ----- 03:23:29 DEBUG "resultType": "vector", 03:23:29 DEBUG "result": [ 03:23:29 DEBUG ] 03:23:29 DEBUG } 03:23:29 DEBUG { 03:23:29 DEBUG "result": [ 03:23:29 DEBUG { 03:23:29 DEBUG } 03:23:29 DEBUG "status": "success", 03:23:29 DEBUG { 03:23:29 DEBUG "metric": { 03:23:29 DEBUG ] 03:23:29 DEBUG "data": { 03:23:29 DEBUG "metric": { 03:23:29 DEBUG "instance": "10.142.0.107:9100" 03:23:29 DEBUG } 03:23:29 DEBUG "resultType": "vector", 03:23:29 DEBUG "pod": "ds-idrepo-1" 03:23:29 DEBUG }, 03:23:29 DEBUG } 03:23:29 DEBUG "result": [ 03:23:29 DEBUG }, 03:23:29 DEBUG "value": [ 03:23:29 DEBUG { 03:23:29 DEBUG "value": [ 03:23:29 DEBUG 03:23:29 DEBUG "metric": { 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG }, 03:23:29 DEBUG 03:23:29 DEBUG [...] 03:23:29 DEBUG "value": [ 03:23:29 DEBUG [...] 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG "instance": "10.142.0.43:9100" 03:23:29 DEBUG [...] 03:23:29 DEBUG "pod": "ds-idrepo-0" 03:23:29 DEBUG }, 03:23:29 DEBUG 03:23:29 DEBUG }, 03:23:29 DEBUG "value": [ 03:23:29 DEBUG 03:23:29 DEBUG "value": [ 03:23:29 DEBUG 1713061409, 03:23:29 DEBUG 03:23:29 DEBUG 1713061409, 03:23:29 DEBUG "0" 03:23:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:23:29 DEBUG "0" 03:23:29 DEBUG ] 03:23:29 DEBUG }, 03:23:29 DEBUG ] 03:23:29 DEBUG } 03:23:29 DEBUG "value": [ 03:23:29 DEBUG } 03:23:29 DEBUG ] 03:23:29 DEBUG 1713061409, 03:23:29 DEBUG ] 03:23:29 DEBUG } 03:23:29 DEBUG "3283546112" 03:23:29 DEBUG } 03:23:29 DEBUG } 03:23:29 DEBUG ] 03:23:29 DEBUG } 03:23:29 DEBUG } 03:23:29 INFO [http_cmd]: http status code OK 03:23:29 DEBUG ] 03:23:29 DEBUG --- status code --- 03:23:29 DEBUG } 03:23:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:29 DEBUG } 03:23:29 DEBUG --- http response --- 03:23:29 INFO [http_cmd]: http status code OK 03:23:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:23:29 DEBUG --- status code --- 03:23:29 DEBUG ----- output ----- 03:23:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:29 DEBUG { 03:23:29 DEBUG --- http response --- 03:23:29 DEBUG "status": "success", 03:23:29 DEBUG "data": { 03:23:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:23:29 DEBUG "resultType": "vector", 03:23:29 DEBUG ----- output ----- 03:23:29 DEBUG "result": [ 03:23:29 DEBUG { 03:23:29 DEBUG { 03:23:29 DEBUG "status": "success", 03:23:29 DEBUG "metric": { 03:23:29 DEBUG "data": { 03:23:29 DEBUG "pod": "ds-idrepo-1" 03:23:29 DEBUG "resultType": "vector", 03:23:29 DEBUG }, 03:23:29 DEBUG "result": [ 03:23:29 DEBUG "value": [ 03:23:29 DEBUG { 03:23:29 DEBUG 03:23:29 DEBUG "metric": { 03:23:29 DEBUG 03:23:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 03:23:29 DEBUG 03:23:29 DEBUG }, 03:23:29 DEBUG [...] 03:23:29 DEBUG "value": [ 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG "pod": "ds-idrepo-0" 03:23:29 DEBUG [...] 03:23:29 DEBUG }, 03:23:29 DEBUG 03:23:29 DEBUG "value": [ 03:23:29 DEBUG 03:23:29 DEBUG 1713061409, 03:23:29 DEBUG 03:23:29 DEBUG "0" 03:23:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:23:29 DEBUG ] 03:23:29 DEBUG }, 03:23:29 DEBUG } 03:23:29 DEBUG "value": [ 03:23:29 DEBUG ] 03:23:29 DEBUG 1713061409, 03:23:29 DEBUG } 03:23:29 DEBUG "8.031092179312825" 03:23:29 DEBUG } 03:23:29 DEBUG ] 03:23:29 DEBUG } 03:23:29 DEBUG ] 03:23:29 DEBUG } 03:23:29 DEBUG } 03:23:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061409" 03:23:29 INFO [http_cmd]: http status code OK 03:23:29 DEBUG --- status code --- 03:23:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:29 DEBUG --- http response --- 03:23:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:23:29 DEBUG ----- output ----- 03:23:29 DEBUG { 03:23:29 DEBUG "status": "success", 03:23:29 DEBUG "data": { 03:23:29 DEBUG "resultType": "vector", 03:23:29 DEBUG "result": [ 03:23:29 DEBUG { 03:23:29 DEBUG "metric": { 03:23:29 DEBUG "pod": "ds-idrepo-1" 03:23:29 DEBUG }, 03:23:29 DEBUG "value": [ 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG [...] 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG "pod": "ds-idrepo-0" 03:23:29 DEBUG }, 03:23:29 DEBUG "value": [ 03:23:29 DEBUG 1713061409, 03:23:29 DEBUG "0" 03:23:29 DEBUG ] 03:23:29 DEBUG } 03:23:29 DEBUG ] 03:23:29 DEBUG } 03:23:29 DEBUG } 03:23:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061409" 03:23:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061409" 03:23:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061409" 03:23:29 INFO [http_cmd]: http status code OK 03:23:29 DEBUG --- status code --- 03:23:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:29 DEBUG --- http response --- 03:23:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:23:29 DEBUG ----- output ----- 03:23:29 DEBUG { 03:23:29 DEBUG "status": "success", 03:23:29 DEBUG "data": { 03:23:29 DEBUG "resultType": "vector", 03:23:29 DEBUG "result": [ 03:23:29 DEBUG { 03:23:29 DEBUG "metric": { 03:23:29 DEBUG "pod": "ds-cts-2" 03:23:29 DEBUG }, 03:23:29 DEBUG "value": [ 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG [...] 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:23:29 DEBUG }, 03:23:29 DEBUG "value": [ 03:23:29 DEBUG 1713061409, 03:23:29 DEBUG "321951106.80406743" 03:23:29 DEBUG ] 03:23:29 DEBUG } 03:23:29 DEBUG ] 03:23:29 DEBUG } 03:23:29 DEBUG } 03:23:29 INFO [http_cmd]: http status code OK 03:23:29 DEBUG --- status code --- 03:23:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:29 DEBUG --- http response --- 03:23:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:23:29 DEBUG ----- output ----- 03:23:29 DEBUG { 03:23:29 DEBUG "status": "success", 03:23:29 DEBUG "data": { 03:23:29 DEBUG "resultType": "vector", 03:23:29 DEBUG "result": [ 03:23:29 DEBUG { 03:23:29 DEBUG "metric": { 03:23:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:23:29 DEBUG }, 03:23:29 DEBUG "value": [ 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG [...] 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:23:29 DEBUG }, 03:23:29 DEBUG "value": [ 03:23:29 DEBUG 1713061409, 03:23:29 DEBUG "0" 03:23:29 DEBUG ] 03:23:29 DEBUG } 03:23:29 DEBUG ] 03:23:29 DEBUG } 03:23:29 DEBUG } 03:23:29 INFO [http_cmd]: http status code OK 03:23:29 DEBUG --- status code --- 03:23:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:29 DEBUG --- http response --- 03:23:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:23:29 DEBUG ----- output ----- 03:23:29 DEBUG { 03:23:29 DEBUG "status": "success", 03:23:29 DEBUG "data": { 03:23:29 DEBUG "resultType": "vector", 03:23:29 DEBUG "result": [ 03:23:29 DEBUG { 03:23:29 DEBUG "metric": { 03:23:29 DEBUG "instance": "10.142.0.107:9100" 03:23:29 DEBUG }, 03:23:29 DEBUG "value": [ 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG [...] 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG "instance": "10.142.0.43:9100" 03:23:29 DEBUG }, 03:23:29 DEBUG "value": [ 03:23:29 DEBUG 1713061409, 03:23:29 DEBUG "5123.000000000001" 03:23:29 DEBUG ] 03:23:29 DEBUG } 03:23:29 DEBUG ] 03:23:29 DEBUG } 03:23:29 DEBUG } 03:23:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061409" 03:23:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061409" 03:23:29 INFO [http_cmd]: http status code OK 03:23:29 DEBUG --- status code --- 03:23:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:29 DEBUG --- http response --- 03:23:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:23:29 DEBUG ----- output ----- 03:23:29 DEBUG { 03:23:29 DEBUG "status": "success", 03:23:29 DEBUG "data": { 03:23:29 DEBUG "resultType": "vector", 03:23:29 DEBUG "result": [ 03:23:29 DEBUG { 03:23:29 DEBUG "metric": { 03:23:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:23:29 DEBUG }, 03:23:29 DEBUG "value": [ 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG [...] 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:23:29 DEBUG }, 03:23:29 DEBUG "value": [ 03:23:29 DEBUG 1713061409, 03:23:29 DEBUG "1311.1333333333332" 03:23:29 DEBUG ] 03:23:29 DEBUG } 03:23:29 DEBUG ] 03:23:29 DEBUG } 03:23:29 DEBUG } 03:23:29 INFO [http_cmd]: http status code OK 03:23:29 DEBUG --- status code --- 03:23:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:29 DEBUG --- http response --- 03:23:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:23:29 DEBUG ----- output ----- 03:23:29 DEBUG { 03:23:29 DEBUG "status": "success", 03:23:29 DEBUG "data": { 03:23:29 DEBUG "resultType": "vector", 03:23:29 DEBUG "result": [ 03:23:29 DEBUG { 03:23:29 DEBUG "metric": { 03:23:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:23:29 DEBUG }, 03:23:29 DEBUG "value": [ 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG [...] 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:23:29 DEBUG }, 03:23:29 DEBUG "value": [ 03:23:29 DEBUG 1713061409, 03:23:29 DEBUG "1319.5666666666666" 03:23:29 DEBUG ] 03:23:29 DEBUG } 03:23:29 DEBUG ] 03:23:29 DEBUG } 03:23:29 DEBUG } 03:23:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061409" 03:23:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061409" 03:23:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061409" 03:23:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061409" 03:23:29 INFO [http_cmd]: http status code OK 03:23:29 DEBUG --- status code --- 03:23:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:29 DEBUG --- http response --- 03:23:29 INFO [http_cmd]: http status code OK 03:23:29 INFO [http_cmd]: http status code OK 03:23:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:23:29 DEBUG --- status code --- 03:23:29 DEBUG --- status code --- 03:23:29 DEBUG ----- output ----- 03:23:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:29 DEBUG { 03:23:29 DEBUG --- http response --- 03:23:29 DEBUG --- http response --- 03:23:29 DEBUG "status": "success", 03:23:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:23:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061409" 03:23:29 DEBUG "data": { 03:23:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:23:29 DEBUG ----- output ----- 03:23:29 INFO [http_cmd]: http status code OK 03:23:29 DEBUG "resultType": "vector", 03:23:29 DEBUG ----- output ----- 03:23:29 DEBUG { 03:23:29 DEBUG --- status code --- 03:23:29 DEBUG "result": [ 03:23:29 DEBUG { 03:23:29 DEBUG "status": "success", 03:23:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:29 DEBUG { 03:23:29 DEBUG "status": "success", 03:23:29 DEBUG "data": { 03:23:29 DEBUG --- http response --- 03:23:29 DEBUG "metric": { 03:23:29 DEBUG "data": { 03:23:29 DEBUG "resultType": "vector", 03:23:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:23:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 03:23:29 DEBUG "resultType": "vector", 03:23:29 DEBUG "result": [ 03:23:29 DEBUG }, 03:23:29 DEBUG ----- output ----- 03:23:29 DEBUG "result": [ 03:23:29 DEBUG { 03:23:29 DEBUG "value": [ 03:23:29 DEBUG { 03:23:29 DEBUG { 03:23:29 DEBUG "metric": { 03:23:29 DEBUG 03:23:29 DEBUG "status": "success", 03:23:29 DEBUG "metric": { 03:23:29 DEBUG "pod": "ds-cts-2" 03:23:29 DEBUG 03:23:29 DEBUG "data": { 03:23:29 DEBUG "pod": "ds-idrepo-1" 03:23:29 DEBUG }, 03:23:29 DEBUG 03:23:29 DEBUG "resultType": "vector", 03:23:29 DEBUG }, 03:23:29 DEBUG "value": [ 03:23:29 DEBUG [...] 03:23:29 DEBUG "result": [ 03:23:29 DEBUG "value": [ 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG { 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG "metric": { 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG "pod": "ds-idrepo-1" 03:23:29 DEBUG 03:23:29 DEBUG [...] 03:23:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:23:29 DEBUG }, 03:23:29 DEBUG [...] 03:23:29 DEBUG 03:23:29 DEBUG }, 03:23:29 DEBUG "value": [ 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG "value": [ 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG 1713061409, 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:23:29 DEBUG "1319.3333333333333" 03:23:29 DEBUG 03:23:29 DEBUG "pod": "ds-idrepo-0" 03:23:29 DEBUG }, 03:23:29 DEBUG ] 03:23:29 DEBUG [...] 03:23:29 DEBUG }, 03:23:29 DEBUG "value": [ 03:23:29 DEBUG } 03:23:29 DEBUG 03:23:29 DEBUG "value": [ 03:23:29 DEBUG 1713061409, 03:23:29 DEBUG ] 03:23:29 DEBUG 03:23:29 DEBUG 1713061409, 03:23:29 DEBUG "0" 03:23:29 DEBUG } 03:23:29 DEBUG 03:23:29 DEBUG "0" 03:23:29 DEBUG ] 03:23:29 DEBUG } 03:23:29 DEBUG "pod": "ds-cts-1" 03:23:29 DEBUG ] 03:23:29 DEBUG } 03:23:29 DEBUG }, 03:23:29 DEBUG } 03:23:29 DEBUG ] 03:23:29 DEBUG "value": [ 03:23:29 DEBUG ] 03:23:29 DEBUG } 03:23:29 DEBUG 1713061409, 03:23:29 DEBUG } 03:23:29 DEBUG } 03:23:29 DEBUG "0.00005419444444444444" 03:23:29 DEBUG } 03:23:29 DEBUG ] 03:23:29 DEBUG } 03:23:29 DEBUG ] 03:23:29 DEBUG } 03:23:29 DEBUG } 03:23:29 INFO [http_cmd]: http status code OK 03:23:29 DEBUG --- status code --- 03:23:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:29 DEBUG --- http response --- 03:23:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:23:29 DEBUG ----- output ----- 03:23:29 DEBUG { 03:23:29 DEBUG "status": "success", 03:23:29 DEBUG "data": { 03:23:29 DEBUG "resultType": "vector", 03:23:29 DEBUG "result": [ 03:23:29 DEBUG { 03:23:29 DEBUG "metric": { 03:23:29 DEBUG "pod": "ds-cts-2" 03:23:29 DEBUG }, 03:23:29 DEBUG "value": [ 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG [...] 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:23:29 DEBUG }, 03:23:29 DEBUG "value": [ 03:23:29 DEBUG 1713061409, 03:23:29 DEBUG "9.566401760731587" 03:23:29 DEBUG ] 03:23:29 DEBUG } 03:23:29 DEBUG ] 03:23:29 DEBUG } 03:23:29 DEBUG } 03:23:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061409" 03:23:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061409" 03:23:29 INFO [http_cmd]: http status code OK 03:23:29 DEBUG --- status code --- 03:23:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:29 DEBUG --- http response --- 03:23:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:23:29 DEBUG ----- output ----- 03:23:29 DEBUG { 03:23:29 DEBUG "status": "success", 03:23:29 DEBUG "data": { 03:23:29 DEBUG "resultType": "vector", 03:23:29 DEBUG "result": [ 03:23:29 DEBUG { 03:23:29 DEBUG "metric": { 03:23:29 DEBUG "instance": "10.142.0.107:9100" 03:23:29 DEBUG }, 03:23:29 DEBUG "value": [ 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG [...] 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG "instance": "10.142.0.43:9100" 03:23:29 DEBUG }, 03:23:29 DEBUG "value": [ 03:23:29 DEBUG 1713061409, 03:23:29 DEBUG "12653151232" 03:23:29 DEBUG ] 03:23:29 DEBUG } 03:23:29 DEBUG ] 03:23:29 DEBUG } 03:23:29 DEBUG } 03:23:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061409" 03:23:29 INFO [http_cmd]: http status code OK 03:23:29 DEBUG --- status code --- 03:23:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:29 DEBUG --- http response --- 03:23:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:23:29 DEBUG ----- output ----- 03:23:29 DEBUG { 03:23:29 DEBUG "status": "success", 03:23:29 DEBUG "data": { 03:23:29 DEBUG "resultType": "vector", 03:23:29 DEBUG "result": [ 03:23:29 DEBUG { 03:23:29 DEBUG "metric": { 03:23:29 DEBUG "pod": "ds-cts-2" 03:23:29 DEBUG }, 03:23:29 DEBUG "value": [ 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG [...] 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:23:29 DEBUG }, 03:23:29 DEBUG "value": [ 03:23:29 DEBUG 1713061409, 03:23:29 DEBUG "5584514.297413499" 03:23:29 DEBUG ] 03:23:29 DEBUG } 03:23:29 DEBUG ] 03:23:29 DEBUG } 03:23:29 DEBUG } 03:23:29 INFO [http_cmd]: http status code OK 03:23:29 DEBUG --- status code --- 03:23:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:29 DEBUG --- http response --- 03:23:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:23:29 DEBUG ----- output ----- 03:23:29 DEBUG { 03:23:29 DEBUG "status": "success", 03:23:29 DEBUG "data": { 03:23:29 DEBUG "resultType": "vector", 03:23:29 DEBUG "result": [ 03:23:29 DEBUG { 03:23:29 DEBUG "metric": { 03:23:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:23:29 DEBUG }, 03:23:29 DEBUG "value": [ 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG [...] 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:23:29 DEBUG }, 03:23:29 DEBUG "value": [ 03:23:29 DEBUG 1713061409, 03:23:29 DEBUG "1320.5666666666666" 03:23:29 DEBUG ] 03:23:29 DEBUG } 03:23:29 DEBUG ] 03:23:29 DEBUG } 03:23:29 DEBUG } 03:23:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061409" 03:23:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061409" 03:23:29 INFO [http_cmd]: http status code OK 03:23:29 DEBUG --- status code --- 03:23:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:29 DEBUG --- http response --- 03:23:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:23:29 DEBUG ----- output ----- 03:23:29 DEBUG { 03:23:29 DEBUG "status": "success", 03:23:29 DEBUG "data": { 03:23:29 DEBUG "resultType": "vector", 03:23:29 DEBUG "result": [ 03:23:29 DEBUG { 03:23:29 DEBUG "metric": { 03:23:29 DEBUG "instance": "10.142.0.107:9100" 03:23:29 DEBUG }, 03:23:29 DEBUG "value": [ 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG [...] 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG "instance": "10.142.0.43:9100" 03:23:29 DEBUG }, 03:23:29 DEBUG "value": [ 03:23:29 DEBUG 1713061409, 03:23:29 DEBUG "2829.2" 03:23:29 DEBUG ] 03:23:29 DEBUG } 03:23:29 DEBUG ] 03:23:29 DEBUG } 03:23:29 DEBUG } 03:23:29 INFO [http_cmd]: http status code OK 03:23:29 DEBUG --- status code --- 03:23:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:29 DEBUG --- http response --- 03:23:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:23:29 DEBUG ----- output ----- 03:23:29 DEBUG { 03:23:29 DEBUG "status": "success", 03:23:29 DEBUG "data": { 03:23:29 DEBUG "resultType": "vector", 03:23:29 DEBUG "result": [ 03:23:29 DEBUG { 03:23:29 DEBUG "metric": { 03:23:29 DEBUG "instance": "10.142.0.107:9100" 03:23:29 DEBUG }, 03:23:29 DEBUG "value": [ 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG [...] 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG 03:23:29 DEBUG "instance": "10.142.0.43:9100" 03:23:29 DEBUG }, 03:23:29 DEBUG "value": [ 03:23:29 DEBUG 1713061409, 03:23:29 DEBUG "0.0004333333333332708" 03:23:29 DEBUG ] 03:23:29 DEBUG } 03:23:29 DEBUG ] 03:23:29 DEBUG } 03:23:29 DEBUG } 03:23:29 INFO 03:23:29 INFO [loop_until]: kubectl --namespace=xlou top pods 03:23:29 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:23:29 INFO [loop_until]: OK (rc = 0) 03:23:29 DEBUG --- stdout --- 03:23:29 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 7075m 7349Mi am-7d597cf7d6-8vr8f 6431m 8395Mi am-7d597cf7d6-9wz9h 8869m 8995Mi ds-cts-0 6004m 7859Mi ds-cts-1 6081m 8022Mi ds-cts-2 6017m 8147Mi ds-idrepo-0 7019m 4510Mi ds-idrepo-1 1094m 4287Mi ds-idrepo-2 1010m 4104Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 10m 990Mi idm-b85f9d978-rl76f 4m 984Mi lodemon-766dc9b5df-2jmd4 1m 63Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 8018m 3379Mi 03:23:29 DEBUG --- stderr --- 03:23:29 DEBUG 03:23:29 INFO 03:23:29 INFO [loop_until]: kubectl --namespace=xlou top node 03:23:29 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:23:29 INFO [loop_until]: OK (rc = 0) 03:23:29 DEBUG --- stdout --- 03:23:29 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 4819Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5838m 19% 10292Mi 9% gke-xlou-cdm-default-pool-1b0769c8-nhlf 7242m 24% 9394Mi 8% gke-xlou-cdm-default-pool-1b0769c8-qkns 11121m 37% 3587Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 1242m 4% 6342Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 5740m 19% 11211Mi 10% gke-xlou-cdm-default-pool-1b0769c8-xnmp 72m 0% 2893Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 7074m 23% 9832Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1182m 3% 5515Mi 4% gke-xlou-cdm-ds-0f601751-j488 5981m 20% 9510Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 63m 0% 2190Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 8876m 29% 10357Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 7210m 24% 5964Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 7944m 49% 7704Mi 13% 03:23:29 DEBUG --- stderr --- 03:23:29 DEBUG 03:24:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061469" 03:24:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713061469" 03:24:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061469" 03:24:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713061469" 03:24:29 INFO [http_cmd]: http status code OK 03:24:29 DEBUG --- status code --- 03:24:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:29 DEBUG --- http response --- 03:24:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:24:29 DEBUG ----- output ----- 03:24:29 DEBUG { 03:24:29 DEBUG "status": "success", 03:24:29 DEBUG "data": { 03:24:29 DEBUG "resultType": "vector", 03:24:29 DEBUG "result": [ 03:24:29 DEBUG { 03:24:29 DEBUG "metric": { 03:24:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:24:29 DEBUG }, 03:24:29 DEBUG "value": [ 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG [...] 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:24:29 DEBUG }, 03:24:29 DEBUG "value": [ 03:24:29 DEBUG 1713061469, 03:24:29 DEBUG "0.015" 03:24:29 DEBUG ] 03:24:29 DEBUG } 03:24:29 DEBUG ] 03:24:29 DEBUG } 03:24:29 DEBUG } 03:24:29 INFO [http_cmd]: http status code OK 03:24:29 DEBUG --- status code --- 03:24:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:29 DEBUG --- http response --- 03:24:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:24:29 DEBUG ----- output ----- 03:24:29 DEBUG { 03:24:29 DEBUG "status": "success", 03:24:29 DEBUG "data": { 03:24:29 DEBUG "resultType": "vector", 03:24:29 DEBUG "result": [ 03:24:29 DEBUG { 03:24:29 DEBUG "metric": { 03:24:29 DEBUG "pod": "ds-idrepo-1" 03:24:29 DEBUG }, 03:24:29 DEBUG "value": [ 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG [...] 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG "pod": "ds-idrepo-0" 03:24:29 DEBUG }, 03:24:29 DEBUG "value": [ 03:24:29 DEBUG 1713061469, 03:24:29 DEBUG "0" 03:24:29 DEBUG ] 03:24:29 DEBUG } 03:24:29 DEBUG ] 03:24:29 DEBUG } 03:24:29 DEBUG } 03:24:29 INFO [http_cmd]: http status code OK 03:24:29 DEBUG --- status code --- 03:24:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:29 DEBUG --- http response --- 03:24:29 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:24:29 INFO [http_cmd]: http status code OK 03:24:29 DEBUG --- status code --- 03:24:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:29 DEBUG --- http response --- 03:24:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:24:29 DEBUG ----- output ----- 03:24:29 DEBUG { 03:24:29 DEBUG "status": "success", 03:24:29 DEBUG "data": { 03:24:29 DEBUG "resultType": "vector", 03:24:29 DEBUG "result": [ 03:24:29 DEBUG { 03:24:29 DEBUG "metric": { 03:24:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 03:24:29 DEBUG }, 03:24:29 DEBUG "value": [ 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG [...] 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:24:29 DEBUG }, 03:24:29 DEBUG "value": [ 03:24:29 DEBUG 1713061469, 03:24:29 DEBUG "6652858368" 03:24:29 DEBUG ] 03:24:29 DEBUG } 03:24:29 DEBUG ] 03:24:29 DEBUG } 03:24:29 DEBUG } 03:24:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713061469" 03:24:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713061469" 03:24:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061469" 03:24:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061469" 03:24:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061469" 03:24:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061469" 03:24:29 ERROR [http_cmd]: ERROR 03:24:29 ERROR --- status code --- 03:24:29 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:24:29 ERROR --- http response --- 03:24:29 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:24:29 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:24:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061469" 03:24:29 INFO [http_cmd]: http status code OK 03:24:29 DEBUG --- status code --- 03:24:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:29 DEBUG --- http response --- 03:24:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:24:29 INFO [http_cmd]: http status code OK 03:24:29 DEBUG ----- output ----- 03:24:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061469" 03:24:29 DEBUG --- status code --- 03:24:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713061469" 03:24:29 DEBUG { 03:24:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:29 DEBUG "status": "success", 03:24:29 INFO [http_cmd]: http status code OK 03:24:29 DEBUG --- http response --- 03:24:29 DEBUG "data": { 03:24:29 DEBUG --- status code --- 03:24:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:24:29 DEBUG "resultType": "vector", 03:24:29 INFO [http_cmd]: http status code OK 03:24:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:29 DEBUG ----- output ----- 03:24:29 DEBUG "result": [ 03:24:29 DEBUG --- status code --- 03:24:29 DEBUG --- http response --- 03:24:29 DEBUG { 03:24:29 DEBUG { 03:24:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:29 DEBUG "status": "success", 03:24:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:24:29 DEBUG "metric": { 03:24:29 DEBUG --- http response --- 03:24:29 DEBUG "data": { 03:24:29 DEBUG ----- output ----- 03:24:29 DEBUG "pod": "ds-idrepo-1" 03:24:29 DEBUG "resultType": "vector", 03:24:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:24:29 DEBUG { 03:24:29 DEBUG }, 03:24:29 DEBUG "result": [ 03:24:29 DEBUG ----- output ----- 03:24:29 DEBUG "status": "success", 03:24:29 DEBUG "value": [ 03:24:29 DEBUG { 03:24:29 DEBUG { 03:24:29 DEBUG "data": { 03:24:29 DEBUG 03:24:29 DEBUG "metric": { 03:24:29 DEBUG "status": "success", 03:24:29 DEBUG "resultType": "vector", 03:24:29 DEBUG 03:24:29 DEBUG "pod": "ds-idrepo-1" 03:24:29 INFO [http_cmd]: http status code OK 03:24:29 DEBUG "data": { 03:24:29 DEBUG "result": [ 03:24:29 DEBUG 03:24:29 DEBUG }, 03:24:29 DEBUG --- status code --- 03:24:29 DEBUG "resultType": "vector", 03:24:29 DEBUG { 03:24:29 DEBUG [...] 03:24:29 DEBUG "value": [ 03:24:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:29 DEBUG "result": [ 03:24:29 DEBUG "metric": { 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG --- http response --- 03:24:29 DEBUG { 03:24:29 DEBUG "pod": "ds-idrepo-1" 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG "metric": { 03:24:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:24:29 DEBUG }, 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG "pod": "ds-idrepo-1" 03:24:29 DEBUG ----- output ----- 03:24:29 DEBUG "value": [ 03:24:29 DEBUG "pod": "ds-cts-1" 03:24:29 DEBUG [...] 03:24:29 DEBUG }, 03:24:29 DEBUG { 03:24:29 DEBUG 03:24:29 DEBUG }, 03:24:29 DEBUG 03:24:29 DEBUG "value": [ 03:24:29 DEBUG "status": "success", 03:24:29 DEBUG 03:24:29 DEBUG "value": [ 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG "data": { 03:24:29 DEBUG 03:24:29 DEBUG 1713061469, 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG "resultType": "vector", 03:24:29 DEBUG [...] 03:24:29 DEBUG "0" 03:24:29 DEBUG "pod": "ds-idrepo-0" 03:24:29 DEBUG 03:24:29 DEBUG "result": [ 03:24:29 DEBUG 03:24:29 DEBUG ] 03:24:29 DEBUG }, 03:24:29 DEBUG [...] 03:24:29 DEBUG { 03:24:29 DEBUG 03:24:29 DEBUG } 03:24:29 DEBUG "value": [ 03:24:29 DEBUG 03:24:29 DEBUG "metric": { 03:24:29 INFO [http_cmd]: http status code OK 03:24:29 DEBUG 03:24:29 DEBUG ] 03:24:29 DEBUG 1713061469, 03:24:29 DEBUG 03:24:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:24:29 DEBUG --- status code --- 03:24:29 DEBUG "pod": "ds-idrepo-0" 03:24:29 DEBUG } 03:24:29 DEBUG "0" 03:24:29 DEBUG 03:24:29 DEBUG }, 03:24:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:29 DEBUG }, 03:24:29 DEBUG } 03:24:29 DEBUG ] 03:24:29 DEBUG "pod": "ds-idrepo-0" 03:24:29 DEBUG "value": [ 03:24:29 DEBUG --- http response --- 03:24:29 DEBUG "value": [ 03:24:29 DEBUG } 03:24:29 DEBUG }, 03:24:29 DEBUG 03:24:29 DEBUG 1713061469, 03:24:29 DEBUG ] 03:24:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:24:29 DEBUG "value": [ 03:24:29 DEBUG 03:24:29 DEBUG "0" 03:24:29 DEBUG } 03:24:29 DEBUG ----- output ----- 03:24:29 DEBUG 1713061469, 03:24:29 DEBUG 03:24:29 DEBUG ] 03:24:29 DEBUG } 03:24:29 DEBUG { 03:24:29 DEBUG "0" 03:24:29 DEBUG [...] 03:24:29 DEBUG } 03:24:29 DEBUG "status": "success", 03:24:29 DEBUG ] 03:24:29 DEBUG 03:24:29 DEBUG ] 03:24:29 DEBUG "data": { 03:24:29 DEBUG } 03:24:29 DEBUG 03:24:29 DEBUG } 03:24:29 DEBUG "resultType": "vector", 03:24:29 DEBUG ] 03:24:29 DEBUG 03:24:29 DEBUG } 03:24:29 DEBUG "result": [ 03:24:29 DEBUG } 03:24:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:24:29 DEBUG { 03:24:29 DEBUG } 03:24:29 DEBUG }, 03:24:29 DEBUG "metric": { 03:24:29 DEBUG "value": [ 03:24:29 DEBUG "instance": "10.142.0.107:9100" 03:24:29 DEBUG 1713061469, 03:24:29 DEBUG }, 03:24:29 DEBUG "183887.5" 03:24:29 DEBUG "value": [ 03:24:29 DEBUG ] 03:24:29 DEBUG 03:24:29 DEBUG } 03:24:29 DEBUG 03:24:29 DEBUG ] 03:24:29 DEBUG 03:24:29 DEBUG } 03:24:29 INFO [http_cmd]: http status code OK 03:24:29 DEBUG [...] 03:24:29 DEBUG } 03:24:29 DEBUG --- status code --- 03:24:29 DEBUG 03:24:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:29 DEBUG 03:24:29 DEBUG --- http response --- 03:24:29 DEBUG 03:24:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:24:29 DEBUG "instance": "10.142.0.43:9100" 03:24:29 DEBUG ----- output ----- 03:24:29 DEBUG }, 03:24:29 DEBUG { 03:24:29 DEBUG "value": [ 03:24:29 DEBUG "status": "success", 03:24:29 DEBUG 1713061469, 03:24:29 DEBUG "data": { 03:24:29 DEBUG "0" 03:24:29 DEBUG "resultType": "vector", 03:24:29 DEBUG ] 03:24:29 DEBUG "result": [ 03:24:29 DEBUG } 03:24:29 DEBUG { 03:24:29 DEBUG ] 03:24:29 DEBUG "metric": { 03:24:29 DEBUG } 03:24:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 03:24:29 DEBUG } 03:24:29 DEBUG }, 03:24:29 DEBUG "value": [ 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG [...] 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:24:29 DEBUG }, 03:24:29 DEBUG "value": [ 03:24:29 DEBUG 1713061469, 03:24:29 DEBUG "8.026962903614786" 03:24:29 DEBUG ] 03:24:29 DEBUG } 03:24:29 DEBUG ] 03:24:29 DEBUG } 03:24:29 DEBUG } 03:24:29 INFO [http_cmd]: http status code OK 03:24:29 DEBUG --- status code --- 03:24:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:29 DEBUG --- http response --- 03:24:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:24:29 DEBUG ----- output ----- 03:24:29 DEBUG { 03:24:29 DEBUG "status": "success", 03:24:29 DEBUG "data": { 03:24:29 DEBUG "resultType": "vector", 03:24:29 DEBUG "result": [ 03:24:29 DEBUG { 03:24:29 DEBUG "metric": { 03:24:29 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 03:24:29 DEBUG }, 03:24:29 DEBUG "value": [ 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG [...] 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 03:24:29 DEBUG }, 03:24:29 DEBUG "value": [ 03:24:29 DEBUG 1713061469, 03:24:29 DEBUG "0" 03:24:29 DEBUG ] 03:24:29 DEBUG } 03:24:29 DEBUG ] 03:24:29 DEBUG } 03:24:29 DEBUG } 03:24:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061469" 03:24:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061469" 03:24:29 INFO [http_cmd]: http status code OK 03:24:29 DEBUG --- status code --- 03:24:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:29 DEBUG --- http response --- 03:24:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:24:29 DEBUG ----- output ----- 03:24:29 INFO [http_cmd]: http status code OK 03:24:29 DEBUG { 03:24:29 DEBUG --- status code --- 03:24:29 DEBUG "status": "success", 03:24:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:29 DEBUG "data": { 03:24:29 DEBUG --- http response --- 03:24:29 DEBUG "resultType": "vector", 03:24:29 DEBUG "result": [ 03:24:29 DEBUG { 03:24:29 DEBUG "metric": { 03:24:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:24:29 DEBUG }, 03:24:29 DEBUG "value": [ 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG [...] 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:24:29 DEBUG }, 03:24:29 DEBUG "value": [ 03:24:29 DEBUG 1713061469, 03:24:29 DEBUG "0" 03:24:29 DEBUG ] 03:24:29 DEBUG } 03:24:29 DEBUG ] 03:24:29 DEBUG } 03:24:29 DEBUG } 03:24:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:24:29 DEBUG ----- output ----- 03:24:29 DEBUG { 03:24:29 DEBUG "status": "success", 03:24:29 DEBUG "data": { 03:24:29 DEBUG "resultType": "vector", 03:24:29 DEBUG "result": [ 03:24:29 DEBUG { 03:24:29 DEBUG "metric": { 03:24:29 DEBUG "instance": "10.142.0.107:9100" 03:24:29 DEBUG }, 03:24:29 DEBUG "value": [ 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG [...] 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG "instance": "10.142.0.43:9100" 03:24:29 DEBUG }, 03:24:29 DEBUG "value": [ 03:24:29 DEBUG 1713061469, 03:24:29 DEBUG "5615.400000000001" 03:24:29 DEBUG ] 03:24:29 DEBUG } 03:24:29 DEBUG ] 03:24:29 DEBUG } 03:24:29 DEBUG } 03:24:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061469" 03:24:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061469" 03:24:29 INFO [http_cmd]: http status code OK 03:24:29 DEBUG --- status code --- 03:24:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:29 DEBUG --- http response --- 03:24:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:24:29 DEBUG ----- output ----- 03:24:29 DEBUG { 03:24:29 DEBUG "status": "success", 03:24:29 DEBUG "data": { 03:24:29 DEBUG "resultType": "vector", 03:24:29 DEBUG "result": [ 03:24:29 DEBUG { 03:24:29 DEBUG "metric": { 03:24:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:24:29 DEBUG }, 03:24:29 DEBUG "value": [ 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG [...] 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:24:29 DEBUG }, 03:24:29 DEBUG "value": [ 03:24:29 DEBUG 1713061469, 03:24:29 DEBUG "1146.3333333333333" 03:24:29 DEBUG ] 03:24:29 DEBUG } 03:24:29 DEBUG ] 03:24:29 DEBUG } 03:24:29 DEBUG } 03:24:29 INFO [http_cmd]: http status code OK 03:24:29 DEBUG --- status code --- 03:24:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:29 DEBUG --- http response --- 03:24:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:24:29 DEBUG ----- output ----- 03:24:29 DEBUG { 03:24:29 DEBUG "status": "success", 03:24:29 DEBUG "data": { 03:24:29 DEBUG "resultType": "vector", 03:24:29 DEBUG "result": [ 03:24:29 DEBUG { 03:24:29 DEBUG "metric": { 03:24:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:24:29 DEBUG }, 03:24:29 DEBUG "value": [ 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG [...] 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:24:29 DEBUG }, 03:24:29 DEBUG "value": [ 03:24:29 DEBUG 1713061469, 03:24:29 DEBUG "1330.8333333333333" 03:24:29 DEBUG ] 03:24:29 DEBUG } 03:24:29 DEBUG ] 03:24:29 DEBUG } 03:24:29 DEBUG } 03:24:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061469" 03:24:29 INFO [http_cmd]: http status code OK 03:24:29 DEBUG --- status code --- 03:24:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:29 DEBUG --- http response --- 03:24:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:24:29 DEBUG ----- output ----- 03:24:29 DEBUG { 03:24:29 DEBUG "status": "success", 03:24:29 DEBUG "data": { 03:24:29 DEBUG "resultType": "vector", 03:24:29 DEBUG "result": [ 03:24:29 DEBUG { 03:24:29 DEBUG "metric": { 03:24:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:24:29 DEBUG }, 03:24:29 DEBUG "value": [ 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG [...] 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:24:29 DEBUG }, 03:24:29 DEBUG "value": [ 03:24:29 DEBUG 1713061469, 03:24:29 DEBUG "1330.8333333333333" 03:24:29 DEBUG ] 03:24:29 DEBUG } 03:24:29 DEBUG ] 03:24:29 DEBUG } 03:24:29 DEBUG } 03:24:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061469" 03:24:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061469" 03:24:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061469" 03:24:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061469" 03:24:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061469" 03:24:29 INFO [http_cmd]: http status code OK 03:24:29 DEBUG --- status code --- 03:24:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:29 DEBUG --- http response --- 03:24:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:24:29 INFO [http_cmd]: http status code OK 03:24:29 DEBUG ----- output ----- 03:24:29 DEBUG --- status code --- 03:24:29 DEBUG { 03:24:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:29 DEBUG "status": "success", 03:24:29 DEBUG --- http response --- 03:24:29 DEBUG "data": { 03:24:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:24:29 DEBUG "resultType": "vector", 03:24:29 DEBUG ----- output ----- 03:24:29 DEBUG "result": [ 03:24:29 DEBUG { 03:24:29 DEBUG { 03:24:29 DEBUG "status": "success", 03:24:29 DEBUG "metric": { 03:24:29 DEBUG "data": { 03:24:29 DEBUG "pod": "ds-idrepo-1" 03:24:29 DEBUG "resultType": "vector", 03:24:29 DEBUG }, 03:24:29 DEBUG "result": [ 03:24:29 DEBUG "value": [ 03:24:29 DEBUG { 03:24:29 DEBUG 03:24:29 DEBUG "metric": { 03:24:29 DEBUG 03:24:29 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 03:24:29 DEBUG 03:24:29 DEBUG }, 03:24:29 DEBUG [...] 03:24:29 DEBUG "value": [ 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG "pod": "ds-idrepo-0" 03:24:29 DEBUG [...] 03:24:29 DEBUG }, 03:24:29 INFO [http_cmd]: http status code OK 03:24:29 DEBUG 03:24:29 DEBUG "value": [ 03:24:29 DEBUG --- status code --- 03:24:29 DEBUG 03:24:29 DEBUG 1713061469, 03:24:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:29 DEBUG 03:24:29 DEBUG "0" 03:24:29 DEBUG --- http response --- 03:24:29 DEBUG "pod": "end-user-ui-58878d7668-zg4h2" 03:24:29 DEBUG ] 03:24:29 DEBUG }, 03:24:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:24:29 DEBUG } 03:24:29 DEBUG "value": [ 03:24:29 DEBUG ----- output ----- 03:24:29 DEBUG ] 03:24:29 DEBUG 1713061469, 03:24:29 DEBUG { 03:24:29 DEBUG } 03:24:29 DEBUG "0" 03:24:29 DEBUG "status": "success", 03:24:29 DEBUG } 03:24:29 DEBUG ] 03:24:29 DEBUG "data": { 03:24:29 DEBUG } 03:24:29 DEBUG "resultType": "vector", 03:24:29 DEBUG ] 03:24:29 DEBUG "result": [ 03:24:29 DEBUG } 03:24:29 DEBUG { 03:24:29 DEBUG } 03:24:29 DEBUG "metric": { 03:24:29 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 03:24:29 DEBUG }, 03:24:29 DEBUG "value": [ 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG [...] 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 03:24:29 DEBUG }, 03:24:29 DEBUG "value": [ 03:24:29 DEBUG 1713061469, 03:24:29 DEBUG "0" 03:24:29 DEBUG ] 03:24:29 DEBUG } 03:24:29 DEBUG ] 03:24:29 DEBUG } 03:24:29 DEBUG } 03:24:29 INFO [http_cmd]: http status code OK 03:24:29 DEBUG --- status code --- 03:24:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:29 DEBUG --- http response --- 03:24:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:24:29 DEBUG ----- output ----- 03:24:29 DEBUG { 03:24:29 DEBUG "status": "success", 03:24:29 DEBUG "data": { 03:24:29 DEBUG "resultType": "vector", 03:24:29 DEBUG "result": [ 03:24:29 DEBUG { 03:24:29 DEBUG "metric": { 03:24:29 DEBUG "instance": "10.142.0.107:9100" 03:24:29 DEBUG }, 03:24:29 DEBUG "value": [ 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG [...] 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG "instance": "10.142.0.43:9100" 03:24:29 DEBUG }, 03:24:29 DEBUG "value": [ 03:24:29 DEBUG 1713061469, 03:24:29 DEBUG "0.00020000000000000757" 03:24:29 DEBUG ] 03:24:29 DEBUG } 03:24:29 DEBUG ] 03:24:29 DEBUG } 03:24:29 DEBUG } 03:24:29 INFO [http_cmd]: http status code OK 03:24:29 DEBUG --- status code --- 03:24:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:29 DEBUG --- http response --- 03:24:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 03:24:29 DEBUG ----- output ----- 03:24:29 DEBUG { 03:24:29 DEBUG "status": "success", 03:24:29 DEBUG "data": { 03:24:29 DEBUG "resultType": "vector", 03:24:29 DEBUG "result": [ 03:24:29 DEBUG { 03:24:29 DEBUG "metric": { 03:24:29 DEBUG "pod": "ds-idrepo-1" 03:24:29 DEBUG }, 03:24:29 DEBUG "value": [ 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG [...] 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG "pod": "ds-cts-1" 03:24:29 DEBUG }, 03:24:29 DEBUG "value": [ 03:24:29 DEBUG 1713061469, 03:24:29 DEBUG "0.00013333333333333334" 03:24:29 DEBUG ] 03:24:29 DEBUG } 03:24:29 DEBUG ] 03:24:29 DEBUG } 03:24:29 DEBUG } 03:24:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061469" 03:24:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061469" 03:24:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061469" 03:24:29 INFO [http_cmd]: http status code OK 03:24:29 DEBUG --- status code --- 03:24:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:29 DEBUG --- http response --- 03:24:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:24:29 DEBUG ----- output ----- 03:24:29 DEBUG { 03:24:29 DEBUG "status": "success", 03:24:29 DEBUG "data": { 03:24:29 DEBUG "resultType": "vector", 03:24:29 DEBUG "result": [ 03:24:29 DEBUG { 03:24:29 DEBUG "metric": { 03:24:29 DEBUG "instance": "10.142.0.107:9100" 03:24:29 DEBUG }, 03:24:29 DEBUG "value": [ 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG [...] 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG "instance": "10.142.0.43:9100" 03:24:29 DEBUG }, 03:24:29 DEBUG "value": [ 03:24:29 DEBUG 1713061469, 03:24:29 DEBUG "12653728768" 03:24:29 DEBUG ] 03:24:29 DEBUG } 03:24:29 DEBUG ] 03:24:29 DEBUG } 03:24:29 DEBUG } 03:24:29 INFO [http_cmd]: http status code OK 03:24:29 DEBUG --- status code --- 03:24:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:29 DEBUG --- http response --- 03:24:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:24:29 DEBUG ----- output ----- 03:24:29 DEBUG { 03:24:29 DEBUG "status": "success", 03:24:29 DEBUG "data": { 03:24:29 DEBUG "resultType": "vector", 03:24:29 DEBUG "result": [ 03:24:29 DEBUG { 03:24:29 DEBUG "metric": { 03:24:29 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 03:24:29 DEBUG }, 03:24:29 DEBUG "value": [ 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG [...] 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG "pod": "end-user-ui-58878d7668-zg4h2" 03:24:29 DEBUG }, 03:24:29 DEBUG "value": [ 03:24:29 DEBUG 1713061469, 03:24:29 DEBUG "0" 03:24:29 DEBUG ] 03:24:29 DEBUG } 03:24:29 DEBUG ] 03:24:29 DEBUG } 03:24:29 DEBUG } 03:24:29 INFO [http_cmd]: http status code OK 03:24:29 DEBUG --- status code --- 03:24:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:29 DEBUG --- http response --- 03:24:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:24:29 DEBUG ----- output ----- 03:24:29 DEBUG { 03:24:29 DEBUG "status": "success", 03:24:29 DEBUG "data": { 03:24:29 DEBUG "resultType": "vector", 03:24:29 DEBUG "result": [ 03:24:29 DEBUG { 03:24:29 DEBUG "metric": { 03:24:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:24:29 DEBUG }, 03:24:29 DEBUG "value": [ 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG [...] 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:24:29 DEBUG }, 03:24:29 DEBUG "value": [ 03:24:29 DEBUG 1713061469, 03:24:29 DEBUG "1328.0666666666666" 03:24:29 DEBUG ] 03:24:29 DEBUG } 03:24:29 DEBUG ] 03:24:29 DEBUG } 03:24:29 DEBUG } 03:24:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061469" 03:24:29 INFO [http_cmd]: http status code OK 03:24:29 DEBUG --- status code --- 03:24:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:29 DEBUG --- http response --- 03:24:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:24:29 DEBUG ----- output ----- 03:24:29 DEBUG { 03:24:29 DEBUG "status": "success", 03:24:29 DEBUG "data": { 03:24:29 DEBUG "resultType": "vector", 03:24:29 DEBUG "result": [ 03:24:29 DEBUG { 03:24:29 DEBUG "metric": { 03:24:29 DEBUG "instance": "10.142.0.107:9100" 03:24:29 DEBUG }, 03:24:29 DEBUG "value": [ 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG [...] 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG 03:24:29 DEBUG "instance": "10.142.0.43:9100" 03:24:29 DEBUG }, 03:24:29 DEBUG "value": [ 03:24:29 DEBUG 1713061469, 03:24:29 DEBUG "4347.433333333333" 03:24:29 DEBUG ] 03:24:29 DEBUG } 03:24:29 DEBUG ] 03:24:29 DEBUG } 03:24:29 DEBUG } 03:24:29 INFO 03:24:29 INFO [loop_until]: kubectl --namespace=xlou top pods 03:24:29 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:24:29 INFO [loop_until]: OK (rc = 0) 03:24:29 DEBUG --- stdout --- 03:24:29 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 6897m 8939Mi am-7d597cf7d6-8vr8f 6521m 8519Mi am-7d597cf7d6-9wz9h 8261m 8813Mi ds-cts-0 5318m 7841Mi ds-cts-1 5045m 8033Mi ds-cts-2 4963m 8119Mi ds-idrepo-0 7318m 4513Mi ds-idrepo-1 1141m 4286Mi ds-idrepo-2 1129m 4104Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 990Mi idm-b85f9d978-rl76f 4m 984Mi lodemon-766dc9b5df-2jmd4 1m 63Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 7963m 6774Mi 03:24:29 DEBUG --- stderr --- 03:24:29 DEBUG 03:24:29 INFO 03:24:29 INFO [loop_until]: kubectl --namespace=xlou top node 03:24:29 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:24:30 INFO [loop_until]: OK (rc = 0) 03:24:30 DEBUG --- stdout --- 03:24:30 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 4823Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5120m 17% 10280Mi 9% gke-xlou-cdm-default-pool-1b0769c8-nhlf 7258m 24% 10990Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 11237m 37% 3582Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 1232m 4% 6350Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 4807m 16% 11241Mi 10% gke-xlou-cdm-default-pool-1b0769c8-xnmp 64m 0% 2903Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 6915m 23% 9952Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1173m 3% 5510Mi 4% gke-xlou-cdm-ds-0f601751-j488 5284m 17% 9513Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 64m 0% 2184Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 8815m 29% 10353Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 7410m 24% 5955Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 8278m 52% 10130Mi 17% 03:24:30 DEBUG --- stderr --- 03:24:30 DEBUG 03:25:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713061529" 03:25:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713061529" 03:25:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061529" 03:25:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061529" 03:25:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061529" 03:25:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061529" 03:25:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713061529" 03:25:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061529" 03:25:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713061529" 03:25:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713061529" 03:25:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061529" 03:25:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061529" 03:25:29 INFO [http_cmd]: http status code OK 03:25:29 INFO [http_cmd]: http status code OK 03:25:29 DEBUG --- status code --- 03:25:29 INFO [http_cmd]: http status code OK 03:25:29 DEBUG --- status code --- 03:25:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:29 INFO [http_cmd]: http status code OK 03:25:29 DEBUG --- status code --- 03:25:29 ERROR [http_cmd]: ERROR 03:25:29 INFO [http_cmd]: http status code OK 03:25:29 INFO [http_cmd]: http status code OK 03:25:29 INFO [http_cmd]: http status code OK 03:25:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:29 INFO [http_cmd]: http status code OK 03:25:29 DEBUG --- http response --- 03:25:29 DEBUG --- status code --- 03:25:29 INFO [http_cmd]: http status code OK 03:25:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:29 ERROR --- status code --- 03:25:29 DEBUG --- status code --- 03:25:29 DEBUG --- status code --- 03:25:29 DEBUG --- status code --- 03:25:29 DEBUG --- http response --- 03:25:29 DEBUG --- status code --- 03:25:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:25:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:29 DEBUG --- status code --- 03:25:29 DEBUG --- http response --- 03:25:29 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:25:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:25:29 DEBUG ----- output ----- 03:25:29 DEBUG --- http response --- 03:25:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:25:29 ERROR --- http response --- 03:25:29 DEBUG --- http response --- 03:25:29 DEBUG --- http response --- 03:25:29 DEBUG --- http response --- 03:25:29 DEBUG --- http response --- 03:25:29 DEBUG ----- output ----- 03:25:29 DEBUG { 03:25:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:25:29 DEBUG --- http response --- 03:25:29 DEBUG ----- output ----- 03:25:29 INFO [http_cmd]: http status code OK 03:25:29 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:25:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:25:29 INFO [http_cmd]: http status code OK 03:25:29 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:25:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:25:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:25:29 DEBUG { 03:25:29 DEBUG "status": "success", 03:25:29 DEBUG ----- output ----- 03:25:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:25:29 DEBUG { 03:25:29 DEBUG --- status code --- 03:25:29 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:25:29 DEBUG ----- output ----- 03:25:29 DEBUG --- status code --- 03:25:29 DEBUG ----- output ----- 03:25:29 DEBUG ----- output ----- 03:25:29 DEBUG "status": "success", 03:25:29 DEBUG "data": { 03:25:29 DEBUG { 03:25:29 DEBUG ----- output ----- 03:25:29 DEBUG "status": "success", 03:25:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:29 DEBUG { 03:25:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:29 DEBUG { 03:25:29 DEBUG { 03:25:29 DEBUG "data": { 03:25:29 DEBUG "resultType": "vector", 03:25:29 DEBUG "status": "success", 03:25:29 DEBUG { 03:25:29 DEBUG "data": { 03:25:29 DEBUG --- http response --- 03:25:29 DEBUG "status": "success", 03:25:29 DEBUG --- http response --- 03:25:29 DEBUG "status": "success", 03:25:29 DEBUG "status": "success", 03:25:29 DEBUG "resultType": "vector", 03:25:29 DEBUG "result": [ 03:25:29 DEBUG "data": { 03:25:29 DEBUG "status": "success", 03:25:29 DEBUG "resultType": "vector", 03:25:29 DEBUG "data": { 03:25:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:25:29 DEBUG "data": { 03:25:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:25:29 DEBUG "data": { 03:25:29 DEBUG "result": [ 03:25:29 DEBUG { 03:25:29 DEBUG "resultType": "vector", 03:25:29 DEBUG "data": { 03:25:29 DEBUG "result": [ 03:25:29 DEBUG "resultType": "vector", 03:25:29 DEBUG ----- output ----- 03:25:29 DEBUG "resultType": "vector", 03:25:29 DEBUG ----- output ----- 03:25:29 DEBUG "resultType": "vector", 03:25:29 DEBUG { 03:25:29 DEBUG "metric": { 03:25:29 DEBUG "result": [ 03:25:29 DEBUG "resultType": "vector", 03:25:29 DEBUG { 03:25:29 DEBUG "result": [ 03:25:29 DEBUG { 03:25:29 DEBUG "result": [ 03:25:29 DEBUG { 03:25:29 DEBUG "result": [ 03:25:29 DEBUG "metric": { 03:25:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:25:29 DEBUG { 03:25:29 DEBUG "result": [ 03:25:29 DEBUG "metric": { 03:25:29 DEBUG { 03:25:29 DEBUG "status": "success", 03:25:29 DEBUG { 03:25:29 DEBUG "status": "success", 03:25:29 DEBUG { 03:25:29 DEBUG "pod": "ds-idrepo-1" 03:25:29 DEBUG }, 03:25:29 DEBUG "metric": { 03:25:29 DEBUG { 03:25:29 DEBUG "pod": "ds-idrepo-1" 03:25:29 DEBUG "metric": { 03:25:29 DEBUG "data": { 03:25:29 DEBUG "metric": { 03:25:29 DEBUG "data": { 03:25:29 DEBUG "metric": { 03:25:29 DEBUG }, 03:25:29 DEBUG "value": [ 03:25:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 03:25:29 DEBUG "metric": { 03:25:29 DEBUG }, 03:25:29 DEBUG "pod": "ds-idrepo-1" 03:25:29 DEBUG "resultType": "vector", 03:25:29 DEBUG "pod": "ds-idrepo-1" 03:25:29 DEBUG "resultType": "vector", 03:25:29 DEBUG "instance": "10.142.0.107:9100" 03:25:29 DEBUG "value": [ 03:25:29 DEBUG 03:25:29 DEBUG }, 03:25:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 03:25:29 DEBUG "value": [ 03:25:29 DEBUG }, 03:25:29 DEBUG "result": [ 03:25:29 DEBUG }, 03:25:29 DEBUG "result": [ 03:25:29 DEBUG }, 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG "value": [ 03:25:29 DEBUG }, 03:25:29 DEBUG 03:25:29 DEBUG "value": [ 03:25:29 DEBUG { 03:25:29 DEBUG "value": [ 03:25:29 DEBUG { 03:25:29 DEBUG "value": [ 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG "value": [ 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG "metric": { 03:25:29 DEBUG 03:25:29 DEBUG "metric": { 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG [...] 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG "pod": "ds-idrepo-1" 03:25:29 DEBUG 03:25:29 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 03:25:29 DEBUG 03:25:29 DEBUG [...] 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG [...] 03:25:29 DEBUG 03:25:29 DEBUG }, 03:25:29 DEBUG 03:25:29 DEBUG }, 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG [...] 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG [...] 03:25:29 DEBUG "value": [ 03:25:29 DEBUG [...] 03:25:29 DEBUG "value": [ 03:25:29 DEBUG [...] 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG [...] 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG "pod": "ds-cts-1" 03:25:29 DEBUG }, 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG "pod": "ds-idrepo-0" 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG }, 03:25:29 DEBUG "value": [ 03:25:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:25:29 DEBUG 03:25:29 DEBUG }, 03:25:29 DEBUG "pod": "ds-idrepo-0" 03:25:29 DEBUG [...] 03:25:29 DEBUG "pod": "ds-idrepo-0" 03:25:29 DEBUG [...] 03:25:29 DEBUG "instance": "10.142.0.43:9100" 03:25:29 DEBUG "value": [ 03:25:29 DEBUG 1713061529, 03:25:29 DEBUG }, 03:25:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:25:29 DEBUG "value": [ 03:25:29 DEBUG }, 03:25:29 DEBUG 03:25:29 DEBUG }, 03:25:29 DEBUG 03:25:29 DEBUG }, 03:25:29 DEBUG 1713061529, 03:25:29 DEBUG "0.015" 03:25:29 DEBUG "value": [ 03:25:29 DEBUG }, 03:25:29 DEBUG 1713061529, 03:25:29 DEBUG "value": [ 03:25:29 DEBUG 03:25:29 DEBUG "value": [ 03:25:29 DEBUG 03:25:29 DEBUG "value": [ 03:25:29 DEBUG "0" 03:25:29 DEBUG ] 03:25:29 DEBUG 1713061529, 03:25:29 DEBUG "value": [ 03:25:29 DEBUG "0" 03:25:29 DEBUG 1713061529, 03:25:29 DEBUG 03:25:29 DEBUG 1713061529, 03:25:29 DEBUG 03:25:29 DEBUG 1713061529, 03:25:29 DEBUG ] 03:25:29 DEBUG } 03:25:29 DEBUG "8.024072736989332" 03:25:29 DEBUG 1713061529, 03:25:29 DEBUG ] 03:25:29 DEBUG "0" 03:25:29 DEBUG "pod": "ds-idrepo-0" 03:25:29 DEBUG "0" 03:25:29 DEBUG "pod": "idm-b85f9d978-j5pbm" 03:25:29 DEBUG "0" 03:25:29 DEBUG } 03:25:29 DEBUG ] 03:25:29 DEBUG ] 03:25:29 DEBUG "7950667776" 03:25:29 DEBUG } 03:25:29 DEBUG ] 03:25:29 DEBUG }, 03:25:29 DEBUG ] 03:25:29 DEBUG }, 03:25:29 DEBUG ] 03:25:29 DEBUG ] 03:25:29 DEBUG } 03:25:29 DEBUG } 03:25:29 DEBUG ] 03:25:29 DEBUG ] 03:25:29 DEBUG } 03:25:29 DEBUG "value": [ 03:25:29 DEBUG } 03:25:29 DEBUG "value": [ 03:25:29 DEBUG } 03:25:29 DEBUG } 03:25:29 DEBUG } 03:25:29 DEBUG ] 03:25:29 DEBUG } 03:25:29 DEBUG } 03:25:29 DEBUG ] 03:25:29 DEBUG 1713061529, 03:25:29 DEBUG ] 03:25:29 DEBUG 1713061529, 03:25:29 DEBUG ] 03:25:29 DEBUG } 03:25:29 DEBUG } 03:25:29 DEBUG ] 03:25:29 DEBUG } 03:25:29 DEBUG } 03:25:29 DEBUG "0" 03:25:29 DEBUG } 03:25:29 DEBUG "3821.7273660311635" 03:25:29 DEBUG } 03:25:29 DEBUG } 03:25:29 DEBUG } 03:25:29 DEBUG } 03:25:29 DEBUG ] 03:25:29 DEBUG } 03:25:29 DEBUG ] 03:25:29 DEBUG } 03:25:29 DEBUG } 03:25:29 DEBUG } 03:25:29 DEBUG } 03:25:29 DEBUG ] 03:25:29 DEBUG ] 03:25:29 DEBUG } 03:25:29 DEBUG } 03:25:29 DEBUG } 03:25:29 DEBUG } 03:25:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061529" 03:25:29 INFO [http_cmd]: http status code OK 03:25:29 DEBUG --- status code --- 03:25:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:29 DEBUG --- http response --- 03:25:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:25:29 DEBUG ----- output ----- 03:25:29 DEBUG { 03:25:29 DEBUG "status": "success", 03:25:29 DEBUG "data": { 03:25:29 DEBUG "resultType": "vector", 03:25:29 DEBUG "result": [ 03:25:29 DEBUG { 03:25:29 DEBUG "metric": { 03:25:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:25:29 DEBUG }, 03:25:29 DEBUG "value": [ 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG [...] 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:25:29 DEBUG }, 03:25:29 DEBUG "value": [ 03:25:29 DEBUG 1713061529, 03:25:29 DEBUG "238069.5" 03:25:29 DEBUG ] 03:25:29 DEBUG } 03:25:29 DEBUG ] 03:25:29 DEBUG } 03:25:29 DEBUG } 03:25:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061529" 03:25:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061529" 03:25:29 INFO [http_cmd]: http status code OK 03:25:29 INFO [http_cmd]: http status code OK 03:25:29 DEBUG --- status code --- 03:25:29 DEBUG --- status code --- 03:25:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:29 DEBUG --- http response --- 03:25:29 DEBUG --- http response --- 03:25:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:25:29 DEBUG ----- output ----- 03:25:29 DEBUG { 03:25:29 DEBUG "status": "success", 03:25:29 DEBUG "data": { 03:25:29 DEBUG "resultType": "vector", 03:25:29 DEBUG "result": [ 03:25:29 DEBUG { 03:25:29 DEBUG "metric": { 03:25:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:25:29 DEBUG }, 03:25:29 DEBUG "value": [ 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG [...] 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:25:29 DEBUG }, 03:25:29 DEBUG "value": [ 03:25:29 DEBUG 1713061529, 03:25:29 DEBUG "0" 03:25:29 DEBUG ] 03:25:29 DEBUG } 03:25:29 DEBUG ] 03:25:29 DEBUG } 03:25:29 DEBUG } 03:25:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:25:29 DEBUG ----- output ----- 03:25:29 DEBUG { 03:25:29 DEBUG "status": "success", 03:25:29 DEBUG "data": { 03:25:29 DEBUG "resultType": "vector", 03:25:29 DEBUG "result": [ 03:25:29 DEBUG { 03:25:29 DEBUG "metric": { 03:25:29 DEBUG "instance": "10.142.0.107:9100" 03:25:29 DEBUG }, 03:25:29 DEBUG "value": [ 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG [...] 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG "instance": "10.142.0.43:9100" 03:25:29 DEBUG }, 03:25:29 DEBUG "value": [ 03:25:29 DEBUG 1713061529, 03:25:29 DEBUG "5458.766666666667" 03:25:29 DEBUG ] 03:25:29 DEBUG } 03:25:29 DEBUG ] 03:25:29 DEBUG } 03:25:29 DEBUG } 03:25:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061529" 03:25:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061529" 03:25:29 INFO [http_cmd]: http status code OK 03:25:29 DEBUG --- status code --- 03:25:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:29 DEBUG --- http response --- 03:25:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:25:29 DEBUG ----- output ----- 03:25:29 DEBUG { 03:25:29 DEBUG "status": "success", 03:25:29 DEBUG "data": { 03:25:29 DEBUG "resultType": "vector", 03:25:29 DEBUG "result": [ 03:25:29 DEBUG { 03:25:29 DEBUG "metric": { 03:25:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:25:29 DEBUG }, 03:25:29 DEBUG "value": [ 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG [...] 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:25:29 DEBUG }, 03:25:29 DEBUG "value": [ 03:25:29 DEBUG 1713061529, 03:25:29 DEBUG "358.23333333333335" 03:25:29 DEBUG ] 03:25:29 DEBUG } 03:25:29 DEBUG ] 03:25:29 DEBUG } 03:25:29 DEBUG } 03:25:29 INFO [http_cmd]: http status code OK 03:25:29 DEBUG --- status code --- 03:25:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:29 DEBUG --- http response --- 03:25:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:25:29 DEBUG ----- output ----- 03:25:29 DEBUG { 03:25:29 DEBUG "status": "success", 03:25:29 DEBUG "data": { 03:25:29 DEBUG "resultType": "vector", 03:25:29 DEBUG "result": [ 03:25:29 DEBUG { 03:25:29 DEBUG "metric": { 03:25:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:25:29 DEBUG }, 03:25:29 DEBUG "value": [ 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG [...] 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:25:29 DEBUG }, 03:25:29 DEBUG "value": [ 03:25:29 DEBUG 1713061529, 03:25:29 DEBUG "1259.8666666666666" 03:25:29 DEBUG ] 03:25:29 DEBUG } 03:25:29 DEBUG ] 03:25:29 DEBUG } 03:25:29 DEBUG } 03:25:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061529" 03:25:29 INFO [http_cmd]: http status code OK 03:25:29 DEBUG --- status code --- 03:25:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:29 DEBUG --- http response --- 03:25:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:25:29 DEBUG ----- output ----- 03:25:29 DEBUG { 03:25:29 DEBUG "status": "success", 03:25:29 DEBUG "data": { 03:25:29 DEBUG "resultType": "vector", 03:25:29 DEBUG "result": [ 03:25:29 DEBUG { 03:25:29 DEBUG "metric": { 03:25:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:25:29 DEBUG }, 03:25:29 DEBUG "value": [ 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG [...] 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:25:29 DEBUG }, 03:25:29 DEBUG "value": [ 03:25:29 DEBUG 1713061529, 03:25:29 DEBUG "1259.8" 03:25:29 DEBUG ] 03:25:29 DEBUG } 03:25:29 DEBUG ] 03:25:29 DEBUG } 03:25:29 DEBUG } 03:25:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061529" 03:25:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061529" 03:25:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061529" 03:25:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061529" 03:25:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061529" 03:25:29 INFO [http_cmd]: http status code OK 03:25:29 INFO [http_cmd]: http status code OK 03:25:29 DEBUG --- status code --- 03:25:29 DEBUG --- status code --- 03:25:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:29 DEBUG --- http response --- 03:25:29 DEBUG --- http response --- 03:25:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:25:29 DEBUG ----- output ----- 03:25:29 DEBUG { 03:25:29 DEBUG "status": "success", 03:25:29 DEBUG "data": { 03:25:29 DEBUG "resultType": "vector", 03:25:29 DEBUG "result": [ 03:25:29 DEBUG { 03:25:29 DEBUG "metric": { 03:25:29 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 03:25:29 DEBUG }, 03:25:29 DEBUG "value": [ 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG [...] 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:25:29 DEBUG "pod": "idm-b85f9d978-j5pbm" 03:25:29 DEBUG ----- output ----- 03:25:29 DEBUG }, 03:25:29 DEBUG { 03:25:29 DEBUG "value": [ 03:25:29 DEBUG "status": "success", 03:25:29 INFO [http_cmd]: http status code OK 03:25:29 DEBUG 1713061529, 03:25:29 DEBUG "data": { 03:25:29 DEBUG --- status code --- 03:25:29 DEBUG "0" 03:25:29 DEBUG "resultType": "vector", 03:25:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:29 DEBUG ] 03:25:29 DEBUG "result": [ 03:25:29 DEBUG --- http response --- 03:25:29 DEBUG } 03:25:29 DEBUG { 03:25:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:25:29 DEBUG ] 03:25:29 DEBUG "metric": { 03:25:29 DEBUG ----- output ----- 03:25:29 DEBUG } 03:25:29 DEBUG "pod": "ds-idrepo-1" 03:25:29 DEBUG { 03:25:29 DEBUG } 03:25:29 DEBUG }, 03:25:29 DEBUG "status": "success", 03:25:29 DEBUG "value": [ 03:25:29 DEBUG "data": { 03:25:29 DEBUG 03:25:29 INFO [http_cmd]: http status code OK 03:25:29 DEBUG "resultType": "vector", 03:25:29 DEBUG 03:25:29 DEBUG --- status code --- 03:25:29 DEBUG "result": [ 03:25:29 DEBUG 03:25:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:29 DEBUG { 03:25:29 DEBUG [...] 03:25:29 DEBUG --- http response --- 03:25:29 INFO [http_cmd]: http status code OK 03:25:29 DEBUG "metric": { 03:25:29 DEBUG 03:25:29 DEBUG --- status code --- 03:25:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 03:25:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061529" 03:25:29 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 03:25:29 DEBUG 03:25:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:29 DEBUG ----- output ----- 03:25:29 DEBUG }, 03:25:29 DEBUG 03:25:29 DEBUG --- http response --- 03:25:29 DEBUG { 03:25:29 DEBUG "value": [ 03:25:29 DEBUG "pod": "ds-idrepo-0" 03:25:29 DEBUG "status": "success", 03:25:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:25:29 DEBUG 03:25:29 DEBUG }, 03:25:29 DEBUG "data": { 03:25:29 DEBUG ----- output ----- 03:25:29 DEBUG 03:25:29 DEBUG "value": [ 03:25:29 DEBUG "resultType": "vector", 03:25:29 DEBUG { 03:25:29 DEBUG 03:25:29 DEBUG 1713061529, 03:25:29 DEBUG "result": [ 03:25:29 DEBUG "status": "success", 03:25:29 DEBUG [...] 03:25:29 DEBUG "0" 03:25:29 DEBUG { 03:25:29 DEBUG "data": { 03:25:29 DEBUG 03:25:29 DEBUG ] 03:25:29 DEBUG "metric": { 03:25:29 DEBUG "resultType": "vector", 03:25:29 DEBUG 03:25:29 DEBUG } 03:25:29 DEBUG "pod": "ds-idrepo-1" 03:25:29 DEBUG "result": [ 03:25:29 DEBUG 03:25:29 DEBUG ] 03:25:29 DEBUG }, 03:25:29 DEBUG { 03:25:29 DEBUG "pod": "idm-b85f9d978-j5pbm" 03:25:29 DEBUG } 03:25:29 DEBUG "value": [ 03:25:29 DEBUG "metric": { 03:25:29 DEBUG }, 03:25:29 DEBUG } 03:25:29 DEBUG 03:25:29 DEBUG "instance": "10.142.0.107:9100" 03:25:29 DEBUG "value": [ 03:25:29 DEBUG 03:25:29 DEBUG }, 03:25:29 DEBUG 1713061529, 03:25:29 DEBUG 03:25:29 DEBUG "value": [ 03:25:29 DEBUG "0.00487519049544402" 03:25:29 DEBUG [...] 03:25:29 DEBUG 03:25:29 DEBUG ] 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG } 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG ] 03:25:29 DEBUG 03:25:29 DEBUG [...] 03:25:29 DEBUG } 03:25:29 DEBUG "pod": "ds-cts-1" 03:25:29 DEBUG 03:25:29 DEBUG } 03:25:29 DEBUG }, 03:25:29 DEBUG 03:25:29 DEBUG "value": [ 03:25:29 DEBUG 03:25:29 DEBUG 1713061529, 03:25:29 DEBUG "instance": "10.142.0.43:9100" 03:25:29 DEBUG "0.00012086111111111111" 03:25:29 DEBUG }, 03:25:29 DEBUG ] 03:25:29 DEBUG "value": [ 03:25:29 DEBUG } 03:25:29 DEBUG 1713061529, 03:25:29 DEBUG ] 03:25:29 DEBUG "0.00030000000000001136" 03:25:29 DEBUG } 03:25:29 DEBUG ] 03:25:29 DEBUG } 03:25:29 DEBUG } 03:25:29 DEBUG ] 03:25:29 DEBUG } 03:25:29 DEBUG } 03:25:29 INFO [http_cmd]: http status code OK 03:25:29 DEBUG --- status code --- 03:25:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:29 DEBUG --- http response --- 03:25:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:25:29 DEBUG ----- output ----- 03:25:29 DEBUG { 03:25:29 DEBUG "status": "success", 03:25:29 DEBUG "data": { 03:25:29 DEBUG "resultType": "vector", 03:25:29 DEBUG "result": [ 03:25:29 DEBUG { 03:25:29 DEBUG "metric": { 03:25:29 DEBUG "instance": "10.142.0.107:9100" 03:25:29 DEBUG }, 03:25:29 DEBUG "value": [ 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG [...] 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG "instance": "10.142.0.43:9100" 03:25:29 DEBUG }, 03:25:29 DEBUG "value": [ 03:25:29 DEBUG 1713061529, 03:25:29 DEBUG "12654261248" 03:25:29 DEBUG ] 03:25:29 DEBUG } 03:25:29 DEBUG ] 03:25:29 DEBUG } 03:25:29 DEBUG } 03:25:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061529" 03:25:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061529" 03:25:29 INFO [http_cmd]: http status code OK 03:25:29 DEBUG --- status code --- 03:25:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:29 DEBUG --- http response --- 03:25:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:25:29 DEBUG ----- output ----- 03:25:29 DEBUG { 03:25:29 DEBUG "status": "success", 03:25:29 DEBUG "data": { 03:25:29 DEBUG "resultType": "vector", 03:25:29 DEBUG "result": [ 03:25:29 DEBUG { 03:25:29 DEBUG "metric": { 03:25:29 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 03:25:29 DEBUG }, 03:25:29 DEBUG "value": [ 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG [...] 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG "pod": "idm-b85f9d978-j5pbm" 03:25:29 DEBUG }, 03:25:29 DEBUG "value": [ 03:25:29 DEBUG 1713061529, 03:25:29 DEBUG "509.56364880415515" 03:25:29 DEBUG ] 03:25:29 DEBUG } 03:25:29 DEBUG ] 03:25:29 DEBUG } 03:25:29 DEBUG } 03:25:29 INFO [http_cmd]: http status code OK 03:25:29 DEBUG --- status code --- 03:25:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:29 DEBUG --- http response --- 03:25:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:25:29 DEBUG ----- output ----- 03:25:29 DEBUG { 03:25:29 DEBUG "status": "success", 03:25:29 DEBUG "data": { 03:25:29 DEBUG "resultType": "vector", 03:25:29 DEBUG "result": [ 03:25:29 DEBUG { 03:25:29 DEBUG "metric": { 03:25:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:25:29 DEBUG }, 03:25:29 DEBUG "value": [ 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG [...] 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:25:29 DEBUG }, 03:25:29 DEBUG "value": [ 03:25:29 DEBUG 1713061529, 03:25:29 DEBUG "1259.4333333333334" 03:25:29 DEBUG ] 03:25:29 DEBUG } 03:25:29 DEBUG ] 03:25:29 DEBUG } 03:25:29 DEBUG } 03:25:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061529" 03:25:29 INFO [http_cmd]: http status code OK 03:25:29 DEBUG --- status code --- 03:25:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:29 DEBUG --- http response --- 03:25:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:25:29 DEBUG ----- output ----- 03:25:29 DEBUG { 03:25:29 DEBUG "status": "success", 03:25:29 DEBUG "data": { 03:25:29 DEBUG "resultType": "vector", 03:25:29 DEBUG "result": [ 03:25:29 DEBUG { 03:25:29 DEBUG "metric": { 03:25:29 DEBUG "instance": "10.142.0.107:9100" 03:25:29 DEBUG }, 03:25:29 DEBUG "value": [ 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG [...] 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG 03:25:29 DEBUG "instance": "10.142.0.43:9100" 03:25:29 DEBUG }, 03:25:29 DEBUG "value": [ 03:25:29 DEBUG 1713061529, 03:25:29 DEBUG "2796.5333333333333" 03:25:29 DEBUG ] 03:25:29 DEBUG } 03:25:29 DEBUG ] 03:25:29 DEBUG } 03:25:29 DEBUG } 03:25:29 INFO 03:25:29 INFO [loop_until]: kubectl --namespace=xlou top pods 03:25:29 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:25:29 INFO [loop_until]: OK (rc = 0) 03:25:29 DEBUG --- stdout --- 03:25:29 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 5819m 8936Mi am-7d597cf7d6-8vr8f 5144m 8521Mi am-7d597cf7d6-9wz9h 7013m 8796Mi ds-cts-0 3179m 7849Mi ds-cts-1 3707m 8047Mi ds-cts-2 3224m 8150Mi ds-idrepo-0 6169m 4513Mi ds-idrepo-1 1011m 4286Mi ds-idrepo-2 993m 4104Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 5m 990Mi idm-b85f9d978-rl76f 4m 984Mi lodemon-766dc9b5df-2jmd4 2m 63Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 7787m 8162Mi 03:25:29 DEBUG --- stderr --- 03:25:29 DEBUG 03:25:30 INFO 03:25:30 INFO [loop_until]: kubectl --namespace=xlou top node 03:25:30 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:25:30 INFO [loop_until]: OK (rc = 0) 03:25:30 DEBUG --- stdout --- 03:25:30 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 4816Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 4297m 14% 10315Mi 9% gke-xlou-cdm-default-pool-1b0769c8-nhlf 5858m 19% 10980Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 10606m 35% 3595Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 1138m 3% 6340Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 4076m 13% 11280Mi 10% gke-xlou-cdm-default-pool-1b0769c8-xnmp 68m 0% 2914Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 5817m 19% 9957Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1061m 3% 5522Mi 4% gke-xlou-cdm-ds-0f601751-j488 4138m 13% 9532Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 63m 0% 2183Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 7731m 25% 10340Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 6552m 21% 5971Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 8188m 51% 10952Mi 18% 03:25:30 DEBUG --- stderr --- 03:25:30 DEBUG 03:25:49 WARNING No values available for selected timeframe. 03:25:49 WARNING No values available for selected timeframe. 03:25:49 WARNING No values available for selected timeframe. 03:25:49 WARNING No values available for selected timeframe. 03:25:49 WARNING No values available for selected timeframe. 03:25:49 WARNING No values available for selected timeframe. 03:25:49 WARNING No values available for selected timeframe. 03:25:49 WARNING No values available for selected timeframe. 03:25:49 WARNING No values available for selected timeframe. 03:25:49 WARNING No values available for selected timeframe. 03:25:49 WARNING No values available for selected timeframe. 03:25:49 WARNING No values available for selected timeframe. 03:25:49 WARNING No values available for selected timeframe. 03:25:49 WARNING No values available for selected timeframe. 03:25:49 WARNING No values available for selected timeframe. 03:25:49 WARNING No values available for selected timeframe. 03:25:49 WARNING File /tmp/lodemon_data-20240414_031429/node_disk_read_bytes_total.json does not exist. Skipping average value computation... 127.0.0.1 - - [14/Apr/2024 03:25:49] "GET /monitoring/average?start_time=20240414_022344&stop_time=20240414_022349 HTTP/1.1" 200 - 03:26:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061589" 03:26:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061589" 03:26:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061589" 03:26:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061589" 03:26:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713061589" 03:26:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061589" 03:26:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713061589" 03:26:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061589" 03:26:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713061589" 03:26:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713061589" 03:26:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061589" 03:26:29 INFO [http_cmd]: http status code OK 03:26:29 INFO [http_cmd]: http status code OK 03:26:29 DEBUG --- status code --- 03:26:29 DEBUG --- status code --- 03:26:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:29 DEBUG --- http response --- 03:26:29 DEBUG --- http response --- 03:26:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:26:29 DEBUG ----- output ----- 03:26:29 DEBUG { 03:26:29 DEBUG "status": "success", 03:26:29 DEBUG "data": { 03:26:29 DEBUG "resultType": "vector", 03:26:29 DEBUG "result": [ 03:26:29 DEBUG { 03:26:29 DEBUG "metric": { 03:26:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:26:29 DEBUG }, 03:26:29 DEBUG "value": [ 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG [...] 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:26:29 DEBUG }, 03:26:29 DEBUG "value": [ 03:26:29 DEBUG 1713061589, 03:26:29 DEBUG "181866.5" 03:26:29 DEBUG ] 03:26:29 DEBUG } 03:26:29 DEBUG ] 03:26:29 DEBUG } 03:26:29 DEBUG } 03:26:29 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:26:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713061589" 03:26:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061589" 03:26:29 ERROR [http_cmd]: ERROR 03:26:29 ERROR --- status code --- 03:26:29 INFO [http_cmd]: http status code OK 03:26:29 INFO [http_cmd]: http status code OK 03:26:29 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:26:29 INFO [http_cmd]: http status code OK 03:26:29 INFO [http_cmd]: http status code OK 03:26:29 DEBUG --- status code --- 03:26:29 DEBUG --- status code --- 03:26:29 ERROR --- http response --- 03:26:29 DEBUG --- status code --- 03:26:29 DEBUG --- status code --- 03:26:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:29 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:26:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:29 DEBUG --- http response --- 03:26:29 DEBUG --- http response --- 03:26:29 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:26:29 DEBUG --- http response --- 03:26:29 DEBUG --- http response --- 03:26:29 INFO [http_cmd]: http status code OK 03:26:29 INFO [http_cmd]: http status code OK 03:26:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:26:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:26:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:26:29 DEBUG --- status code --- 03:26:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:26:29 INFO [http_cmd]: http status code OK 03:26:29 DEBUG --- status code --- 03:26:29 DEBUG ----- output ----- 03:26:29 DEBUG ----- output ----- 03:26:29 DEBUG ----- output ----- 03:26:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:29 DEBUG ----- output ----- 03:26:29 DEBUG --- status code --- 03:26:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:29 DEBUG { 03:26:29 DEBUG { 03:26:29 DEBUG { 03:26:29 DEBUG --- http response --- 03:26:29 DEBUG { 03:26:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:29 INFO [http_cmd]: http status code OK 03:26:29 DEBUG --- http response --- 03:26:29 DEBUG "status": "success", 03:26:29 DEBUG "status": "success", 03:26:29 DEBUG "status": "success", 03:26:29 DEBUG "status": "success", 03:26:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:26:29 DEBUG --- http response --- 03:26:29 DEBUG --- status code --- 03:26:29 INFO [http_cmd]: http status code OK 03:26:29 DEBUG "data": { 03:26:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:26:29 DEBUG "data": { 03:26:29 DEBUG "data": { 03:26:29 DEBUG "data": { 03:26:29 DEBUG ----- output ----- 03:26:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:26:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:29 DEBUG --- status code --- 03:26:29 DEBUG "resultType": "vector", 03:26:29 DEBUG ----- output ----- 03:26:29 DEBUG "resultType": "vector", 03:26:29 DEBUG "resultType": "vector", 03:26:29 DEBUG "resultType": "vector", 03:26:29 DEBUG { 03:26:29 DEBUG ----- output ----- 03:26:29 DEBUG --- http response --- 03:26:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:29 DEBUG "result": [ 03:26:29 DEBUG { 03:26:29 DEBUG "result": [ 03:26:29 DEBUG "result": [ 03:26:29 DEBUG "result": [ 03:26:29 DEBUG "status": "success", 03:26:29 DEBUG { 03:26:29 DEBUG --- http response --- 03:26:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:26:29 DEBUG { 03:26:29 INFO [http_cmd]: http status code OK 03:26:29 DEBUG "status": "success", 03:26:29 DEBUG { 03:26:29 DEBUG { 03:26:29 DEBUG { 03:26:29 DEBUG "data": { 03:26:29 DEBUG "status": "success", 03:26:29 DEBUG ----- output ----- 03:26:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:26:29 DEBUG "metric": { 03:26:29 DEBUG --- status code --- 03:26:29 DEBUG "data": { 03:26:29 DEBUG "metric": { 03:26:29 DEBUG "metric": { 03:26:29 DEBUG "metric": { 03:26:29 DEBUG "resultType": "vector", 03:26:29 DEBUG "data": { 03:26:29 DEBUG { 03:26:29 DEBUG ----- output ----- 03:26:29 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 03:26:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:29 DEBUG "resultType": "vector", 03:26:29 DEBUG "pod": "ds-idrepo-1" 03:26:29 DEBUG "pod": "ds-idrepo-1" 03:26:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 03:26:29 DEBUG "result": [ 03:26:29 DEBUG "resultType": "vector", 03:26:29 DEBUG "status": "success", 03:26:29 DEBUG { 03:26:29 DEBUG }, 03:26:29 DEBUG --- http response --- 03:26:29 DEBUG "result": [ 03:26:29 DEBUG }, 03:26:29 DEBUG }, 03:26:29 DEBUG }, 03:26:29 DEBUG { 03:26:29 DEBUG "result": [ 03:26:29 DEBUG "data": { 03:26:29 DEBUG "status": "success", 03:26:29 DEBUG "value": [ 03:26:29 DEBUG { 03:26:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:26:29 DEBUG "value": [ 03:26:29 DEBUG "value": [ 03:26:29 DEBUG "value": [ 03:26:29 DEBUG "metric": { 03:26:29 DEBUG { 03:26:29 DEBUG "resultType": "vector", 03:26:29 DEBUG "data": { 03:26:29 DEBUG 03:26:29 DEBUG "metric": { 03:26:29 DEBUG ----- output ----- 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:26:29 DEBUG "metric": { 03:26:29 DEBUG "result": [ 03:26:29 DEBUG "resultType": "vector", 03:26:29 DEBUG 03:26:29 DEBUG "pod": "ds-idrepo-1" 03:26:29 DEBUG { 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG }, 03:26:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 03:26:29 DEBUG { 03:26:29 DEBUG "result": [ 03:26:29 DEBUG 03:26:29 DEBUG }, 03:26:29 DEBUG "status": "success", 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG "value": [ 03:26:29 DEBUG }, 03:26:29 DEBUG "metric": { 03:26:29 DEBUG { 03:26:29 DEBUG [...] 03:26:29 DEBUG "value": [ 03:26:29 DEBUG "data": { 03:26:29 DEBUG [...] 03:26:29 DEBUG [...] 03:26:29 DEBUG [...] 03:26:29 DEBUG 03:26:29 DEBUG "value": [ 03:26:29 DEBUG "instance": "10.142.0.107:9100" 03:26:29 DEBUG "metric": { 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG "resultType": "vector", 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG }, 03:26:29 DEBUG "pod": "ds-idrepo-1" 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG "result": [ 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG "value": [ 03:26:29 DEBUG }, 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG { 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG [...] 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG "value": [ 03:26:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:26:29 DEBUG [...] 03:26:29 DEBUG "metric": { 03:26:29 DEBUG "pod": "ds-idrepo-0" 03:26:29 DEBUG "pod": "ds-idrepo-0" 03:26:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:26:29 DEBUG 03:26:29 DEBUG [...] 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG }, 03:26:29 DEBUG 03:26:29 DEBUG "pod": "ds-idrepo-1" 03:26:29 DEBUG }, 03:26:29 DEBUG }, 03:26:29 DEBUG }, 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG "value": [ 03:26:29 DEBUG 03:26:29 DEBUG }, 03:26:29 DEBUG "value": [ 03:26:29 DEBUG "value": [ 03:26:29 DEBUG "value": [ 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG [...] 03:26:29 DEBUG 03:26:29 DEBUG 1713061589, 03:26:29 DEBUG 03:26:29 DEBUG "value": [ 03:26:29 DEBUG 1713061589, 03:26:29 DEBUG 1713061589, 03:26:29 DEBUG 1713061589, 03:26:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG [...] 03:26:29 DEBUG "0" 03:26:29 DEBUG "pod": "ds-idrepo-0" 03:26:29 DEBUG 03:26:29 DEBUG "0" 03:26:29 DEBUG "0" 03:26:29 DEBUG "111640576" 03:26:29 DEBUG }, 03:26:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG ] 03:26:29 DEBUG }, 03:26:29 DEBUG 03:26:29 DEBUG ] 03:26:29 DEBUG ] 03:26:29 DEBUG ] 03:26:29 DEBUG "value": [ 03:26:29 DEBUG }, 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG } 03:26:29 DEBUG "value": [ 03:26:29 DEBUG 03:26:29 DEBUG } 03:26:29 DEBUG } 03:26:29 DEBUG } 03:26:29 DEBUG 1713061589, 03:26:29 DEBUG "value": [ 03:26:29 DEBUG "instance": "10.142.0.43:9100" 03:26:29 DEBUG 03:26:29 DEBUG ] 03:26:29 DEBUG 1713061589, 03:26:29 DEBUG [...] 03:26:29 DEBUG ] 03:26:29 DEBUG ] 03:26:29 DEBUG ] 03:26:29 DEBUG "0.015" 03:26:29 DEBUG 1713061589, 03:26:29 DEBUG }, 03:26:29 DEBUG "pod": "ds-cts-1" 03:26:29 DEBUG } 03:26:29 DEBUG "0" 03:26:29 DEBUG 03:26:29 DEBUG } 03:26:29 DEBUG } 03:26:29 DEBUG } 03:26:29 DEBUG ] 03:26:29 DEBUG "1.1462233756758775" 03:26:29 DEBUG "value": [ 03:26:29 DEBUG }, 03:26:29 DEBUG } 03:26:29 DEBUG ] 03:26:29 DEBUG 03:26:29 DEBUG } 03:26:29 DEBUG } 03:26:29 DEBUG } 03:26:29 DEBUG } 03:26:29 DEBUG ] 03:26:29 DEBUG 1713061589, 03:26:29 DEBUG "value": [ 03:26:29 DEBUG } 03:26:29 DEBUG 03:26:29 DEBUG ] 03:26:29 DEBUG } 03:26:29 DEBUG "0" 03:26:29 DEBUG 1713061589, 03:26:29 DEBUG ] 03:26:29 DEBUG "pod": "ds-idrepo-0" 03:26:29 DEBUG } 03:26:29 DEBUG ] 03:26:29 DEBUG ] 03:26:29 DEBUG "0" 03:26:29 DEBUG } 03:26:29 DEBUG }, 03:26:29 DEBUG } 03:26:29 DEBUG } 03:26:29 DEBUG } 03:26:29 DEBUG ] 03:26:29 DEBUG } 03:26:29 DEBUG "value": [ 03:26:29 DEBUG } 03:26:29 DEBUG ] 03:26:29 DEBUG } 03:26:29 DEBUG 1713061589, 03:26:29 DEBUG } 03:26:29 DEBUG ] 03:26:29 DEBUG "0" 03:26:29 DEBUG } 03:26:29 DEBUG } 03:26:29 DEBUG ] 03:26:29 DEBUG } 03:26:29 DEBUG } 03:26:29 DEBUG ] 03:26:29 DEBUG } 03:26:29 DEBUG } 03:26:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061589" 03:26:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061589" 03:26:29 INFO [http_cmd]: http status code OK 03:26:29 DEBUG --- status code --- 03:26:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:29 DEBUG --- http response --- 03:26:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:26:29 DEBUG ----- output ----- 03:26:29 DEBUG { 03:26:29 DEBUG "status": "success", 03:26:29 DEBUG "data": { 03:26:29 DEBUG "resultType": "vector", 03:26:29 DEBUG "result": [ 03:26:29 DEBUG { 03:26:29 DEBUG "metric": { 03:26:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:26:29 DEBUG }, 03:26:29 DEBUG "value": [ 03:26:29 INFO [http_cmd]: http status code OK 03:26:29 DEBUG 03:26:29 DEBUG --- status code --- 03:26:29 DEBUG 03:26:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:29 DEBUG 03:26:29 DEBUG --- http response --- 03:26:29 DEBUG [...] 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:26:29 DEBUG }, 03:26:29 DEBUG "value": [ 03:26:29 DEBUG 1713061589, 03:26:29 DEBUG "0" 03:26:29 DEBUG ] 03:26:29 DEBUG } 03:26:29 DEBUG ] 03:26:29 DEBUG } 03:26:29 DEBUG } 03:26:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:26:29 DEBUG ----- output ----- 03:26:29 DEBUG { 03:26:29 DEBUG "status": "success", 03:26:29 DEBUG "data": { 03:26:29 DEBUG "resultType": "vector", 03:26:29 DEBUG "result": [ 03:26:29 DEBUG { 03:26:29 DEBUG "metric": { 03:26:29 DEBUG "instance": "10.142.0.107:9100" 03:26:29 DEBUG }, 03:26:29 DEBUG "value": [ 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG [...] 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG "instance": "10.142.0.43:9100" 03:26:29 DEBUG }, 03:26:29 DEBUG "value": [ 03:26:29 DEBUG 1713061589, 03:26:29 DEBUG "5749.866666666667" 03:26:29 DEBUG ] 03:26:29 DEBUG } 03:26:29 DEBUG ] 03:26:29 DEBUG } 03:26:29 DEBUG } 03:26:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061589" 03:26:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061589" 03:26:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061589" 03:26:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061589" 03:26:29 INFO [http_cmd]: http status code OK 03:26:29 DEBUG --- status code --- 03:26:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:29 DEBUG --- http response --- 03:26:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 03:26:29 DEBUG ----- output ----- 03:26:29 DEBUG { 03:26:29 DEBUG "status": "success", 03:26:29 DEBUG "data": { 03:26:29 DEBUG "resultType": "vector", 03:26:29 DEBUG "result": [ 03:26:29 DEBUG { 03:26:29 DEBUG "metric": { 03:26:29 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 03:26:29 DEBUG }, 03:26:29 DEBUG "value": [ 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG [...] 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG "pod": "idm-b85f9d978-j5pbm" 03:26:29 DEBUG }, 03:26:29 DEBUG "value": [ 03:26:29 DEBUG 1713061589, 03:26:29 DEBUG "0" 03:26:29 DEBUG ] 03:26:29 DEBUG } 03:26:29 DEBUG ] 03:26:29 DEBUG } 03:26:29 DEBUG } 03:26:29 INFO [http_cmd]: http status code OK 03:26:29 INFO [http_cmd]: http status code OK 03:26:29 DEBUG --- status code --- 03:26:29 DEBUG --- status code --- 03:26:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:29 DEBUG --- http response --- 03:26:29 DEBUG --- http response --- 03:26:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:26:29 DEBUG ----- output ----- 03:26:29 DEBUG { 03:26:29 DEBUG "status": "success", 03:26:29 DEBUG "data": { 03:26:29 DEBUG "resultType": "vector", 03:26:29 DEBUG "result": [ 03:26:29 DEBUG { 03:26:29 DEBUG "metric": { 03:26:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:26:29 DEBUG }, 03:26:29 DEBUG "value": [ 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG [...] 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:26:29 DEBUG }, 03:26:29 DEBUG "value": [ 03:26:29 DEBUG 1713061589, 03:26:29 DEBUG "0" 03:26:29 DEBUG ] 03:26:29 DEBUG } 03:26:29 DEBUG ] 03:26:29 DEBUG } 03:26:29 DEBUG } 03:26:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:26:29 DEBUG ----- output ----- 03:26:29 DEBUG { 03:26:29 DEBUG "status": "success", 03:26:29 DEBUG "data": { 03:26:29 DEBUG "resultType": "vector", 03:26:29 DEBUG "result": [ 03:26:29 DEBUG { 03:26:29 DEBUG "metric": { 03:26:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:26:29 DEBUG }, 03:26:29 DEBUG "value": [ 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG [...] 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:26:29 DEBUG }, 03:26:29 DEBUG "value": [ 03:26:29 DEBUG 1713061589, 03:26:29 DEBUG "1320.9" 03:26:29 DEBUG ] 03:26:29 DEBUG } 03:26:29 DEBUG ] 03:26:29 DEBUG } 03:26:29 DEBUG } 03:26:29 INFO [http_cmd]: http status code OK 03:26:29 DEBUG --- status code --- 03:26:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:29 DEBUG --- http response --- 03:26:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:26:29 DEBUG ----- output ----- 03:26:29 DEBUG { 03:26:29 DEBUG "status": "success", 03:26:29 DEBUG "data": { 03:26:29 DEBUG "resultType": "vector", 03:26:29 DEBUG "result": [ 03:26:29 DEBUG { 03:26:29 DEBUG "metric": { 03:26:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:26:29 DEBUG }, 03:26:29 DEBUG "value": [ 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG [...] 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:26:29 DEBUG }, 03:26:29 DEBUG "value": [ 03:26:29 DEBUG 1713061589, 03:26:29 DEBUG "0" 03:26:29 DEBUG ] 03:26:29 DEBUG } 03:26:29 DEBUG ] 03:26:29 DEBUG } 03:26:29 DEBUG } 03:26:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061589" 03:26:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061589" 03:26:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061589" 03:26:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061589" 03:26:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061589" 03:26:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061589" 03:26:29 INFO [http_cmd]: http status code OK 03:26:29 INFO [http_cmd]: http status code OK 03:26:29 DEBUG --- status code --- 03:26:29 INFO [http_cmd]: http status code OK 03:26:29 INFO [http_cmd]: http status code OK 03:26:29 DEBUG --- status code --- 03:26:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:29 DEBUG --- status code --- 03:26:29 DEBUG --- status code --- 03:26:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:29 DEBUG --- http response --- 03:26:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:29 DEBUG --- http response --- 03:26:29 DEBUG --- http response --- 03:26:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:26:29 DEBUG --- http response --- 03:26:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:26:29 DEBUG ----- output ----- 03:26:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:26:29 DEBUG ----- output ----- 03:26:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:26:29 DEBUG { 03:26:29 DEBUG ----- output ----- 03:26:29 DEBUG { 03:26:29 DEBUG ----- output ----- 03:26:29 DEBUG "status": "success", 03:26:29 DEBUG { 03:26:29 DEBUG "status": "success", 03:26:29 DEBUG { 03:26:29 DEBUG "data": { 03:26:29 DEBUG "status": "success", 03:26:29 DEBUG "data": { 03:26:29 DEBUG "status": "success", 03:26:29 DEBUG "resultType": "vector", 03:26:29 DEBUG "data": { 03:26:29 DEBUG "resultType": "vector", 03:26:29 DEBUG "data": { 03:26:29 DEBUG "result": [ 03:26:29 DEBUG "resultType": "vector", 03:26:29 DEBUG "result": [ 03:26:29 DEBUG "resultType": "vector", 03:26:29 INFO [http_cmd]: http status code OK 03:26:29 DEBUG { 03:26:29 DEBUG "result": [ 03:26:29 DEBUG { 03:26:29 DEBUG "result": [ 03:26:29 DEBUG --- status code --- 03:26:29 DEBUG "metric": { 03:26:29 DEBUG { 03:26:29 DEBUG "metric": { 03:26:29 DEBUG { 03:26:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:29 DEBUG "pod": "ds-idrepo-1" 03:26:29 DEBUG "metric": { 03:26:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:26:29 DEBUG "metric": { 03:26:29 DEBUG --- http response --- 03:26:29 DEBUG }, 03:26:29 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 03:26:29 DEBUG }, 03:26:29 DEBUG "instance": "10.142.0.107:9100" 03:26:29 DEBUG "value": [ 03:26:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 03:26:29 DEBUG }, 03:26:29 DEBUG "value": [ 03:26:29 DEBUG }, 03:26:29 DEBUG 03:26:29 DEBUG ----- output ----- 03:26:29 DEBUG "value": [ 03:26:29 DEBUG 03:26:29 DEBUG "value": [ 03:26:29 DEBUG 03:26:29 DEBUG { 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG "status": "success", 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG [...] 03:26:29 DEBUG "data": { 03:26:29 INFO [http_cmd]: http status code OK 03:26:29 DEBUG 03:26:29 DEBUG [...] 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG "resultType": "vector", 03:26:29 DEBUG --- status code --- 03:26:29 DEBUG [...] 03:26:29 DEBUG 03:26:29 DEBUG [...] 03:26:29 DEBUG 03:26:29 DEBUG "result": [ 03:26:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG { 03:26:29 DEBUG --- http response --- 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG "pod": "ds-idrepo-0" 03:26:29 DEBUG "metric": { 03:26:29 DEBUG 03:26:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:26:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:26:29 DEBUG 03:26:29 DEBUG }, 03:26:29 DEBUG "pod": "ds-idrepo-1" 03:26:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:26:29 DEBUG ----- output ----- 03:26:29 DEBUG }, 03:26:29 DEBUG "instance": "10.142.0.43:9100" 03:26:29 DEBUG "value": [ 03:26:29 DEBUG }, 03:26:29 DEBUG }, 03:26:29 DEBUG { 03:26:29 DEBUG "value": [ 03:26:29 DEBUG }, 03:26:29 DEBUG 1713061589, 03:26:29 DEBUG "value": [ 03:26:29 DEBUG "value": [ 03:26:29 DEBUG "status": "success", 03:26:29 DEBUG 1713061589, 03:26:29 DEBUG "value": [ 03:26:29 DEBUG "0" 03:26:29 DEBUG 03:26:29 DEBUG 1713061589, 03:26:29 DEBUG "data": { 03:26:29 DEBUG "0" 03:26:29 DEBUG 1713061589, 03:26:29 DEBUG ] 03:26:29 DEBUG 03:26:29 DEBUG "0" 03:26:29 DEBUG "resultType": "vector", 03:26:29 DEBUG ] 03:26:29 DEBUG "0.00030000000000001136" 03:26:29 DEBUG } 03:26:29 DEBUG 03:26:29 DEBUG ] 03:26:29 DEBUG "result": [ 03:26:29 DEBUG } 03:26:29 DEBUG ] 03:26:29 DEBUG ] 03:26:29 DEBUG [...] 03:26:29 DEBUG } 03:26:29 DEBUG { 03:26:29 DEBUG ] 03:26:29 DEBUG } 03:26:29 DEBUG } 03:26:29 DEBUG 03:26:29 DEBUG ] 03:26:29 DEBUG "metric": { 03:26:29 DEBUG } 03:26:29 DEBUG ] 03:26:29 DEBUG } 03:26:29 DEBUG 03:26:29 DEBUG } 03:26:29 DEBUG "instance": "10.142.0.107:9100" 03:26:29 DEBUG } 03:26:29 DEBUG } 03:26:29 DEBUG 03:26:29 DEBUG } 03:26:29 DEBUG }, 03:26:29 DEBUG } 03:26:29 DEBUG "pod": "ds-cts-1" 03:26:29 DEBUG "value": [ 03:26:29 DEBUG }, 03:26:29 DEBUG 03:26:29 DEBUG "value": [ 03:26:29 DEBUG 03:26:29 DEBUG 1713061589, 03:26:29 DEBUG 03:26:29 DEBUG "0.0001" 03:26:29 DEBUG [...] 03:26:29 DEBUG ] 03:26:29 DEBUG 03:26:29 DEBUG } 03:26:29 DEBUG 03:26:29 DEBUG ] 03:26:29 DEBUG 03:26:29 DEBUG } 03:26:29 DEBUG "instance": "10.142.0.43:9100" 03:26:29 DEBUG } 03:26:29 DEBUG }, 03:26:29 DEBUG "value": [ 03:26:29 DEBUG 1713061589, 03:26:29 DEBUG "12654861312" 03:26:29 DEBUG ] 03:26:29 DEBUG } 03:26:29 DEBUG ] 03:26:29 DEBUG } 03:26:29 DEBUG } 03:26:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061589" 03:26:29 INFO [http_cmd]: http status code OK 03:26:29 DEBUG --- status code --- 03:26:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:29 DEBUG --- http response --- 03:26:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 03:26:29 DEBUG ----- output ----- 03:26:29 DEBUG { 03:26:29 DEBUG "status": "success", 03:26:29 DEBUG "data": { 03:26:29 DEBUG "resultType": "vector", 03:26:29 DEBUG "result": [ 03:26:29 DEBUG { 03:26:29 DEBUG "metric": { 03:26:29 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 03:26:29 DEBUG }, 03:26:29 DEBUG "value": [ 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG [...] 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG "pod": "idm-b85f9d978-j5pbm" 03:26:29 DEBUG }, 03:26:29 DEBUG "value": [ 03:26:29 DEBUG 1713061589, 03:26:29 DEBUG "551.5755453810935" 03:26:29 DEBUG ] 03:26:29 DEBUG } 03:26:29 DEBUG ] 03:26:29 DEBUG } 03:26:29 DEBUG } 03:26:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061589" 03:26:29 INFO [http_cmd]: http status code OK 03:26:29 DEBUG --- status code --- 03:26:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:29 DEBUG --- http response --- 03:26:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:26:29 DEBUG ----- output ----- 03:26:29 DEBUG { 03:26:29 DEBUG "status": "success", 03:26:29 DEBUG "data": { 03:26:29 DEBUG "resultType": "vector", 03:26:29 DEBUG "result": [ 03:26:29 DEBUG { 03:26:29 DEBUG "metric": { 03:26:29 DEBUG "instance": "10.142.0.107:9100" 03:26:29 DEBUG }, 03:26:29 DEBUG "value": [ 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG [...] 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG 03:26:29 DEBUG "instance": "10.142.0.43:9100" 03:26:29 DEBUG }, 03:26:29 DEBUG "value": [ 03:26:29 DEBUG 1713061589, 03:26:29 DEBUG "3375.6" 03:26:29 DEBUG ] 03:26:29 DEBUG } 03:26:29 DEBUG ] 03:26:29 DEBUG } 03:26:29 DEBUG } 03:26:29 INFO 03:26:29 INFO [loop_until]: kubectl --namespace=xlou top pods 03:26:29 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:26:29 INFO [loop_until]: OK (rc = 0) 03:26:29 DEBUG --- stdout --- 03:26:29 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 1215m 8941Mi am-7d597cf7d6-8vr8f 1163m 8525Mi am-7d597cf7d6-9wz9h 1438m 8793Mi ds-cts-0 2866m 7862Mi ds-cts-1 2877m 8076Mi ds-cts-2 3018m 8149Mi ds-idrepo-0 828m 4519Mi ds-idrepo-1 104m 4287Mi ds-idrepo-2 102m 4104Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 5m 989Mi idm-b85f9d978-rl76f 4m 983Mi lodemon-766dc9b5df-2jmd4 1m 63Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 2m 15Mi 03:26:29 DEBUG --- stderr --- 03:26:29 DEBUG 03:26:30 INFO 03:26:30 INFO [loop_until]: kubectl --namespace=xlou top node 03:26:30 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:26:30 INFO [loop_until]: OK (rc = 0) 03:26:30 DEBUG --- stdout --- 03:26:30 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 4809Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 2939m 9% 10302Mi 9% gke-xlou-cdm-default-pool-1b0769c8-nhlf 1244m 4% 11040Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 91m 0% 3049Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 154m 0% 6323Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 3061m 10% 11254Mi 10% gke-xlou-cdm-default-pool-1b0769c8-xnmp 68m 0% 2927Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 1280m 4% 9995Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 161m 0% 5515Mi 4% gke-xlou-cdm-ds-0f601751-j488 2902m 9% 9530Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 68m 0% 2189Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 1575m 5% 10403Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 946m 3% 6014Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 72m 0% 3819Mi 6% 03:26:30 DEBUG --- stderr --- 03:26:30 DEBUG 03:27:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061649" 03:27:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061649" 03:27:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713061649" 03:27:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061649" 03:27:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713061649" 03:27:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061649" 03:27:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713061649" 03:27:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713061649" 03:27:29 INFO [http_cmd]: http status code OK 03:27:29 DEBUG --- status code --- 03:27:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:27:29 DEBUG --- http response --- 03:27:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:27:29 DEBUG ----- output ----- 03:27:29 DEBUG { 03:27:29 DEBUG "status": "success", 03:27:29 DEBUG "data": { 03:27:29 DEBUG "resultType": "vector", 03:27:29 DEBUG "result": [ 03:27:29 DEBUG { 03:27:29 DEBUG "metric": { 03:27:29 DEBUG "pod": "ds-idrepo-1" 03:27:29 DEBUG }, 03:27:29 DEBUG "value": [ 03:27:29 DEBUG 03:27:29 DEBUG 03:27:29 DEBUG 03:27:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061649" 03:27:29 DEBUG [...] 03:27:29 DEBUG 03:27:29 DEBUG 03:27:29 DEBUG 03:27:29 DEBUG "pod": "ds-idrepo-0" 03:27:29 DEBUG }, 03:27:29 DEBUG "value": [ 03:27:29 DEBUG 1713061649, 03:27:29 DEBUG "0" 03:27:29 DEBUG ] 03:27:29 DEBUG } 03:27:29 DEBUG ] 03:27:29 DEBUG } 03:27:29 DEBUG } 03:27:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713061649" 03:27:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061649" 03:27:29 INFO [http_cmd]: http status code OK 03:27:29 DEBUG --- status code --- 03:27:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:27:29 DEBUG --- http response --- 03:27:29 INFO [http_cmd]: http status code OK 03:27:29 INFO [http_cmd]: http status code OK 03:27:29 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:27:29 INFO [http_cmd]: http status code OK 03:27:29 DEBUG --- status code --- 03:27:29 DEBUG --- status code --- 03:27:29 DEBUG --- status code --- 03:27:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:27:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:27:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:27:29 DEBUG --- http response --- 03:27:29 DEBUG --- http response --- 03:27:29 DEBUG --- http response --- 03:27:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:27:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:27:29 ERROR [http_cmd]: ERROR 03:27:29 DEBUG ----- output ----- 03:27:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:27:29 DEBUG ----- output ----- 03:27:29 ERROR --- status code --- 03:27:29 DEBUG { 03:27:29 DEBUG ----- output ----- 03:27:29 DEBUG { 03:27:29 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:27:29 DEBUG "status": "success", 03:27:29 DEBUG { 03:27:29 DEBUG "status": "success", 03:27:29 ERROR --- http response --- 03:27:29 DEBUG "data": { 03:27:29 DEBUG "status": "success", 03:27:29 DEBUG "data": { 03:27:29 DEBUG "resultType": "vector", 03:27:29 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:27:29 DEBUG "data": { 03:27:29 DEBUG "resultType": "vector", 03:27:29 DEBUG "result": [ 03:27:29 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:27:29 DEBUG "resultType": "vector", 03:27:29 DEBUG "result": [ 03:27:29 DEBUG { 03:27:29 DEBUG "result": [ 03:27:29 DEBUG { 03:27:29 DEBUG "metric": { 03:27:29 DEBUG { 03:27:29 DEBUG "metric": { 03:27:29 DEBUG "pod": "ds-idrepo-1" 03:27:29 DEBUG "metric": { 03:27:29 DEBUG "pod": "ds-idrepo-1" 03:27:29 DEBUG }, 03:27:29 DEBUG "instance": "10.142.0.107:9100" 03:27:29 DEBUG }, 03:27:29 DEBUG "value": [ 03:27:29 DEBUG }, 03:27:29 DEBUG "value": [ 03:27:29 DEBUG 03:27:29 DEBUG "value": [ 03:27:29 DEBUG 03:27:29 DEBUG 03:27:29 DEBUG 03:27:29 DEBUG 03:27:29 DEBUG 03:27:29 DEBUG 03:27:29 DEBUG 03:27:29 DEBUG [...] 03:27:29 DEBUG 03:27:29 DEBUG [...] 03:27:29 DEBUG 03:27:29 DEBUG [...] 03:27:29 DEBUG 03:27:29 DEBUG 03:27:29 DEBUG 03:27:29 DEBUG 03:27:29 DEBUG 03:27:29 DEBUG 03:27:29 DEBUG 03:27:29 DEBUG "pod": "ds-cts-1" 03:27:29 DEBUG 03:27:29 DEBUG "pod": "ds-idrepo-0" 03:27:29 DEBUG }, 03:27:29 DEBUG "instance": "10.142.0.43:9100" 03:27:29 DEBUG }, 03:27:29 DEBUG "value": [ 03:27:29 DEBUG }, 03:27:29 DEBUG "value": [ 03:27:29 DEBUG 1713061649, 03:27:29 DEBUG "value": [ 03:27:29 DEBUG 1713061649, 03:27:29 DEBUG "0" 03:27:29 DEBUG 1713061649, 03:27:29 DEBUG "0" 03:27:29 DEBUG ] 03:27:29 DEBUG "0" 03:27:29 DEBUG ] 03:27:29 DEBUG } 03:27:29 DEBUG ] 03:27:29 DEBUG } 03:27:29 DEBUG ] 03:27:29 DEBUG } 03:27:29 DEBUG ] 03:27:29 DEBUG } 03:27:29 DEBUG ] 03:27:29 DEBUG } 03:27:29 DEBUG } 03:27:29 DEBUG } 03:27:29 DEBUG } 03:27:29 DEBUG } 03:27:29 INFO [http_cmd]: http status code OK 03:27:29 DEBUG --- status code --- 03:27:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:27:29 DEBUG --- http response --- 03:27:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:27:29 DEBUG ----- output ----- 03:27:29 DEBUG { 03:27:29 DEBUG "status": "success", 03:27:29 DEBUG "data": { 03:27:29 DEBUG "resultType": "vector", 03:27:29 DEBUG "result": [ 03:27:29 DEBUG { 03:27:29 DEBUG "metric": { 03:27:29 DEBUG "pod": "ds-idrepo-1" 03:27:29 DEBUG }, 03:27:29 DEBUG "value": [ 03:27:29 DEBUG 03:27:29 DEBUG 03:27:29 DEBUG 03:27:29 DEBUG [...] 03:27:29 DEBUG 03:27:29 DEBUG 03:27:29 DEBUG 03:27:29 DEBUG "pod": "ds-idrepo-0" 03:27:29 DEBUG }, 03:27:29 DEBUG "value": [ 03:27:29 DEBUG 1713061649, 03:27:29 DEBUG "0" 03:27:29 DEBUG ] 03:27:29 DEBUG } 03:27:29 DEBUG ] 03:27:29 DEBUG } 03:27:29 DEBUG } 03:27:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061649" 03:27:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061649" 03:27:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713061649" 03:27:29 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713061649" 03:27:29 INFO [http_cmd]: http status code OK 03:27:29 INFO [http_cmd]: http status code OK 03:27:29 DEBUG --- status code --- 03:27:29 INFO [http_cmd]: http status code OK 03:27:29 INFO [http_cmd]: http status code OK 03:27:29 DEBUG --- status code --- 03:27:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:27:29 DEBUG --- status code --- 03:27:29 DEBUG --- status code --- 03:27:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:27:29 DEBUG --- http response --- 03:27:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:27:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:27:29 DEBUG --- http response --- 03:27:29 DEBUG --- http response --- 03:27:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:27:29 DEBUG --- http response --- 03:27:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:27:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:27:29 DEBUG ----- output ----- 03:27:29 DEBUG ----- output ----- 03:27:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:27:29 DEBUG ----- output ----- 03:27:29 INFO [http_cmd]: http status code OK 03:27:29 INFO [http_cmd]: http status code OK 03:27:29 DEBUG { 03:27:29 DEBUG { 03:27:29 DEBUG ----- output ----- 03:27:29 DEBUG { 03:27:29 DEBUG --- status code --- 03:27:29 DEBUG --- status code --- 03:27:29 DEBUG "status": "success", 03:27:29 DEBUG "status": "success", 03:27:29 DEBUG { 03:27:29 DEBUG "status": "success", 03:27:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:27:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:27:29 DEBUG "data": { 03:27:29 DEBUG "data": { 03:27:29 DEBUG "status": "success", 03:27:29 DEBUG "data": { 03:27:29 DEBUG --- http response --- 03:27:29 DEBUG --- http response --- 03:27:29 DEBUG "resultType": "vector", 03:27:29 DEBUG "resultType": "vector", 03:27:29 DEBUG "data": { 03:27:29 DEBUG "resultType": "vector", 03:27:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:27:29 DEBUG "result": [ 03:27:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:27:29 DEBUG "result": [ 03:27:29 DEBUG "resultType": "vector", 03:27:29 DEBUG "result": [ 03:27:29 DEBUG ----- output ----- 03:27:29 DEBUG { 03:27:29 DEBUG ----- output ----- 03:27:29 DEBUG { 03:27:29 DEBUG "result": [ 03:27:29 DEBUG { 03:27:29 DEBUG { 03:27:29 DEBUG "metric": { 03:27:29 DEBUG { 03:27:29 DEBUG "metric": { 03:27:29 DEBUG { 03:27:29 DEBUG "metric": { 03:27:29 DEBUG "status": "success", 03:27:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:27:29 DEBUG "status": "success", 03:27:29 DEBUG "instance": "10.142.0.107:9100" 03:27:29 DEBUG "metric": { 03:27:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 03:27:29 DEBUG "data": { 03:27:29 DEBUG }, 03:27:29 INFO [http_cmd]: http status code OK 03:27:29 DEBUG "data": { 03:27:29 DEBUG }, 03:27:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:27:29 DEBUG }, 03:27:29 DEBUG "resultType": "vector", 03:27:29 DEBUG "value": [ 03:27:29 DEBUG --- status code --- 03:27:29 DEBUG "resultType": "vector", 03:27:29 DEBUG "value": [ 03:27:29 DEBUG }, 03:27:29 DEBUG "value": [ 03:27:29 DEBUG "result": [ 03:27:29 DEBUG 03:27:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:27:29 DEBUG "result": [ 03:27:29 DEBUG 03:27:29 DEBUG "value": [ 03:27:29 DEBUG 03:27:29 DEBUG { 03:27:29 DEBUG 03:27:29 DEBUG --- http response --- 03:27:29 DEBUG { 03:27:29 DEBUG 03:27:29 DEBUG 03:27:29 DEBUG 03:27:29 DEBUG "metric": { 03:27:29 DEBUG 03:27:29 DEBUG "metric": { 03:27:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:27:29 DEBUG 03:27:29 DEBUG 03:27:29 DEBUG 03:27:29 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 03:27:29 DEBUG [...] 03:27:29 DEBUG "pod": "ds-idrepo-1" 03:27:29 DEBUG ----- output ----- 03:27:29 INFO [http_cmd]: http status code OK 03:27:29 DEBUG [...] 03:27:29 DEBUG 03:27:29 DEBUG [...] 03:27:29 DEBUG }, 03:27:29 DEBUG 03:27:29 DEBUG }, 03:27:29 DEBUG { 03:27:29 DEBUG --- status code --- 03:27:29 DEBUG 03:27:29 DEBUG [...] 03:27:29 DEBUG 03:27:29 DEBUG "value": [ 03:27:29 DEBUG 03:27:29 DEBUG "value": [ 03:27:29 DEBUG "status": "success", 03:27:29 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:27:29 DEBUG 03:27:29 DEBUG 03:27:29 DEBUG 03:27:29 DEBUG 03:27:29 DEBUG 03:27:29 DEBUG 03:27:29 DEBUG "data": { 03:27:29 DEBUG --- http response --- 03:27:29 DEBUG 03:27:29 DEBUG 03:27:29 DEBUG 03:27:29 DEBUG 03:27:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:27:29 DEBUG 03:27:29 DEBUG "resultType": "vector", 03:27:29 DEBUG "instance": "10.142.0.43:9100" 03:27:29 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:27:29 DEBUG 03:27:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:27:29 DEBUG 03:27:29 DEBUG }, 03:27:29 DEBUG 03:27:29 DEBUG "result": [ 03:27:29 DEBUG }, 03:27:29 DEBUG ----- output ----- 03:27:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:27:29 DEBUG }, 03:27:29 DEBUG [...] 03:27:29 DEBUG "value": [ 03:27:29 DEBUG [...] 03:27:29 DEBUG { 03:27:29 DEBUG "value": [ 03:27:29 DEBUG { 03:27:29 DEBUG }, 03:27:29 DEBUG "value": [ 03:27:29 DEBUG 03:27:29 DEBUG 1713061649, 03:27:29 DEBUG 03:27:29 DEBUG "metric": { 03:27:29 DEBUG 1713061649, 03:27:29 DEBUG "status": "success", 03:27:29 DEBUG "value": [ 03:27:29 DEBUG 1713061649, 03:27:29 DEBUG 03:27:29 DEBUG "0.015" 03:27:29 DEBUG 03:27:29 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 03:27:29 DEBUG "5452.533333333334" 03:27:29 DEBUG "data": { 03:27:29 DEBUG 1713061649, 03:27:29 DEBUG "231936000" 03:27:29 DEBUG 03:27:29 DEBUG ] 03:27:29 DEBUG 03:27:29 DEBUG }, 03:27:29 DEBUG ] 03:27:29 DEBUG "resultType": "vector", 03:27:29 DEBUG "103827.5" 03:27:29 DEBUG ] 03:27:29 DEBUG "pod": "lodemon-766dc9b5df-2jmd4" 03:27:29 DEBUG } 03:27:29 DEBUG "pod": "ds-idrepo-0" 03:27:29 DEBUG "value": [ 03:27:29 DEBUG } 03:27:29 DEBUG "result": [ 03:27:29 DEBUG ] 03:27:29 DEBUG } 03:27:29 DEBUG }, 03:27:29 DEBUG ] 03:27:29 DEBUG }, 03:27:29 DEBUG 03:27:29 DEBUG ] 03:27:29 DEBUG { 03:27:29 DEBUG } 03:27:29 DEBUG ] 03:27:29 DEBUG "value": [ 03:27:29 DEBUG } 03:27:29 DEBUG "value": [ 03:27:29 DEBUG 03:27:29 DEBUG } 03:27:29 DEBUG "metric": { 03:27:29 DEBUG ] 03:27:29 DEBUG } 03:27:29 DEBUG 1713061649, 03:27:29 DEBUG } 03:27:29 DEBUG 1713061649, 03:27:29 DEBUG 03:27:29 DEBUG } 03:27:29 DEBUG "pod": "am-7d597cf7d6-8ctft" 03:27:29 DEBUG } 03:27:29 DEBUG } 03:27:29 DEBUG "0" 03:27:29 DEBUG "0" 03:27:29 DEBUG [...] 03:27:29 DEBUG }, 03:27:29 DEBUG } 03:27:29 DEBUG ] 03:27:29 DEBUG ] 03:27:29 DEBUG 03:27:29 DEBUG "value": [ 03:27:29 DEBUG } 03:27:29 DEBUG } 03:27:29 DEBUG 03:27:29 DEBUG 03:27:29 DEBUG ] 03:27:29 DEBUG ] 03:27:29 DEBUG 03:27:29 DEBUG 03:27:29 DEBUG } 03:27:29 DEBUG } 03:27:29 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:27:29 DEBUG 03:27:29 DEBUG } 03:27:29 DEBUG } 03:27:29 DEBUG }, 03:27:29 DEBUG [...] 03:27:29 DEBUG "value": [ 03:27:29 DEBUG 03:27:29 DEBUG 1713061649, 03:27:29 DEBUG 03:27:29 DEBUG "0.5993655648653354" 03:27:29 DEBUG 03:27:29 DEBUG ] 03:27:29 DEBUG "pod": "am-7d597cf7d6-9wz9h" 03:27:29 DEBUG } 03:27:29 DEBUG }, 03:27:29 DEBUG ] 03:27:29 DEBUG "value": [ 03:27:29 DEBUG } 03:27:29 DEBUG 1713061649, 03:27:29 DEBUG } 03:27:29 DEBUG "0" 03:27:29 DEBUG ] 03:27:29 DEBUG } 03:27:29 DEBUG ] 03:27:29 DEBUG } 03:27:29 DEBUG } 03:27:29 INFO Finished: True 03:27:29 INFO Waiting for threads to register finish flag 03:28:29 INFO Done. Have a nice day! :) 127.0.0.1 - - [14/Apr/2024 03:28:29] "GET /monitoring/stop HTTP/1.1" 200 - 03:28:32 WARNING File /tmp/lodemon_data-20240414_031429/node_disk_read_bytes_total.json does not exist. Skipping graph generation... WARNING:bokeh.core.validation.check:W-1000 (MISSING_RENDERERS): Plot has no renderers: figure(id='p6511', ...) 127.0.0.1 - - [14/Apr/2024 03:28:36] "GET /monitoring/process HTTP/1.1" 200 -